1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/circ_buf.h> 10 #include <linux/clk-provider.h> 11 #include <linux/clk.h> 12 #include <linux/crc32.h> 13 #include <linux/delay.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/etherdevice.h> 16 #include <linux/firmware/xlnx-zynqmp.h> 17 #include <linux/inetdevice.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/io.h> 21 #include <linux/iopoll.h> 22 #include <linux/ip.h> 23 #include <linux/kernel.h> 24 #include <linux/module.h> 25 #include <linux/moduleparam.h> 26 #include <linux/netdevice.h> 27 #include <linux/of.h> 28 #include <linux/of_mdio.h> 29 #include <linux/of_net.h> 30 #include <linux/phy/phy.h> 31 #include <linux/phylink.h> 32 #include <linux/platform_device.h> 33 #include <linux/pm_runtime.h> 34 #include <linux/ptp_classify.h> 35 #include <linux/reset.h> 36 #include <linux/slab.h> 37 #include <linux/tcp.h> 38 #include <linux/types.h> 39 #include <linux/udp.h> 40 #include <linux/gcd.h> 41 #include <net/pkt_sched.h> 42 #include "macb.h" 43 44 /* This structure is only used for MACB on SiFive FU540 devices */ 45 struct sifive_fu540_macb_mgmt { 46 void __iomem *reg; 47 unsigned long rate; 48 struct clk_hw hw; 49 }; 50 51 #define MACB_RX_BUFFER_SIZE 128 52 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 53 #define RX_BUFFER_MAX (0xFF * RX_BUFFER_MULTIPLE) /* 16320 bytes */ 54 55 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 56 #define MIN_RX_RING_SIZE 64 57 #define MAX_RX_RING_SIZE 8192 58 59 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 60 #define MIN_TX_RING_SIZE 64 61 #define MAX_TX_RING_SIZE 4096 62 63 /* level of occupied TX descriptors under which we wake up TX process */ 64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->ctx->tx_ring_size / 4) 65 66 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 67 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 68 | MACB_BIT(ISR_RLE) \ 69 | MACB_BIT(TXERR)) 70 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 71 | MACB_BIT(TXUBR)) 72 73 #define MACB_INT_MISC_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(RXUBR) | \ 74 MACB_BIT(ISR_ROVR) | MACB_BIT(HRESP) | \ 75 GEM_BIT(WOL) | MACB_BIT(WOL)) 76 77 /* Max length of transmit frame must be a multiple of 8 bytes */ 78 #define MACB_TX_LEN_ALIGN 8 79 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 80 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a 81 * false amba_error in TX path from the DMA assuming there is not enough 82 * space in the SRAM (16KB) even when there is. 83 */ 84 #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) 85 86 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 87 #define MACB_NETIF_LSO NETIF_F_TSO 88 89 #define MACB_WOL_ENABLED BIT(0) 90 91 #define HS_SPEED_10000M 4 92 #define MACB_SERDES_RATE_10G 1 93 94 /* Graceful stop timeouts in us. We should allow up to 95 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 96 */ 97 #define MACB_HALT_TIMEOUT 14000 98 #define MACB_PM_TIMEOUT 100 /* ms */ 99 100 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 101 102 /* DMA buffer descriptor might be different size 103 * depends on hardware configuration: 104 * 105 * 1. dma address width 32 bits: 106 * word 1: 32 bit address of Data Buffer 107 * word 2: control 108 * 109 * 2. dma address width 64 bits: 110 * word 1: 32 bit address of Data Buffer 111 * word 2: control 112 * word 3: upper 32 bit address of Data Buffer 113 * word 4: unused 114 * 115 * 3. dma address width 32 bits with hardware timestamping: 116 * word 1: 32 bit address of Data Buffer 117 * word 2: control 118 * word 3: timestamp word 1 119 * word 4: timestamp word 2 120 * 121 * 4. dma address width 64 bits with hardware timestamping: 122 * word 1: 32 bit address of Data Buffer 123 * word 2: control 124 * word 3: upper 32 bit address of Data Buffer 125 * word 4: unused 126 * word 5: timestamp word 1 127 * word 6: timestamp word 2 128 */ 129 static unsigned int macb_dma_desc_get_size(const struct macb_info *info) 130 { 131 unsigned int desc_size = sizeof(struct macb_dma_desc); 132 133 if (macb_dma64(info)) 134 desc_size += sizeof(struct macb_dma_desc_64); 135 if (macb_dma_ptp(info)) 136 desc_size += sizeof(struct macb_dma_desc_ptp); 137 138 return desc_size; 139 } 140 141 static unsigned int macb_adj_dma_desc_idx(struct macb_context *ctx, 142 unsigned int desc_idx) 143 { 144 return desc_idx * (1 + macb_dma64(ctx->info) + 145 macb_dma_ptp(ctx->info)); 146 } 147 148 static struct macb_dma_desc_64 *macb_64b_desc(struct macb_dma_desc *desc) 149 { 150 return (struct macb_dma_desc_64 *)((void *)desc 151 + sizeof(struct macb_dma_desc)); 152 } 153 154 /* Ring buffer accessors */ 155 static unsigned int macb_tx_ring_wrap(struct macb_context *ctx, 156 unsigned int index) 157 { 158 return index & (ctx->tx_ring_size - 1); 159 } 160 161 static struct macb_txq *macb_txq(struct macb_queue *queue) 162 { 163 struct macb *bp = queue->bp; 164 unsigned int q = queue - bp->queues; 165 166 return &bp->ctx->txq[q]; 167 } 168 169 static struct macb_rxq *macb_rxq(struct macb_queue *queue) 170 { 171 struct macb *bp = queue->bp; 172 unsigned int q = queue - bp->queues; 173 174 return &bp->ctx->rxq[q]; 175 } 176 177 static struct macb_dma_desc *macb_tx_desc(struct macb_context *ctx, 178 unsigned int q, 179 unsigned int index) 180 { 181 index = macb_tx_ring_wrap(ctx, index); 182 index = macb_adj_dma_desc_idx(ctx, index); 183 return &ctx->txq[q].ring[index]; 184 } 185 186 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 187 unsigned int index) 188 { 189 struct macb_txq *txq = macb_txq(queue); 190 191 return &txq->skb[macb_tx_ring_wrap(queue->bp->ctx, index)]; 192 } 193 194 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 195 { 196 struct macb_context *ctx = queue->bp->ctx; 197 struct macb_txq *txq = macb_txq(queue); 198 dma_addr_t offset; 199 200 offset = macb_tx_ring_wrap(ctx, index) * 201 macb_dma_desc_get_size(&queue->bp->info); 202 203 return txq->ring_dma + offset; 204 } 205 206 static unsigned int macb_rx_ring_wrap(struct macb_context *ctx, 207 unsigned int index) 208 { 209 return index & (ctx->rx_ring_size - 1); 210 } 211 212 static struct macb_dma_desc *macb_rx_desc(struct macb_context *ctx, 213 unsigned int q, unsigned int index) 214 { 215 index = macb_rx_ring_wrap(ctx, index); 216 index = macb_adj_dma_desc_idx(ctx, index); 217 return &ctx->rxq[q].ring[index]; 218 } 219 220 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 221 { 222 struct macb_context *ctx = queue->bp->ctx; 223 struct macb_rxq *rxq = macb_rxq(queue); 224 225 return rxq->buffers + ctx->rx_buffer_size * 226 macb_rx_ring_wrap(ctx, index); 227 } 228 229 /* I/O accessors */ 230 static u32 hw_readl_native(struct macb *bp, int offset) 231 { 232 return __raw_readl(bp->regs + offset); 233 } 234 235 static void hw_writel_native(struct macb *bp, int offset, u32 value) 236 { 237 __raw_writel(value, bp->regs + offset); 238 } 239 240 static u32 hw_readl(struct macb *bp, int offset) 241 { 242 return readl_relaxed(bp->regs + offset); 243 } 244 245 static void hw_writel(struct macb *bp, int offset, u32 value) 246 { 247 writel_relaxed(value, bp->regs + offset); 248 } 249 250 /* Find the CPU endianness by using the loopback bit of NCR register. When the 251 * CPU is in big endian we need to program swapped mode for management 252 * descriptor access. 253 */ 254 static bool hw_is_native_io(void __iomem *addr) 255 { 256 u32 value = MACB_BIT(LLB); 257 258 __raw_writel(value, addr + MACB_NCR); 259 value = __raw_readl(addr + MACB_NCR); 260 261 /* Write 0 back to disable everything */ 262 __raw_writel(0, addr + MACB_NCR); 263 264 return value == MACB_BIT(LLB); 265 } 266 267 static bool hw_is_gem(void __iomem *addr, bool native_io) 268 { 269 u32 id; 270 271 if (native_io) 272 id = __raw_readl(addr + MACB_MID); 273 else 274 id = readl_relaxed(addr + MACB_MID); 275 276 return MACB_BFEXT(IDNUM, id) >= 0x2; 277 } 278 279 static void macb_set_hwaddr(struct macb *bp) 280 { 281 u32 bottom; 282 u16 top; 283 284 bottom = get_unaligned_le32(bp->netdev->dev_addr); 285 macb_or_gem_writel(bp, SA1B, bottom); 286 top = get_unaligned_le16(bp->netdev->dev_addr + 4); 287 macb_or_gem_writel(bp, SA1T, top); 288 289 if (gem_has_ptp(&bp->info)) { 290 gem_writel(bp, RXPTPUNI, bottom); 291 gem_writel(bp, TXPTPUNI, bottom); 292 } 293 294 /* Clear unused address register sets */ 295 macb_or_gem_writel(bp, SA2B, 0); 296 macb_or_gem_writel(bp, SA2T, 0); 297 macb_or_gem_writel(bp, SA3B, 0); 298 macb_or_gem_writel(bp, SA3T, 0); 299 macb_or_gem_writel(bp, SA4B, 0); 300 macb_or_gem_writel(bp, SA4T, 0); 301 } 302 303 static void macb_get_hwaddr(struct macb *bp) 304 { 305 u32 bottom; 306 u16 top; 307 u8 addr[6]; 308 int i; 309 310 /* Check all 4 address register for valid address */ 311 for (i = 0; i < 4; i++) { 312 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 313 top = macb_or_gem_readl(bp, SA1T + i * 8); 314 315 addr[0] = bottom & 0xff; 316 addr[1] = (bottom >> 8) & 0xff; 317 addr[2] = (bottom >> 16) & 0xff; 318 addr[3] = (bottom >> 24) & 0xff; 319 addr[4] = top & 0xff; 320 addr[5] = (top >> 8) & 0xff; 321 322 if (is_valid_ether_addr(addr)) { 323 eth_hw_addr_set(bp->netdev, addr); 324 return; 325 } 326 } 327 328 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 329 eth_hw_addr_random(bp->netdev); 330 } 331 332 static int macb_mdio_wait_for_idle(struct macb *bp) 333 { 334 u32 val; 335 336 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 337 1, MACB_MDIO_TIMEOUT); 338 } 339 340 static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum) 341 { 342 struct macb *bp = bus->priv; 343 int status; 344 345 status = pm_runtime_resume_and_get(&bp->pdev->dev); 346 if (status < 0) 347 goto mdio_pm_exit; 348 349 status = macb_mdio_wait_for_idle(bp); 350 if (status < 0) 351 goto mdio_read_exit; 352 353 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 354 | MACB_BF(RW, MACB_MAN_C22_READ) 355 | MACB_BF(PHYA, mii_id) 356 | MACB_BF(REGA, regnum) 357 | MACB_BF(CODE, MACB_MAN_C22_CODE))); 358 359 status = macb_mdio_wait_for_idle(bp); 360 if (status < 0) 361 goto mdio_read_exit; 362 363 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 364 365 mdio_read_exit: 366 pm_runtime_put_autosuspend(&bp->pdev->dev); 367 mdio_pm_exit: 368 return status; 369 } 370 371 static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad, 372 int regnum) 373 { 374 struct macb *bp = bus->priv; 375 int status; 376 377 status = pm_runtime_get_sync(&bp->pdev->dev); 378 if (status < 0) { 379 pm_runtime_put_noidle(&bp->pdev->dev); 380 goto mdio_pm_exit; 381 } 382 383 status = macb_mdio_wait_for_idle(bp); 384 if (status < 0) 385 goto mdio_read_exit; 386 387 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 388 | MACB_BF(RW, MACB_MAN_C45_ADDR) 389 | MACB_BF(PHYA, mii_id) 390 | MACB_BF(REGA, devad & 0x1F) 391 | MACB_BF(DATA, regnum & 0xFFFF) 392 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 393 394 status = macb_mdio_wait_for_idle(bp); 395 if (status < 0) 396 goto mdio_read_exit; 397 398 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 399 | MACB_BF(RW, MACB_MAN_C45_READ) 400 | MACB_BF(PHYA, mii_id) 401 | MACB_BF(REGA, devad & 0x1F) 402 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 403 404 status = macb_mdio_wait_for_idle(bp); 405 if (status < 0) 406 goto mdio_read_exit; 407 408 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 409 410 mdio_read_exit: 411 pm_runtime_put_autosuspend(&bp->pdev->dev); 412 mdio_pm_exit: 413 return status; 414 } 415 416 static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum, 417 u16 value) 418 { 419 struct macb *bp = bus->priv; 420 int status; 421 422 status = pm_runtime_resume_and_get(&bp->pdev->dev); 423 if (status < 0) 424 goto mdio_pm_exit; 425 426 status = macb_mdio_wait_for_idle(bp); 427 if (status < 0) 428 goto mdio_write_exit; 429 430 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 431 | MACB_BF(RW, MACB_MAN_C22_WRITE) 432 | MACB_BF(PHYA, mii_id) 433 | MACB_BF(REGA, regnum) 434 | MACB_BF(CODE, MACB_MAN_C22_CODE) 435 | MACB_BF(DATA, value))); 436 437 status = macb_mdio_wait_for_idle(bp); 438 if (status < 0) 439 goto mdio_write_exit; 440 441 mdio_write_exit: 442 pm_runtime_put_autosuspend(&bp->pdev->dev); 443 mdio_pm_exit: 444 return status; 445 } 446 447 static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id, 448 int devad, int regnum, 449 u16 value) 450 { 451 struct macb *bp = bus->priv; 452 int status; 453 454 status = pm_runtime_get_sync(&bp->pdev->dev); 455 if (status < 0) { 456 pm_runtime_put_noidle(&bp->pdev->dev); 457 goto mdio_pm_exit; 458 } 459 460 status = macb_mdio_wait_for_idle(bp); 461 if (status < 0) 462 goto mdio_write_exit; 463 464 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 465 | MACB_BF(RW, MACB_MAN_C45_ADDR) 466 | MACB_BF(PHYA, mii_id) 467 | MACB_BF(REGA, devad & 0x1F) 468 | MACB_BF(DATA, regnum & 0xFFFF) 469 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 470 471 status = macb_mdio_wait_for_idle(bp); 472 if (status < 0) 473 goto mdio_write_exit; 474 475 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 476 | MACB_BF(RW, MACB_MAN_C45_WRITE) 477 | MACB_BF(PHYA, mii_id) 478 | MACB_BF(REGA, devad & 0x1F) 479 | MACB_BF(CODE, MACB_MAN_C45_CODE) 480 | MACB_BF(DATA, value))); 481 482 status = macb_mdio_wait_for_idle(bp); 483 if (status < 0) 484 goto mdio_write_exit; 485 486 mdio_write_exit: 487 pm_runtime_put_autosuspend(&bp->pdev->dev); 488 mdio_pm_exit: 489 return status; 490 } 491 492 static void macb_init_buffers(struct macb *bp) 493 { 494 struct macb_queue *queue; 495 struct macb_rxq *rxq; 496 struct macb_txq *txq; 497 unsigned int q; 498 499 /* Single register for all queues' high 32 bits. */ 500 if (macb_dma64(&bp->info)) { 501 rxq = &bp->ctx->rxq[0]; 502 txq = &bp->ctx->txq[0]; 503 macb_writel(bp, RBQPH, upper_32_bits(rxq->ring_dma)); 504 macb_writel(bp, TBQPH, upper_32_bits(txq->ring_dma)); 505 } 506 507 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 508 rxq = &bp->ctx->rxq[q]; 509 txq = &bp->ctx->txq[q]; 510 queue_writel(queue, RBQP, lower_32_bits(rxq->ring_dma)); 511 queue_writel(queue, TBQP, lower_32_bits(txq->ring_dma)); 512 } 513 } 514 515 /** 516 * macb_set_tx_clk() - Set a clock to a new frequency 517 * @bp: pointer to struct macb 518 * @speed: New frequency in Hz 519 */ 520 static void macb_set_tx_clk(struct macb *bp, int speed) 521 { 522 long ferr, rate, rate_rounded; 523 524 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) 525 return; 526 527 /* In case of MII the PHY is the clock master */ 528 if (bp->phy_interface == PHY_INTERFACE_MODE_MII) 529 return; 530 531 rate = rgmii_clock(speed); 532 if (rate < 0) 533 return; 534 535 rate_rounded = clk_round_rate(bp->tx_clk, rate); 536 if (rate_rounded < 0) 537 return; 538 539 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 540 * is not satisfied. 541 */ 542 ferr = abs(rate_rounded - rate); 543 ferr = DIV_ROUND_UP(ferr, rate / 100000); 544 if (ferr > 5) 545 netdev_warn(bp->netdev, 546 "unable to generate target frequency: %ld Hz\n", 547 rate); 548 549 if (clk_set_rate(bp->tx_clk, rate_rounded)) 550 netdev_err(bp->netdev, "adjusting tx_clk failed.\n"); 551 } 552 553 static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode, 554 phy_interface_t interface, int speed, 555 int duplex) 556 { 557 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 558 u32 config; 559 560 config = gem_readl(bp, USX_CONTROL); 561 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); 562 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); 563 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); 564 config |= GEM_BIT(TX_EN); 565 gem_writel(bp, USX_CONTROL, config); 566 } 567 568 static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, 569 unsigned int neg_mode, 570 struct phylink_link_state *state) 571 { 572 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 573 u32 val; 574 575 state->speed = SPEED_10000; 576 state->duplex = 1; 577 state->an_complete = 1; 578 579 val = gem_readl(bp, USX_STATUS); 580 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); 581 val = gem_readl(bp, NCFGR); 582 if (val & GEM_BIT(PAE)) 583 state->pause = MLO_PAUSE_RX; 584 } 585 586 static int macb_usx_pcs_config(struct phylink_pcs *pcs, 587 unsigned int neg_mode, 588 phy_interface_t interface, 589 const unsigned long *advertising, 590 bool permit_pause_to_mac) 591 { 592 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 593 594 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | 595 GEM_BIT(SIGNAL_OK)); 596 597 return 0; 598 } 599 600 static unsigned int macb_pcs_inband_caps(struct phylink_pcs *pcs, 601 phy_interface_t interface) 602 { 603 return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE; 604 } 605 606 static void macb_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode, 607 struct phylink_link_state *state) 608 { 609 struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs); 610 u16 bmsr, lpa; 611 612 bmsr = gem_readl(bp, PCSSTS); 613 lpa = gem_readl(bp, PCSANLPBASE); 614 phylink_mii_c22_pcs_decode_state(state, neg_mode, bmsr, lpa); 615 } 616 617 static void macb_pcs_an_restart(struct phylink_pcs *pcs) 618 { 619 struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs); 620 u32 old, new; 621 622 old = gem_readl(bp, PCSCNTRL); 623 new = old | BMCR_ANRESTART; 624 gem_writel(bp, PCSCNTRL, new); 625 } 626 627 static int macb_pcs_config(struct phylink_pcs *pcs, 628 unsigned int neg_mode, 629 phy_interface_t interface, 630 const unsigned long *advertising, 631 bool permit_pause_to_mac) 632 { 633 struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs); 634 u32 old, new; 635 int ret = 0; 636 637 old = gem_readl(bp, PCSANADV); 638 new = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); 639 if (new != -EINVAL && old != new) { 640 /* pcs_config() is supposed to return 1 if AN advertisement 641 * has changed 642 */ 643 ret = 1; 644 gem_writel(bp, PCSANADV, new); 645 } 646 647 /* Disable AN if it's not to be used, enable otherwise. 648 * Must be written after PCSSEL is set in NCFGR which is done in 649 * macb_mac_config(), otherwise writes will not take effect. 650 */ 651 old = gem_readl(bp, PCSCNTRL); 652 if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED) 653 new = old | BMCR_ANENABLE; 654 else 655 new = old & ~BMCR_ANENABLE; 656 if (old != new) 657 gem_writel(bp, PCSCNTRL, new); 658 659 return ret; 660 } 661 662 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { 663 .pcs_get_state = macb_usx_pcs_get_state, 664 .pcs_config = macb_usx_pcs_config, 665 .pcs_link_up = macb_usx_pcs_link_up, 666 }; 667 668 static const struct phylink_pcs_ops macb_phylink_pcs_ops = { 669 .pcs_inband_caps = macb_pcs_inband_caps, 670 .pcs_get_state = macb_pcs_get_state, 671 .pcs_an_restart = macb_pcs_an_restart, 672 .pcs_config = macb_pcs_config, 673 }; 674 675 static bool macb_tx_lpi_set(struct macb *bp, bool enable) 676 { 677 u32 old, ncr; 678 679 lockdep_assert_held(&bp->lock); 680 681 ncr = macb_readl(bp, NCR); 682 old = ncr; 683 if (enable) 684 ncr |= GEM_BIT(TXLPIEN); 685 else 686 ncr &= ~GEM_BIT(TXLPIEN); 687 if (old != ncr) 688 macb_writel(bp, NCR, ncr); 689 690 return old != ncr; 691 } 692 693 static bool macb_tx_all_queues_idle(struct macb *bp) 694 { 695 struct macb_txq *txq; 696 unsigned int q; 697 698 for (q = 0; q < bp->num_queues; ++q) { 699 txq = &bp->ctx->txq[q]; 700 if (READ_ONCE(txq->head) != READ_ONCE(txq->tail)) 701 return false; 702 } 703 return true; 704 } 705 706 static void macb_tx_lpi_work_fn(struct work_struct *work) 707 { 708 struct macb *bp = container_of(work, struct macb, tx_lpi_work.work); 709 unsigned long flags; 710 711 spin_lock_irqsave(&bp->lock, flags); 712 if (bp->eee_active && macb_tx_all_queues_idle(bp)) 713 macb_tx_lpi_set(bp, true); 714 spin_unlock_irqrestore(&bp->lock, flags); 715 } 716 717 static void macb_tx_lpi_schedule(struct macb *bp) 718 { 719 if (bp->eee_active) 720 mod_delayed_work(system_wq, &bp->tx_lpi_work, 721 usecs_to_jiffies(bp->tx_lpi_timer)); 722 } 723 724 /* Wake from LPI before transmitting. The MAC must deassert TXLPIEN 725 * and wait for the PHY to exit LPI before any frame can be sent. 726 * IEEE 802.3az Tw_sys is ~17us for 1000BASE-T, ~30us for 100BASE-TX; 727 * we use a conservative 50us. 728 */ 729 static void macb_tx_lpi_wake(struct macb *bp) 730 { 731 lockdep_assert_held(&bp->lock); 732 733 if (!bp->eee_active) 734 return; 735 736 if (!macb_tx_lpi_set(bp, false)) 737 return; 738 739 cancel_delayed_work(&bp->tx_lpi_work); 740 udelay(50); 741 } 742 743 static void macb_mac_disable_tx_lpi(struct phylink_config *config) 744 { 745 struct net_device *netdev = to_net_dev(config->dev); 746 struct macb *bp = netdev_priv(netdev); 747 unsigned long flags; 748 749 mutex_lock(&bp->mac_cfg_lock); 750 751 cancel_delayed_work_sync(&bp->tx_lpi_work); 752 753 spin_lock_irqsave(&bp->lock, flags); 754 bp->eee_active = false; 755 macb_tx_lpi_set(bp, false); 756 spin_unlock_irqrestore(&bp->lock, flags); 757 758 mutex_unlock(&bp->mac_cfg_lock); 759 } 760 761 static void macb_txp_lpi_initial_defer(struct macb *bp) 762 { 763 lockdep_assert_held(&bp->mac_cfg_lock); 764 765 /* Defer initial LPI entry by 1 second after link-up per 766 * IEEE 802.3az section 22.7a. 767 */ 768 mod_delayed_work(system_wq, &bp->tx_lpi_work, msecs_to_jiffies(1000)); 769 } 770 771 static int macb_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 772 bool tx_clk_stop) 773 { 774 struct net_device *netdev = to_net_dev(config->dev); 775 struct macb *bp = netdev_priv(netdev); 776 unsigned long flags; 777 778 mutex_lock(&bp->mac_cfg_lock); 779 780 spin_lock_irqsave(&bp->lock, flags); 781 bp->tx_lpi_timer = timer; 782 bp->eee_active = true; 783 spin_unlock_irqrestore(&bp->lock, flags); 784 785 macb_txp_lpi_initial_defer(bp); 786 787 mutex_unlock(&bp->mac_cfg_lock); 788 789 return 0; 790 } 791 792 static void macb_mac_config(struct phylink_config *config, unsigned int mode, 793 const struct phylink_link_state *state) 794 { 795 struct net_device *netdev = to_net_dev(config->dev); 796 struct macb *bp = netdev_priv(netdev); 797 unsigned long flags; 798 u32 old_ctrl, ctrl; 799 u32 old_ncr, ncr; 800 801 mutex_lock(&bp->mac_cfg_lock); 802 spin_lock_irqsave(&bp->lock, flags); 803 804 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 805 old_ncr = ncr = macb_or_gem_readl(bp, NCR); 806 807 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { 808 if (state->interface == PHY_INTERFACE_MODE_RMII) 809 ctrl |= MACB_BIT(RM9200_RMII); 810 } else if (macb_is_gem(&bp->info)) { 811 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 812 ncr &= ~GEM_BIT(ENABLE_HS_MAC); 813 814 if (state->interface == PHY_INTERFACE_MODE_1000BASEX) { 815 ctrl |= GEM_BIT(PCSSEL); 816 } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { 817 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 818 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) { 819 ctrl |= GEM_BIT(PCSSEL); 820 ncr |= GEM_BIT(ENABLE_HS_MAC); 821 } else if (bp->caps & MACB_CAPS_MIIONRGMII && 822 bp->phy_interface == PHY_INTERFACE_MODE_MII) { 823 ncr |= MACB_BIT(MIIONRGMII); 824 } 825 } 826 827 /* Apply the new configuration, if any */ 828 if (old_ctrl ^ ctrl) 829 macb_or_gem_writel(bp, NCFGR, ctrl); 830 831 if (old_ncr ^ ncr) 832 macb_or_gem_writel(bp, NCR, ncr); 833 834 spin_unlock_irqrestore(&bp->lock, flags); 835 mutex_unlock(&bp->mac_cfg_lock); 836 } 837 838 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 839 phy_interface_t interface) 840 { 841 struct net_device *netdev = to_net_dev(config->dev); 842 struct macb *bp = netdev_priv(netdev); 843 struct macb_queue *queue; 844 unsigned int q; 845 u32 ctrl; 846 847 mutex_lock(&bp->mac_cfg_lock); 848 849 bp->link_up = false; 850 851 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 852 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 853 queue_writel(queue, IDR, 854 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 855 856 /* Disable Rx and Tx */ 857 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 858 macb_writel(bp, NCR, ctrl); 859 860 netif_tx_stop_all_queues(netdev); 861 862 mutex_unlock(&bp->mac_cfg_lock); 863 } 864 865 /* Use juggling algorithm to left rotate tx ring and tx skb array */ 866 static void gem_shuffle_tx_one_ring(struct macb_queue *queue) 867 { 868 unsigned int head, tail, count, ring_size, desc_size; 869 struct macb_tx_skb tx_skb, *skb_curr, *skb_next; 870 struct macb_dma_desc *desc_curr, *desc_next; 871 unsigned int q = queue - queue->bp->queues; 872 unsigned int i, cycles, shift, curr, next; 873 struct macb_context *ctx = queue->bp->ctx; 874 struct macb_txq *txq = macb_txq(queue); 875 unsigned char desc[24]; 876 unsigned long flags; 877 878 desc_size = macb_dma_desc_get_size(&queue->bp->info); 879 880 if (WARN_ON_ONCE(desc_size > ARRAY_SIZE(desc))) 881 return; 882 883 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 884 head = txq->head; 885 tail = txq->tail; 886 ring_size = ctx->tx_ring_size; 887 count = CIRC_CNT(head, tail, ring_size); 888 889 if (!(tail % ring_size)) 890 goto unlock; 891 892 if (!count) { 893 txq->head = 0; 894 txq->tail = 0; 895 goto unlock; 896 } 897 898 shift = tail % ring_size; 899 cycles = gcd(ring_size, shift); 900 901 for (i = 0; i < cycles; i++) { 902 memcpy(&desc, macb_tx_desc(ctx, q, i), desc_size); 903 memcpy(&tx_skb, macb_tx_skb(queue, i), 904 sizeof(struct macb_tx_skb)); 905 906 curr = i; 907 next = (curr + shift) % ring_size; 908 909 while (next != i) { 910 desc_curr = macb_tx_desc(ctx, q, curr); 911 desc_next = macb_tx_desc(ctx, q, next); 912 913 memcpy(desc_curr, desc_next, desc_size); 914 915 if (next == ring_size - 1) 916 desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); 917 if (curr == ring_size - 1) 918 desc_curr->ctrl |= MACB_BIT(TX_WRAP); 919 920 skb_curr = macb_tx_skb(queue, curr); 921 skb_next = macb_tx_skb(queue, next); 922 memcpy(skb_curr, skb_next, sizeof(struct macb_tx_skb)); 923 924 curr = next; 925 next = (curr + shift) % ring_size; 926 } 927 928 desc_curr = macb_tx_desc(ctx, q, curr); 929 memcpy(desc_curr, &desc, desc_size); 930 if (i == ring_size - 1) 931 desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); 932 if (curr == ring_size - 1) 933 desc_curr->ctrl |= MACB_BIT(TX_WRAP); 934 memcpy(macb_tx_skb(queue, curr), &tx_skb, 935 sizeof(struct macb_tx_skb)); 936 } 937 938 txq->head = count; 939 txq->tail = 0; 940 941 /* Make descriptor updates visible to hardware */ 942 wmb(); 943 944 unlock: 945 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 946 } 947 948 /* Rotate the queue so that the tail is at index 0 */ 949 static void gem_shuffle_tx_rings(struct macb *bp) 950 { 951 struct macb_queue *queue; 952 unsigned int q; 953 954 for (q = 0, queue = bp->queues; q < bp->num_queues; q++, queue++) 955 gem_shuffle_tx_one_ring(queue); 956 } 957 958 static void macb_mac_link_up(struct phylink_config *config, 959 struct phy_device *phydev, 960 unsigned int mode, phy_interface_t interface, 961 int speed, int duplex, 962 bool tx_pause, bool rx_pause) 963 { 964 struct net_device *netdev = to_net_dev(config->dev); 965 struct macb *bp = netdev_priv(netdev); 966 struct macb_queue *queue; 967 unsigned long flags; 968 unsigned int q; 969 u32 ctrl; 970 971 mutex_lock(&bp->mac_cfg_lock); 972 spin_lock_irqsave(&bp->lock, flags); 973 974 bp->link_up = true; 975 976 ctrl = macb_or_gem_readl(bp, NCFGR); 977 978 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 979 980 if (speed == SPEED_100) 981 ctrl |= MACB_BIT(SPD); 982 983 if (duplex) 984 ctrl |= MACB_BIT(FD); 985 986 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 987 ctrl &= ~MACB_BIT(PAE); 988 if (macb_is_gem(&bp->info)) { 989 ctrl &= ~GEM_BIT(GBE); 990 991 if (speed == SPEED_1000) 992 ctrl |= GEM_BIT(GBE); 993 } 994 995 if (rx_pause) 996 ctrl |= MACB_BIT(PAE); 997 998 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 999 queue_writel(queue, IER, 1000 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 1001 } 1002 } 1003 1004 macb_or_gem_writel(bp, NCFGR, ctrl); 1005 1006 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER) 1007 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, 1008 gem_readl(bp, HS_MAC_CONFIG))); 1009 1010 spin_unlock_irqrestore(&bp->lock, flags); 1011 1012 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 1013 macb_set_tx_clk(bp, speed); 1014 gem_shuffle_tx_rings(bp); 1015 } 1016 1017 /* Enable Rx and Tx; Enable PTP unicast */ 1018 ctrl = macb_readl(bp, NCR); 1019 if (gem_has_ptp(&bp->info)) 1020 ctrl |= MACB_BIT(PTPUNI); 1021 1022 macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); 1023 1024 netif_tx_wake_all_queues(netdev); 1025 1026 mutex_unlock(&bp->mac_cfg_lock); 1027 } 1028 1029 static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, 1030 phy_interface_t interface) 1031 { 1032 struct net_device *netdev = to_net_dev(config->dev); 1033 struct macb *bp = netdev_priv(netdev); 1034 1035 if (interface == PHY_INTERFACE_MODE_10GBASER) 1036 return &bp->phylink_usx_pcs; 1037 else if (interface == PHY_INTERFACE_MODE_1000BASEX || 1038 interface == PHY_INTERFACE_MODE_SGMII) 1039 return &bp->phylink_sgmii_pcs; 1040 else 1041 return NULL; 1042 } 1043 1044 static const struct phylink_mac_ops macb_phylink_ops = { 1045 .mac_select_pcs = macb_mac_select_pcs, 1046 .mac_config = macb_mac_config, 1047 .mac_link_down = macb_mac_link_down, 1048 .mac_link_up = macb_mac_link_up, 1049 .mac_disable_tx_lpi = macb_mac_disable_tx_lpi, 1050 .mac_enable_tx_lpi = macb_mac_enable_tx_lpi, 1051 }; 1052 1053 static bool macb_phy_handle_exists(struct device_node *dn) 1054 { 1055 dn = of_parse_phandle(dn, "phy-handle", 0); 1056 of_node_put(dn); 1057 return dn != NULL; 1058 } 1059 1060 static int macb_phylink_connect(struct macb *bp) 1061 { 1062 struct device_node *dn = bp->pdev->dev.of_node; 1063 struct net_device *netdev = bp->netdev; 1064 struct phy_device *phydev; 1065 int ret; 1066 1067 if (dn) 1068 ret = phylink_of_phy_connect(bp->phylink, dn, 0); 1069 1070 if (!dn || (ret && !macb_phy_handle_exists(dn))) { 1071 phydev = phy_find_first(bp->mii_bus); 1072 if (!phydev) { 1073 netdev_err(netdev, "no PHY found\n"); 1074 return -ENXIO; 1075 } 1076 1077 /* attach the mac to the phy */ 1078 ret = phylink_connect_phy(bp->phylink, phydev); 1079 } 1080 1081 if (ret) { 1082 netdev_err(netdev, "Could not attach PHY (%d)\n", ret); 1083 return ret; 1084 } 1085 1086 phylink_start(bp->phylink); 1087 1088 return 0; 1089 } 1090 1091 static void macb_get_pcs_fixed_state(struct phylink_config *config, 1092 struct phylink_link_state *state) 1093 { 1094 struct net_device *netdev = to_net_dev(config->dev); 1095 struct macb *bp = netdev_priv(netdev); 1096 1097 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; 1098 } 1099 1100 /* based on au1000_eth. c*/ 1101 static int macb_mii_probe(struct net_device *netdev) 1102 { 1103 struct macb *bp = netdev_priv(netdev); 1104 1105 bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; 1106 bp->phylink_sgmii_pcs.poll = true; 1107 bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; 1108 1109 bp->phylink_config.dev = &netdev->dev; 1110 bp->phylink_config.type = PHYLINK_NETDEV; 1111 bp->phylink_config.mac_managed_pm = true; 1112 1113 if (bp->phy_interface == PHY_INTERFACE_MODE_1000BASEX || 1114 bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 1115 bp->phylink_config.poll_fixed_state = true; 1116 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; 1117 /* The PCSAUTONEG bit in PCSCNTRL is on out of reset. Setting 1118 * default_an_inband to true tells phylink to turn it off only 1119 * if necessary (e.g. a fixed link or a PHY that doesn't support 1120 * inband). 1121 */ 1122 bp->phylink_config.default_an_inband = true; 1123 } 1124 1125 bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | 1126 MAC_10 | MAC_100; 1127 1128 __set_bit(PHY_INTERFACE_MODE_MII, 1129 bp->phylink_config.supported_interfaces); 1130 __set_bit(PHY_INTERFACE_MODE_RMII, 1131 bp->phylink_config.supported_interfaces); 1132 1133 /* Determine what modes are supported */ 1134 if (macb_is_gem(&bp->info) && 1135 (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { 1136 bp->phylink_config.mac_capabilities |= MAC_1000FD; 1137 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 1138 bp->phylink_config.mac_capabilities |= MAC_1000HD; 1139 1140 __set_bit(PHY_INTERFACE_MODE_GMII, 1141 bp->phylink_config.supported_interfaces); 1142 phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); 1143 1144 if (bp->caps & MACB_CAPS_PCS) { 1145 __set_bit(PHY_INTERFACE_MODE_1000BASEX, 1146 bp->phylink_config.supported_interfaces); 1147 __set_bit(PHY_INTERFACE_MODE_SGMII, 1148 bp->phylink_config.supported_interfaces); 1149 } 1150 1151 if (bp->caps & MACB_CAPS_HIGH_SPEED) { 1152 __set_bit(PHY_INTERFACE_MODE_10GBASER, 1153 bp->phylink_config.supported_interfaces); 1154 bp->phylink_config.mac_capabilities |= MAC_10000FD; 1155 } 1156 } 1157 1158 /* Configure EEE LPI if supported */ 1159 if (bp->caps & MACB_CAPS_EEE) { 1160 __set_bit(PHY_INTERFACE_MODE_MII, 1161 bp->phylink_config.lpi_interfaces); 1162 __set_bit(PHY_INTERFACE_MODE_GMII, 1163 bp->phylink_config.lpi_interfaces); 1164 phy_interface_set_rgmii(bp->phylink_config.lpi_interfaces); 1165 bp->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD; 1166 bp->phylink_config.lpi_timer_default = 250000; 1167 bp->phylink_config.eee_enabled_default = true; 1168 } 1169 1170 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 1171 bp->phy_interface, &macb_phylink_ops); 1172 if (IS_ERR(bp->phylink)) { 1173 netdev_err(netdev, "Could not create a phylink instance (%ld)\n", 1174 PTR_ERR(bp->phylink)); 1175 return PTR_ERR(bp->phylink); 1176 } 1177 1178 return 0; 1179 } 1180 1181 static int macb_mdiobus_register(struct macb *bp, struct device_node *mdio_np) 1182 { 1183 struct device_node *child, *np = bp->pdev->dev.of_node; 1184 1185 /* If we have a child named mdio, probe it instead of looking for PHYs 1186 * directly under the MAC node 1187 */ 1188 if (mdio_np) 1189 return of_mdiobus_register(bp->mii_bus, mdio_np); 1190 1191 /* Only create the PHY from the device tree if at least one PHY is 1192 * described. Otherwise scan the entire MDIO bus. We do this to support 1193 * old device tree that did not follow the best practices and did not 1194 * describe their network PHYs. 1195 */ 1196 for_each_available_child_of_node(np, child) 1197 if (of_mdiobus_child_is_phy(child)) { 1198 /* The loop increments the child refcount, 1199 * decrement it before returning. 1200 */ 1201 of_node_put(child); 1202 1203 return of_mdiobus_register(bp->mii_bus, np); 1204 } 1205 1206 return mdiobus_register(bp->mii_bus); 1207 } 1208 1209 static int macb_mii_init(struct macb *bp) 1210 { 1211 struct device_node *mdio_np, *np = bp->pdev->dev.of_node; 1212 int err = -ENXIO; 1213 1214 /* With fixed-link, we don't need to register the MDIO bus, 1215 * except if we have a child named "mdio" in the device tree. 1216 * In that case, some devices may be attached to the MACB's MDIO bus. 1217 */ 1218 mdio_np = of_get_child_by_name(np, "mdio"); 1219 if (!mdio_np && of_phy_is_fixed_link(np)) 1220 return macb_mii_probe(bp->netdev); 1221 1222 /* Enable management port */ 1223 macb_writel(bp, NCR, MACB_BIT(MPE)); 1224 1225 bp->mii_bus = mdiobus_alloc(); 1226 if (!bp->mii_bus) { 1227 err = -ENOMEM; 1228 goto err_out; 1229 } 1230 1231 bp->mii_bus->name = "MACB_mii_bus"; 1232 bp->mii_bus->read = &macb_mdio_read_c22; 1233 bp->mii_bus->write = &macb_mdio_write_c22; 1234 bp->mii_bus->read_c45 = &macb_mdio_read_c45; 1235 bp->mii_bus->write_c45 = &macb_mdio_write_c45; 1236 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 1237 bp->pdev->name, bp->pdev->id); 1238 bp->mii_bus->priv = bp; 1239 bp->mii_bus->parent = &bp->pdev->dev; 1240 1241 dev_set_drvdata(&bp->netdev->dev, bp->mii_bus); 1242 1243 err = macb_mdiobus_register(bp, mdio_np); 1244 if (err) 1245 goto err_out_free_mdiobus; 1246 1247 err = macb_mii_probe(bp->netdev); 1248 if (err) 1249 goto err_out_unregister_bus; 1250 1251 of_node_put(mdio_np); 1252 1253 return 0; 1254 1255 err_out_unregister_bus: 1256 mdiobus_unregister(bp->mii_bus); 1257 err_out_free_mdiobus: 1258 mdiobus_free(bp->mii_bus); 1259 err_out: 1260 of_node_put(mdio_np); 1261 1262 return err; 1263 } 1264 1265 static void macb_update_stats(struct macb *bp) 1266 { 1267 u64 *p = &bp->hw_stats.macb.rx_pause_frames; 1268 u64 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 1269 int offset = MACB_PFR; 1270 1271 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 1272 1273 for (; p < end; p++, offset += 4) 1274 *p += bp->macb_reg_readl(bp, offset); 1275 } 1276 1277 static int macb_halt_tx(struct macb *bp) 1278 { 1279 u32 status; 1280 1281 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 1282 1283 /* Poll TSR until TGO is cleared or timeout. */ 1284 return read_poll_timeout_atomic(macb_readl, status, 1285 !(status & MACB_BIT(TGO)), 1286 250, MACB_HALT_TIMEOUT, false, 1287 bp, TSR); 1288 } 1289 1290 static void macb_tx_unmap(const struct macb_info *info, 1291 struct macb_tx_skb *tx_skb, int budget) 1292 { 1293 if (tx_skb->mapping) { 1294 if (tx_skb->mapped_as_page) 1295 dma_unmap_page(&info->pdev->dev, tx_skb->mapping, 1296 tx_skb->size, DMA_TO_DEVICE); 1297 else 1298 dma_unmap_single(&info->pdev->dev, tx_skb->mapping, 1299 tx_skb->size, DMA_TO_DEVICE); 1300 tx_skb->mapping = 0; 1301 } 1302 1303 if (tx_skb->skb) { 1304 dev_consume_skb_any(tx_skb->skb); 1305 tx_skb->skb = NULL; 1306 } 1307 } 1308 1309 static void macb_set_addr(const struct macb_info *info, 1310 struct macb_dma_desc *desc, dma_addr_t addr) 1311 { 1312 if (macb_dma64(info)) { 1313 struct macb_dma_desc_64 *desc_64; 1314 1315 desc_64 = macb_64b_desc(desc); 1316 desc_64->addrh = upper_32_bits(addr); 1317 /* The low bits of RX address contain the RX_USED bit, clearing 1318 * of which allows packet RX. Make sure the high bits are also 1319 * visible to HW at that point. 1320 */ 1321 dma_wmb(); 1322 } 1323 1324 desc->addr = lower_32_bits(addr); 1325 } 1326 1327 static dma_addr_t macb_get_addr(struct macb_context *ctx, 1328 struct macb_dma_desc *desc) 1329 { 1330 dma_addr_t addr = 0; 1331 1332 if (macb_dma64(ctx->info)) { 1333 struct macb_dma_desc_64 *desc_64; 1334 1335 desc_64 = macb_64b_desc(desc); 1336 addr = ((u64)(desc_64->addrh) << 32); 1337 } 1338 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 1339 if (macb_dma_ptp(ctx->info)) 1340 addr &= ~GEM_BIT(DMA_RXVALID); 1341 return addr; 1342 } 1343 1344 static void macb_tx_error_task(struct work_struct *work) 1345 { 1346 struct macb_queue *queue = container_of(work, struct macb_queue, 1347 tx_error_task); 1348 unsigned int q = queue - queue->bp->queues; 1349 struct macb_context *ctx = queue->bp->ctx; 1350 struct macb_txq *txq = macb_txq(queue); 1351 struct macb *bp = queue->bp; 1352 struct macb_tx_skb *tx_skb; 1353 struct macb_dma_desc *desc; 1354 bool halt_timeout = false; 1355 bool buggy_driver = false; 1356 struct sk_buff *skb; 1357 unsigned long flags; 1358 unsigned int tail; 1359 u32 packets = 0; 1360 u32 bytes = 0; 1361 1362 netdev_vdbg(bp->netdev, "%s: q = %u, t = %u, h = %u\n", 1363 __func__, q, txq->tail, txq->head); 1364 1365 /* Prevent the queue NAPI TX poll from running, as it calls 1366 * macb_tx_complete(), which in turn may call netif_wake_subqueue(). 1367 * As explained below, we have to halt the transmission before updating 1368 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 1369 * network engine about the macb/gem being halted. 1370 */ 1371 napi_disable(&queue->napi_tx); 1372 spin_lock_irqsave(&bp->lock, flags); 1373 1374 /* Make sure nobody is trying to queue up new packets */ 1375 netif_tx_stop_all_queues(bp->netdev); 1376 1377 /* Stop transmission now 1378 * (in case we have just queued new packets) 1379 * macb/gem must be halted to write TBQP register 1380 */ 1381 if (macb_halt_tx(bp)) { 1382 macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE))); 1383 halt_timeout = true; 1384 } 1385 1386 /* Treat frames in TX queue including the ones that caused the error. 1387 * Free transmit buffers in upper layer. 1388 */ 1389 for (tail = txq->tail; tail != txq->head; tail++) { 1390 u32 ctrl; 1391 1392 desc = macb_tx_desc(ctx, q, tail); 1393 ctrl = desc->ctrl; 1394 tx_skb = macb_tx_skb(queue, tail); 1395 skb = tx_skb->skb; 1396 1397 if (ctrl & MACB_BIT(TX_USED)) { 1398 /* skb is set for the last buffer of the frame */ 1399 while (!skb) { 1400 macb_tx_unmap(&bp->info, tx_skb, 0); 1401 tail++; 1402 tx_skb = macb_tx_skb(queue, tail); 1403 skb = tx_skb->skb; 1404 } 1405 1406 /* ctrl still refers to the first buffer descriptor 1407 * since it's the only one written back by the hardware 1408 */ 1409 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 1410 netdev_vdbg(bp->netdev, "txerr skb %u (data %p) TX complete\n", 1411 macb_tx_ring_wrap(ctx, tail), 1412 skb->data); 1413 bp->netdev->stats.tx_packets++; 1414 queue->stats.tx_packets++; 1415 packets++; 1416 bp->netdev->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1417 queue->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1418 bytes += skb->len; 1419 } 1420 } else { 1421 /* "Buffers exhausted mid-frame" errors may only happen 1422 * if the driver is buggy, so complain loudly about 1423 * those. Statistics are updated by hardware. 1424 */ 1425 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 1426 buggy_driver = true; 1427 1428 desc->ctrl = ctrl | MACB_BIT(TX_USED); 1429 } 1430 1431 macb_tx_unmap(&bp->info, tx_skb, 0); 1432 } 1433 1434 netdev_tx_completed_queue(netdev_get_tx_queue(bp->netdev, q), 1435 packets, bytes); 1436 1437 /* Set end of TX queue */ 1438 desc = macb_tx_desc(ctx, q, 0); 1439 macb_set_addr(&bp->info, desc, 0); 1440 desc->ctrl = MACB_BIT(TX_USED); 1441 1442 /* Make descriptor updates visible to hardware */ 1443 wmb(); 1444 1445 /* Reinitialize the TX desc queue */ 1446 queue_writel(queue, TBQP, lower_32_bits(txq->ring_dma)); 1447 /* Make TX ring reflect state of hardware */ 1448 txq->head = 0; 1449 txq->tail = 0; 1450 1451 /* Housework before enabling TX IRQ */ 1452 macb_writel(bp, TSR, macb_readl(bp, TSR)); 1453 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 1454 1455 if (halt_timeout) 1456 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE)); 1457 1458 /* Now we are ready to start transmission again */ 1459 netif_tx_start_all_queues(bp->netdev); 1460 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1461 1462 spin_unlock_irqrestore(&bp->lock, flags); 1463 1464 napi_enable(&queue->napi_tx); 1465 1466 if (halt_timeout) 1467 netdev_err(bp->netdev, "BUG: halt tx timed out, we ignored it\n"); 1468 1469 if (buggy_driver) 1470 netdev_err(bp->netdev, "BUG: TX buffers exhausted mid-frame\n"); 1471 } 1472 1473 static bool ptp_one_step_sync(struct sk_buff *skb) 1474 { 1475 struct ptp_header *hdr; 1476 unsigned int ptp_class; 1477 u8 msgtype; 1478 1479 /* No need to parse packet if PTP TS is not involved */ 1480 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 1481 goto not_oss; 1482 1483 /* Identify and return whether PTP one step sync is being processed */ 1484 ptp_class = ptp_classify_raw(skb); 1485 if (ptp_class == PTP_CLASS_NONE) 1486 goto not_oss; 1487 1488 hdr = ptp_parse_header(skb, ptp_class); 1489 if (!hdr) 1490 goto not_oss; 1491 1492 if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) 1493 goto not_oss; 1494 1495 msgtype = ptp_get_msgtype(hdr, ptp_class); 1496 if (msgtype == PTP_MSGTYPE_SYNC) 1497 return true; 1498 1499 not_oss: 1500 return false; 1501 } 1502 1503 static int macb_tx_complete(struct macb_queue *queue, int budget) 1504 { 1505 struct macb *bp = queue->bp; 1506 struct macb_txq *txq = macb_txq(queue); 1507 unsigned int q = queue - bp->queues; 1508 struct macb_context *ctx = bp->ctx; 1509 unsigned long flags; 1510 unsigned int tail; 1511 unsigned int head; 1512 int packets = 0; 1513 u32 bytes = 0; 1514 1515 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 1516 head = txq->head; 1517 for (tail = txq->tail; tail != head && packets < budget; tail++) { 1518 struct macb_tx_skb *tx_skb; 1519 struct sk_buff *skb; 1520 struct macb_dma_desc *desc; 1521 u32 ctrl; 1522 1523 desc = macb_tx_desc(ctx, q, tail); 1524 1525 /* Make hw descriptor updates visible to CPU */ 1526 rmb(); 1527 1528 ctrl = desc->ctrl; 1529 1530 /* TX_USED bit is only set by hardware on the very first buffer 1531 * descriptor of the transmitted frame. 1532 */ 1533 if (!(ctrl & MACB_BIT(TX_USED))) 1534 break; 1535 1536 /* Process all buffers of the current transmitted frame */ 1537 for (;; tail++) { 1538 tx_skb = macb_tx_skb(queue, tail); 1539 skb = tx_skb->skb; 1540 1541 /* First, update TX stats if needed */ 1542 if (skb) { 1543 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 1544 !ptp_one_step_sync(skb)) 1545 gem_ptp_do_txstamp(bp, skb, desc); 1546 1547 netdev_vdbg(bp->netdev, "skb %u (data %p) TX complete\n", 1548 macb_tx_ring_wrap(ctx, tail), 1549 skb->data); 1550 bp->netdev->stats.tx_packets++; 1551 queue->stats.tx_packets++; 1552 bp->netdev->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1553 queue->stats.tx_bytes += skb->len - tx_skb->fcs_len; 1554 packets++; 1555 bytes += skb->len; 1556 } 1557 1558 /* Now we can safely release resources */ 1559 macb_tx_unmap(&bp->info, tx_skb, budget); 1560 1561 /* skb is set only for the last buffer of the frame. 1562 * WARNING: at this point skb has been freed by 1563 * macb_tx_unmap(). 1564 */ 1565 if (skb) 1566 break; 1567 } 1568 } 1569 1570 netdev_tx_completed_queue(netdev_get_tx_queue(bp->netdev, q), 1571 packets, bytes); 1572 1573 txq->tail = tail; 1574 if (__netif_subqueue_stopped(bp->netdev, q) && 1575 CIRC_CNT(txq->head, txq->tail, 1576 bp->ctx->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 1577 netif_wake_subqueue(bp->netdev, q); 1578 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 1579 1580 if (packets) 1581 macb_tx_lpi_schedule(bp); 1582 1583 return packets; 1584 } 1585 1586 static int gem_rx_refill(struct macb_context *ctx, unsigned int q) 1587 { 1588 struct device *dev = &ctx->info->pdev->dev; 1589 struct macb_rxq *rxq = &ctx->rxq[q]; 1590 struct macb_dma_desc *desc; 1591 struct sk_buff *skb; 1592 unsigned int entry; 1593 dma_addr_t paddr; 1594 1595 while (CIRC_SPACE(rxq->prepared_head, rxq->tail, 1596 ctx->rx_ring_size) > 0) { 1597 entry = macb_rx_ring_wrap(ctx, rxq->prepared_head); 1598 1599 /* Make hw descriptor updates visible to CPU */ 1600 rmb(); 1601 1602 desc = macb_rx_desc(ctx, q, entry); 1603 1604 if (!rxq->skbuff[entry]) { 1605 /* allocate sk_buff for this free entry in ring */ 1606 skb = netdev_alloc_skb(ctx->info->netdev, 1607 ctx->rx_buffer_size); 1608 if (unlikely(!skb)) { 1609 netdev_err(ctx->info->netdev, 1610 "Unable to allocate sk_buff\n"); 1611 break; 1612 } 1613 1614 /* now fill corresponding descriptor entry */ 1615 paddr = dma_map_single(dev, skb->data, 1616 ctx->rx_buffer_size, 1617 DMA_FROM_DEVICE); 1618 if (dma_mapping_error(dev, paddr)) { 1619 dev_kfree_skb(skb); 1620 break; 1621 } 1622 1623 rxq->skbuff[entry] = skb; 1624 1625 if (entry == ctx->rx_ring_size - 1) 1626 paddr |= MACB_BIT(RX_WRAP); 1627 desc->ctrl = 0; 1628 /* Setting addr clears RX_USED and allows reception, 1629 * make sure ctrl is cleared first to avoid a race. 1630 */ 1631 dma_wmb(); 1632 macb_set_addr(ctx->info, desc, paddr); 1633 1634 /* Properly align Ethernet header. 1635 * 1636 * Hardware can add dummy bytes if asked using the RBOF 1637 * field inside the NCFGR register. That feature isn't 1638 * available if hardware is RSC capable. 1639 * 1640 * We cannot fallback to doing the 2-byte shift before 1641 * DMA mapping because the address field does not allow 1642 * setting the low 2/3 bits. 1643 * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. 1644 */ 1645 if (!(ctx->info->caps & MACB_CAPS_RSC)) 1646 skb_reserve(skb, NET_IP_ALIGN); 1647 } else { 1648 desc->ctrl = 0; 1649 dma_wmb(); 1650 desc->addr &= ~MACB_BIT(RX_USED); 1651 } 1652 rxq->prepared_head++; 1653 } 1654 1655 /* Make descriptor updates visible to hardware */ 1656 wmb(); 1657 1658 netdev_vdbg(ctx->info->netdev, 1659 "rx ring: queue: %u, prepared head %d, tail %d\n", 1660 q, rxq->prepared_head, rxq->tail); 1661 1662 /* Fail if queue has zero prepared descriptors. This is critical because 1663 * nothing will ever trigger a refill again. 1664 */ 1665 if (rxq->prepared_head == rxq->tail) 1666 return -ENOMEM; 1667 1668 return 0; 1669 } 1670 1671 /* Mark DMA descriptors from begin up to and not including end as unused */ 1672 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1673 unsigned int end) 1674 { 1675 unsigned int q = queue - queue->bp->queues; 1676 struct macb_context *ctx = queue->bp->ctx; 1677 unsigned int frag; 1678 1679 for (frag = begin; frag != end; frag++) { 1680 struct macb_dma_desc *desc = macb_rx_desc(ctx, q, frag); 1681 1682 desc->addr &= ~MACB_BIT(RX_USED); 1683 } 1684 1685 /* Make descriptor updates visible to hardware */ 1686 wmb(); 1687 1688 /* When this happens, the hardware stats registers for 1689 * whatever caused this is updated, so we don't have to record 1690 * anything. 1691 */ 1692 } 1693 1694 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1695 int budget) 1696 { 1697 unsigned int q = queue - queue->bp->queues; 1698 struct macb_context *ctx = queue->bp->ctx; 1699 struct macb_rxq *rxq = macb_rxq(queue); 1700 struct macb *bp = queue->bp; 1701 struct macb_dma_desc *desc; 1702 struct sk_buff *skb; 1703 unsigned int entry; 1704 unsigned int len; 1705 int count = 0; 1706 1707 while (count < budget) { 1708 u32 ctrl; 1709 dma_addr_t addr; 1710 bool rxused; 1711 1712 /* Only descriptors in [rx_tail, rx_prepared_head) were armed 1713 * for hardware. Outside, we might have RX_USED descriptors for 1714 * alloc failures. 1715 */ 1716 if (rxq->tail == rxq->prepared_head) 1717 break; 1718 1719 entry = macb_rx_ring_wrap(ctx, rxq->tail); 1720 desc = macb_rx_desc(ctx, q, entry); 1721 1722 /* Make hw descriptor updates visible to CPU */ 1723 rmb(); 1724 1725 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1726 addr = macb_get_addr(ctx, desc); 1727 1728 if (!rxused) 1729 break; 1730 1731 /* Ensure ctrl is at least as up-to-date as rxused */ 1732 dma_rmb(); 1733 1734 ctrl = desc->ctrl; 1735 1736 rxq->tail++; 1737 count++; 1738 1739 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1740 netdev_err(bp->netdev, 1741 "not whole frame pointed by descriptor\n"); 1742 bp->netdev->stats.rx_dropped++; 1743 queue->stats.rx_dropped++; 1744 break; 1745 } 1746 skb = rxq->skbuff[entry]; 1747 if (unlikely(!skb)) { 1748 netdev_err(bp->netdev, 1749 "inconsistent Rx descriptor chain\n"); 1750 bp->netdev->stats.rx_dropped++; 1751 queue->stats.rx_dropped++; 1752 break; 1753 } 1754 /* now everything is ready for receiving packet */ 1755 rxq->skbuff[entry] = NULL; 1756 len = ctrl & bp->rx_frm_len_mask; 1757 1758 netdev_vdbg(bp->netdev, "%s %u (len %u)\n", 1759 __func__, entry, len); 1760 1761 skb_put(skb, len); 1762 dma_unmap_single(&bp->pdev->dev, addr, 1763 bp->ctx->rx_buffer_size, DMA_FROM_DEVICE); 1764 1765 skb->protocol = eth_type_trans(skb, bp->netdev); 1766 skb_checksum_none_assert(skb); 1767 if (bp->netdev->features & NETIF_F_RXCSUM && 1768 !(bp->netdev->flags & IFF_PROMISC) && 1769 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1770 skb->ip_summed = CHECKSUM_UNNECESSARY; 1771 1772 bp->netdev->stats.rx_packets++; 1773 queue->stats.rx_packets++; 1774 bp->netdev->stats.rx_bytes += skb->len; 1775 queue->stats.rx_bytes += skb->len; 1776 1777 gem_ptp_do_rxstamp(bp, skb, desc); 1778 1779 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1780 netdev_vdbg(bp->netdev, "received skb of length %u, csum: %08x\n", 1781 skb->len, skb->csum); 1782 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1783 skb_mac_header(skb), 16, true); 1784 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1785 skb->data, 32, true); 1786 #endif 1787 1788 napi_gro_receive(napi, skb); 1789 } 1790 1791 gem_rx_refill(ctx, q); 1792 1793 return count; 1794 } 1795 1796 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1797 unsigned int first_frag, unsigned int last_frag) 1798 { 1799 unsigned int q = queue - queue->bp->queues; 1800 struct macb_context *ctx = queue->bp->ctx; 1801 struct macb *bp = queue->bp; 1802 struct macb_dma_desc *desc; 1803 unsigned int offset; 1804 struct sk_buff *skb; 1805 unsigned int frag; 1806 unsigned int len; 1807 1808 desc = macb_rx_desc(ctx, q, last_frag); 1809 len = desc->ctrl & bp->rx_frm_len_mask; 1810 1811 netdev_vdbg(bp->netdev, "%s frags %u - %u (len %u)\n", 1812 __func__, 1813 macb_rx_ring_wrap(ctx, first_frag), 1814 macb_rx_ring_wrap(ctx, last_frag), len); 1815 1816 /* The ethernet header starts NET_IP_ALIGN bytes into the 1817 * first buffer. Since the header is 14 bytes, this makes the 1818 * payload word-aligned. 1819 * 1820 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1821 * the two padding bytes into the skb so that we avoid hitting 1822 * the slowpath in memcpy(), and pull them off afterwards. 1823 */ 1824 skb = netdev_alloc_skb(bp->netdev, len + NET_IP_ALIGN); 1825 if (!skb) { 1826 bp->netdev->stats.rx_dropped++; 1827 for (frag = first_frag; ; frag++) { 1828 desc = macb_rx_desc(ctx, q, frag); 1829 desc->addr &= ~MACB_BIT(RX_USED); 1830 if (frag == last_frag) 1831 break; 1832 } 1833 1834 /* Make descriptor updates visible to hardware */ 1835 wmb(); 1836 1837 return 1; 1838 } 1839 1840 offset = 0; 1841 len += NET_IP_ALIGN; 1842 skb_checksum_none_assert(skb); 1843 skb_put(skb, len); 1844 1845 for (frag = first_frag; ; frag++) { 1846 unsigned int frag_len = bp->ctx->rx_buffer_size; 1847 1848 if (offset + frag_len > len) { 1849 if (unlikely(frag != last_frag)) { 1850 dev_kfree_skb_any(skb); 1851 return -1; 1852 } 1853 frag_len = len - offset; 1854 } 1855 skb_copy_to_linear_data_offset(skb, offset, 1856 macb_rx_buffer(queue, frag), 1857 frag_len); 1858 offset += bp->ctx->rx_buffer_size; 1859 desc = macb_rx_desc(ctx, q, frag); 1860 desc->addr &= ~MACB_BIT(RX_USED); 1861 1862 if (frag == last_frag) 1863 break; 1864 } 1865 1866 /* Make descriptor updates visible to hardware */ 1867 wmb(); 1868 1869 __skb_pull(skb, NET_IP_ALIGN); 1870 skb->protocol = eth_type_trans(skb, bp->netdev); 1871 1872 bp->netdev->stats.rx_packets++; 1873 bp->netdev->stats.rx_bytes += skb->len; 1874 netdev_vdbg(bp->netdev, "received skb of length %u, csum: %08x\n", 1875 skb->len, skb->csum); 1876 napi_gro_receive(napi, skb); 1877 1878 return 0; 1879 } 1880 1881 static inline void macb_init_rx_ring(struct macb_context *ctx, unsigned int q) 1882 { 1883 struct macb_rxq *rxq = &ctx->rxq[q]; 1884 struct macb_dma_desc *desc = NULL; 1885 dma_addr_t addr; 1886 int i; 1887 1888 addr = rxq->buffers_dma; 1889 for (i = 0; i < ctx->rx_ring_size; i++) { 1890 desc = macb_rx_desc(ctx, q, i); 1891 macb_set_addr(ctx->info, desc, addr); 1892 desc->ctrl = 0; 1893 addr += ctx->rx_buffer_size; 1894 } 1895 desc->addr |= MACB_BIT(RX_WRAP); 1896 rxq->tail = 0; 1897 } 1898 1899 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1900 int budget) 1901 { 1902 unsigned int q = queue - queue->bp->queues; 1903 struct macb_context *ctx = queue->bp->ctx; 1904 struct macb_rxq *rxq = macb_rxq(queue); 1905 struct macb *bp = queue->bp; 1906 bool reset_rx_queue = false; 1907 int first_frag = -1; 1908 unsigned int tail; 1909 int received = 0; 1910 1911 for (tail = rxq->tail; budget > 0; tail++) { 1912 struct macb_dma_desc *desc = macb_rx_desc(ctx, q, tail); 1913 u32 ctrl; 1914 1915 /* Make hw descriptor updates visible to CPU */ 1916 rmb(); 1917 1918 if (!(desc->addr & MACB_BIT(RX_USED))) 1919 break; 1920 1921 /* Ensure ctrl is at least as up-to-date as addr */ 1922 dma_rmb(); 1923 1924 ctrl = desc->ctrl; 1925 1926 if (ctrl & MACB_BIT(RX_SOF)) { 1927 if (first_frag != -1) 1928 discard_partial_frame(queue, first_frag, tail); 1929 first_frag = tail; 1930 } 1931 1932 if (ctrl & MACB_BIT(RX_EOF)) { 1933 int dropped; 1934 1935 if (unlikely(first_frag == -1)) { 1936 reset_rx_queue = true; 1937 continue; 1938 } 1939 1940 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1941 first_frag = -1; 1942 if (unlikely(dropped < 0)) { 1943 reset_rx_queue = true; 1944 continue; 1945 } 1946 if (!dropped) { 1947 received++; 1948 budget--; 1949 } 1950 } 1951 } 1952 1953 if (unlikely(reset_rx_queue)) { 1954 unsigned long flags; 1955 u32 ctrl; 1956 1957 netdev_err(bp->netdev, "RX queue corruption: reset it\n"); 1958 1959 spin_lock_irqsave(&bp->lock, flags); 1960 1961 ctrl = macb_readl(bp, NCR); 1962 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1963 1964 macb_init_rx_ring(ctx, q); 1965 queue_writel(queue, RBQP, rxq->ring_dma); 1966 1967 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1968 1969 spin_unlock_irqrestore(&bp->lock, flags); 1970 return received; 1971 } 1972 1973 if (first_frag != -1) 1974 rxq->tail = first_frag; 1975 else 1976 rxq->tail = tail; 1977 1978 return received; 1979 } 1980 1981 static bool macb_rx_pending(struct macb_queue *queue) 1982 { 1983 unsigned int q = queue - queue->bp->queues; 1984 struct macb_context *ctx = queue->bp->ctx; 1985 struct macb_rxq *rxq = macb_rxq(queue); 1986 struct macb_dma_desc *desc; 1987 unsigned int entry; 1988 1989 /* No armed descriptor left: nothing can be pending. */ 1990 if (macb_is_gem(&queue->bp->info) && rxq->tail == rxq->prepared_head) 1991 return false; 1992 1993 entry = macb_rx_ring_wrap(ctx, rxq->tail); 1994 desc = macb_rx_desc(ctx, q, entry); 1995 1996 /* Make hw descriptor updates visible to CPU */ 1997 rmb(); 1998 1999 return (desc->addr & MACB_BIT(RX_USED)) != 0; 2000 } 2001 2002 static int macb_rx_poll(struct napi_struct *napi, int budget) 2003 { 2004 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx); 2005 struct macb *bp = queue->bp; 2006 int work_done; 2007 2008 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 2009 2010 netdev_vdbg(bp->netdev, "RX poll: queue = %u, work_done = %d, budget = %d\n", 2011 (unsigned int)(queue - bp->queues), work_done, budget); 2012 2013 if (work_done < budget && napi_complete_done(napi, work_done)) { 2014 queue_writel(queue, IER, bp->rx_intr_mask); 2015 2016 /* Packet completions only seem to propagate to raise 2017 * interrupts when interrupts are enabled at the time, so if 2018 * packets were received while interrupts were disabled, 2019 * they will not cause another interrupt to be generated when 2020 * interrupts are re-enabled. 2021 * Check for this case here to avoid losing a wakeup. This can 2022 * potentially race with the interrupt handler doing the same 2023 * actions if an interrupt is raised just after enabling them, 2024 * but this should be harmless. 2025 */ 2026 if (macb_rx_pending(queue)) { 2027 queue_writel(queue, IDR, bp->rx_intr_mask); 2028 macb_queue_isr_clear(bp, queue, MACB_BIT(RCOMP)); 2029 netdev_vdbg(bp->netdev, "poll: packets pending, reschedule\n"); 2030 napi_schedule(napi); 2031 } 2032 } 2033 2034 /* TODO: Handle errors */ 2035 2036 return work_done; 2037 } 2038 2039 static void macb_tx_restart(struct macb_queue *queue) 2040 { 2041 struct macb *bp = queue->bp; 2042 unsigned int head_idx, tbqp; 2043 struct macb_context *ctx; 2044 struct macb_txq *txq; 2045 unsigned long flags; 2046 2047 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 2048 2049 spin_lock(&bp->lock); 2050 2051 /* context swap ongoing? */ 2052 if (bp->irq_quiesced) 2053 goto out_unlock; 2054 2055 ctx = queue->bp->ctx; 2056 txq = macb_txq(queue); 2057 2058 if (txq->head == txq->tail) 2059 goto out_unlock; 2060 2061 tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(ctx->info); 2062 tbqp = macb_adj_dma_desc_idx(ctx, macb_tx_ring_wrap(ctx, tbqp)); 2063 head_idx = macb_adj_dma_desc_idx(ctx, macb_tx_ring_wrap(ctx, txq->head)); 2064 2065 if (tbqp == head_idx) 2066 goto out_unlock; 2067 2068 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 2069 2070 out_unlock: 2071 spin_unlock(&bp->lock); 2072 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 2073 } 2074 2075 static bool macb_tx_complete_pending(struct macb_queue *queue) 2076 { 2077 unsigned int q = queue - queue->bp->queues; 2078 struct macb_context *ctx = queue->bp->ctx; 2079 struct macb_txq *txq = macb_txq(queue); 2080 bool retval = false; 2081 unsigned long flags; 2082 2083 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 2084 if (txq->head != txq->tail) { 2085 /* Make hw descriptor updates visible to CPU */ 2086 rmb(); 2087 2088 if (macb_tx_desc(ctx, q, txq->tail)->ctrl & MACB_BIT(TX_USED)) 2089 retval = true; 2090 } 2091 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 2092 return retval; 2093 } 2094 2095 static int macb_tx_poll(struct napi_struct *napi, int budget) 2096 { 2097 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx); 2098 struct macb *bp = queue->bp; 2099 int work_done; 2100 2101 work_done = macb_tx_complete(queue, budget); 2102 2103 rmb(); // ensure txubr_pending is up to date 2104 if (queue->txubr_pending) { 2105 queue->txubr_pending = false; 2106 netdev_vdbg(bp->netdev, "poll: tx restart\n"); 2107 macb_tx_restart(queue); 2108 } 2109 2110 netdev_vdbg(bp->netdev, "TX poll: queue = %u, work_done = %d, budget = %d\n", 2111 (unsigned int)(queue - bp->queues), work_done, budget); 2112 2113 if (work_done < budget && napi_complete_done(napi, work_done)) { 2114 queue_writel(queue, IER, MACB_BIT(TCOMP)); 2115 2116 /* Packet completions only seem to propagate to raise 2117 * interrupts when interrupts are enabled at the time, so if 2118 * packets were sent while interrupts were disabled, 2119 * they will not cause another interrupt to be generated when 2120 * interrupts are re-enabled. 2121 * Check for this case here to avoid losing a wakeup. This can 2122 * potentially race with the interrupt handler doing the same 2123 * actions if an interrupt is raised just after enabling them, 2124 * but this should be harmless. 2125 */ 2126 if (macb_tx_complete_pending(queue)) { 2127 queue_writel(queue, IDR, MACB_BIT(TCOMP)); 2128 macb_queue_isr_clear(bp, queue, MACB_BIT(TCOMP)); 2129 netdev_vdbg(bp->netdev, "TX poll: packets pending, reschedule\n"); 2130 napi_schedule(napi); 2131 } 2132 } 2133 2134 return work_done; 2135 } 2136 2137 static void macb_quiesce_start(struct macb *bp) 2138 { 2139 struct macb_queue *queue; 2140 unsigned long flags; 2141 unsigned int q; 2142 2143 spin_lock_irqsave(&bp->lock, flags); 2144 bp->irq_quiesced = true; 2145 spin_unlock_irqrestore(&bp->lock, flags); 2146 2147 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2148 synchronize_irq(queue->irq); 2149 2150 cancel_work_sync(&bp->hresp_err_bh_work); 2151 2152 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2153 /* Must be done before NAPI is disabled: the task ends with a 2154 * napi_enable() call. 2155 */ 2156 cancel_work_sync(&queue->tx_error_task); 2157 2158 napi_disable(&queue->napi_rx); 2159 napi_disable(&queue->napi_tx); 2160 } 2161 2162 /* NAPI instances still access Rx/Tx rings after their 2163 * napi_complete_done() calls! Callers to macb_quiesce_start() 2164 * free rings, so wait for all NAPI handlers to be done. 2165 */ 2166 synchronize_net(); 2167 2168 /* Must be done after napi_tx is disabled: its completion re-arms 2169 * the LPI timer. 2170 */ 2171 cancel_delayed_work_sync(&bp->tx_lpi_work); 2172 } 2173 2174 static void macb_quiesce_end(struct macb *bp) 2175 { 2176 struct macb_queue *queue; 2177 unsigned long flags; 2178 unsigned int q; 2179 2180 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2181 napi_enable(&queue->napi_rx); 2182 napi_enable(&queue->napi_tx); 2183 } 2184 2185 spin_lock_irqsave(&bp->lock, flags); 2186 bp->irq_quiesced = false; 2187 spin_unlock_irqrestore(&bp->lock, flags); 2188 } 2189 2190 static void macb_hresp_error_task(struct work_struct *work) 2191 { 2192 struct macb *bp = from_work(bp, work, hresp_err_bh_work); 2193 struct net_device *netdev = bp->netdev; 2194 struct macb_context *ctx = bp->ctx; 2195 struct macb_queue *queue; 2196 unsigned int q; 2197 u32 ctrl; 2198 2199 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2200 queue_writel(queue, IDR, bp->rx_intr_mask | 2201 MACB_TX_INT_FLAGS | 2202 MACB_BIT(HRESP)); 2203 } 2204 ctrl = macb_readl(bp, NCR); 2205 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2206 macb_writel(bp, NCR, ctrl); 2207 2208 netif_tx_stop_all_queues(netdev); 2209 netif_carrier_off(netdev); 2210 2211 bp->macbgem_ops.mog_init_rings(ctx); 2212 2213 /* Initialize TX and RX buffers */ 2214 macb_init_buffers(bp); 2215 2216 /* Enable interrupts */ 2217 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2218 queue_writel(queue, IER, 2219 bp->rx_intr_mask | 2220 MACB_TX_INT_FLAGS | 2221 MACB_BIT(HRESP)); 2222 2223 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 2224 macb_writel(bp, NCR, ctrl); 2225 2226 netif_carrier_on(netdev); 2227 netif_tx_start_all_queues(netdev); 2228 } 2229 2230 static void macb_wol_interrupt(struct macb_queue *queue, u32 status) 2231 { 2232 struct macb *bp = queue->bp; 2233 2234 queue_writel(queue, IDR, MACB_BIT(WOL)); 2235 macb_writel(bp, WOL, 0); 2236 netdev_vdbg(bp->netdev, "MACB WoL: queue = %u, isr = 0x%08lx\n", 2237 (unsigned int)(queue - bp->queues), 2238 (unsigned long)status); 2239 macb_queue_isr_clear(bp, queue, MACB_BIT(WOL)); 2240 pm_wakeup_event(&bp->pdev->dev, 0); 2241 } 2242 2243 static void gem_wol_interrupt(struct macb_queue *queue, u32 status) 2244 { 2245 struct macb *bp = queue->bp; 2246 2247 queue_writel(queue, IDR, GEM_BIT(WOL)); 2248 gem_writel(bp, WOL, 0); 2249 netdev_vdbg(bp->netdev, "GEM WoL: queue = %u, isr = 0x%08lx\n", 2250 (unsigned int)(queue - bp->queues), 2251 (unsigned long)status); 2252 macb_queue_isr_clear(bp, queue, GEM_BIT(WOL)); 2253 pm_wakeup_event(&bp->pdev->dev, 0); 2254 } 2255 2256 static int macb_interrupt_misc(struct macb_queue *queue, u32 status) 2257 { 2258 struct macb *bp = queue->bp; 2259 u32 ctrl; 2260 2261 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 2262 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 2263 schedule_work(&queue->tx_error_task); 2264 macb_queue_isr_clear(bp, queue, MACB_TX_ERR_FLAGS); 2265 return -1; 2266 } 2267 2268 /* Link change detection isn't possible with RMII, so we'll 2269 * add that if/when we get our hands on a full-blown MII PHY. 2270 */ 2271 2272 /* There is a hardware issue under heavy load where DMA can 2273 * stop, this causes endless "used buffer descriptor read" 2274 * interrupts but it can be cleared by re-enabling RX. See 2275 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 2276 * section 16.7.4 for details. RXUBR is only enabled for 2277 * these two versions. 2278 */ 2279 if (status & MACB_BIT(RXUBR)) { 2280 ctrl = macb_readl(bp, NCR); 2281 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 2282 wmb(); 2283 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 2284 macb_queue_isr_clear(bp, queue, MACB_BIT(RXUBR)); 2285 } 2286 2287 if (status & MACB_BIT(ISR_ROVR)) { 2288 /* We missed at least one packet */ 2289 spin_lock(&bp->stats_lock); 2290 if (macb_is_gem(&bp->info)) 2291 bp->hw_stats.gem.rx_overruns++; 2292 else 2293 bp->hw_stats.macb.rx_overruns++; 2294 spin_unlock(&bp->stats_lock); 2295 macb_queue_isr_clear(bp, queue, MACB_BIT(ISR_ROVR)); 2296 } 2297 2298 if (status & MACB_BIT(HRESP)) { 2299 queue_work(system_bh_wq, &bp->hresp_err_bh_work); 2300 macb_queue_isr_clear(bp, queue, MACB_BIT(HRESP)); 2301 } 2302 2303 if (macb_is_gem(&bp->info)) { 2304 if (status & GEM_BIT(WOL)) 2305 gem_wol_interrupt(queue, status); 2306 } else { 2307 if (status & MACB_BIT(WOL)) 2308 macb_wol_interrupt(queue, status); 2309 } 2310 2311 return 0; 2312 } 2313 2314 static irqreturn_t macb_interrupt(int irq, void *dev_id) 2315 { 2316 struct macb_queue *queue = dev_id; 2317 struct macb *bp = queue->bp; 2318 struct net_device *netdev = bp->netdev; 2319 bool hresp_err = false; 2320 u32 status; 2321 2322 status = queue_readl(queue, ISR); 2323 2324 if (unlikely(!status)) 2325 return IRQ_NONE; 2326 2327 spin_lock(&bp->lock); 2328 2329 while (status) { 2330 /* self-disarm while the netdev is closed 2331 * or during context swap 2332 */ 2333 if (unlikely(bp->irq_quiesced)) { 2334 queue_writel(queue, IDR, -1); 2335 macb_queue_isr_clear(bp, queue, -1); 2336 break; 2337 } 2338 2339 netdev_vdbg(netdev, "queue = %u, isr = 0x%08lx\n", 2340 (unsigned int)(queue - bp->queues), 2341 (unsigned long)status); 2342 2343 if (status & bp->rx_intr_mask) { 2344 /* There's no point taking any more interrupts 2345 * until we have processed the buffers. The 2346 * scheduling call may fail if the poll routine 2347 * is already scheduled, so disable interrupts 2348 * now. 2349 */ 2350 queue_writel(queue, IDR, bp->rx_intr_mask); 2351 macb_queue_isr_clear(bp, queue, MACB_BIT(RCOMP)); 2352 napi_schedule_irqoff(&queue->napi_rx); 2353 } 2354 2355 if (status & (MACB_BIT(TCOMP) | 2356 MACB_BIT(TXUBR))) { 2357 queue_writel(queue, IDR, MACB_BIT(TCOMP)); 2358 macb_queue_isr_clear(bp, queue, MACB_BIT(TCOMP) | 2359 MACB_BIT(TXUBR)); 2360 if (status & MACB_BIT(TXUBR)) { 2361 queue->txubr_pending = true; 2362 wmb(); // ensure softirq can see update 2363 } 2364 2365 napi_schedule_irqoff(&queue->napi_tx); 2366 } 2367 2368 if (unlikely(status & MACB_INT_MISC_FLAGS)) { 2369 if (macb_interrupt_misc(queue, status)) 2370 break; 2371 2372 if (status & MACB_BIT(HRESP)) 2373 hresp_err = true; 2374 } 2375 2376 status = queue_readl(queue, ISR); 2377 } 2378 2379 spin_unlock(&bp->lock); 2380 2381 if (hresp_err) 2382 netdev_err(netdev, "DMA bus error: HRESP not OK\n"); 2383 2384 return IRQ_HANDLED; 2385 } 2386 2387 #ifdef CONFIG_NET_POLL_CONTROLLER 2388 /* Polling receive - used by netconsole and other diagnostic tools 2389 * to allow network i/o with interrupts disabled. 2390 */ 2391 static void macb_poll_controller(struct net_device *netdev) 2392 { 2393 struct macb *bp = netdev_priv(netdev); 2394 struct macb_queue *queue; 2395 unsigned long flags; 2396 unsigned int q; 2397 2398 local_irq_save(flags); 2399 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2400 macb_interrupt(netdev->irq, queue); 2401 local_irq_restore(flags); 2402 } 2403 #endif 2404 2405 static unsigned int macb_tx_map(struct macb *bp, 2406 struct macb_queue *queue, 2407 struct sk_buff *skb, 2408 unsigned int hdrlen, 2409 u8 fcs_len) 2410 { 2411 struct macb_txq *txq = macb_txq(queue); 2412 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 2413 unsigned int len, i, tx_head = txq->head; 2414 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 2415 unsigned int q = queue - bp->queues; 2416 struct macb_context *ctx = bp->ctx; 2417 unsigned int eof = 1, mss_mfs = 0; 2418 struct macb_tx_skb *tx_skb = NULL; 2419 struct macb_dma_desc *desc; 2420 unsigned int offset, size; 2421 dma_addr_t mapping; 2422 2423 /* LSO */ 2424 if (skb_shinfo(skb)->gso_size != 0) { 2425 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2426 /* UDP - UFO */ 2427 lso_ctrl = MACB_LSO_UFO_ENABLE; 2428 else 2429 /* TCP - TSO */ 2430 lso_ctrl = MACB_LSO_TSO_ENABLE; 2431 } 2432 2433 /* First, map non-paged data */ 2434 len = skb_headlen(skb); 2435 2436 /* first buffer length */ 2437 size = hdrlen; 2438 2439 offset = 0; 2440 while (len) { 2441 tx_skb = macb_tx_skb(queue, tx_head); 2442 2443 mapping = dma_map_single(&bp->pdev->dev, 2444 skb->data + offset, 2445 size, DMA_TO_DEVICE); 2446 if (dma_mapping_error(&bp->pdev->dev, mapping)) 2447 goto dma_error; 2448 2449 /* Save info to properly release resources */ 2450 tx_skb->skb = NULL; 2451 tx_skb->mapping = mapping; 2452 tx_skb->size = size; 2453 tx_skb->mapped_as_page = false; 2454 2455 len -= size; 2456 offset += size; 2457 tx_head++; 2458 2459 size = umin(len, bp->max_tx_length); 2460 } 2461 2462 /* Then, map paged data from fragments */ 2463 for (f = 0; f < nr_frags; f++) { 2464 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2465 2466 len = skb_frag_size(frag); 2467 offset = 0; 2468 while (len) { 2469 size = umin(len, bp->max_tx_length); 2470 tx_skb = macb_tx_skb(queue, tx_head); 2471 2472 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 2473 offset, size, DMA_TO_DEVICE); 2474 if (dma_mapping_error(&bp->pdev->dev, mapping)) 2475 goto dma_error; 2476 2477 /* Save info to properly release resources */ 2478 tx_skb->skb = NULL; 2479 tx_skb->mapping = mapping; 2480 tx_skb->size = size; 2481 tx_skb->mapped_as_page = true; 2482 2483 len -= size; 2484 offset += size; 2485 tx_head++; 2486 } 2487 } 2488 2489 /* Should never happen */ 2490 if (unlikely(!tx_skb)) { 2491 netdev_err(bp->netdev, "BUG! empty skb!\n"); 2492 return 0; 2493 } 2494 2495 /* This is the last buffer of the frame: save socket buffer */ 2496 tx_skb->skb = skb; 2497 tx_skb->fcs_len = fcs_len; 2498 2499 /* Update TX ring: update buffer descriptors in reverse order 2500 * to avoid race condition 2501 */ 2502 2503 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 2504 * to set the end of TX queue 2505 */ 2506 i = tx_head; 2507 ctrl = MACB_BIT(TX_USED); 2508 desc = macb_tx_desc(ctx, q, i); 2509 desc->ctrl = ctrl; 2510 2511 if (lso_ctrl) { 2512 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 2513 /* include header and FCS in value given to h/w */ 2514 mss_mfs = skb_shinfo(skb)->gso_size + 2515 skb_transport_offset(skb) + 2516 ETH_FCS_LEN; 2517 else /* TSO */ { 2518 mss_mfs = skb_shinfo(skb)->gso_size; 2519 /* TCP Sequence Number Source Select 2520 * can be set only for TSO 2521 */ 2522 seq_ctrl = 0; 2523 } 2524 } 2525 2526 do { 2527 i--; 2528 tx_skb = macb_tx_skb(queue, i); 2529 desc = macb_tx_desc(ctx, q, i); 2530 2531 ctrl = (u32)tx_skb->size; 2532 if (eof) { 2533 ctrl |= MACB_BIT(TX_LAST); 2534 eof = 0; 2535 } 2536 if (unlikely(macb_tx_ring_wrap(ctx, i) == 2537 bp->ctx->tx_ring_size - 1)) 2538 ctrl |= MACB_BIT(TX_WRAP); 2539 2540 /* First descriptor is header descriptor */ 2541 if (i == txq->head) { 2542 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 2543 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 2544 if ((bp->netdev->features & NETIF_F_HW_CSUM) && 2545 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && 2546 !ptp_one_step_sync(skb)) 2547 ctrl |= MACB_BIT(TX_NOCRC); 2548 } else 2549 /* Only set MSS/MFS on payload descriptors 2550 * (second or later descriptor) 2551 */ 2552 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 2553 2554 /* Set TX buffer descriptor */ 2555 macb_set_addr(&bp->info, desc, tx_skb->mapping); 2556 /* desc->addr must be visible to hardware before clearing 2557 * 'TX_USED' bit in desc->ctrl. 2558 */ 2559 wmb(); 2560 desc->ctrl = ctrl; 2561 } while (i != txq->head); 2562 2563 txq->head = tx_head; 2564 2565 return 0; 2566 2567 dma_error: 2568 netdev_err(bp->netdev, "TX DMA map failed\n"); 2569 2570 for (i = txq->head; i != tx_head; i++) { 2571 tx_skb = macb_tx_skb(queue, i); 2572 2573 macb_tx_unmap(&bp->info, tx_skb, 0); 2574 } 2575 2576 return -ENOMEM; 2577 } 2578 2579 static netdev_features_t macb_features_check(struct sk_buff *skb, 2580 struct net_device *netdev, 2581 netdev_features_t features) 2582 { 2583 unsigned int nr_frags, f; 2584 unsigned int hdrlen; 2585 2586 /* Validate LSO compatibility */ 2587 2588 /* there is only one buffer or protocol is not UDP */ 2589 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) 2590 return features; 2591 2592 /* length of header */ 2593 hdrlen = skb_transport_offset(skb); 2594 2595 /* For UFO only: 2596 * When software supplies two or more payload buffers all payload buffers 2597 * apart from the last must be a multiple of 8 bytes in size. 2598 */ 2599 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 2600 return features & ~MACB_NETIF_LSO; 2601 2602 nr_frags = skb_shinfo(skb)->nr_frags; 2603 /* No need to check last fragment */ 2604 nr_frags--; 2605 for (f = 0; f < nr_frags; f++) { 2606 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2607 2608 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 2609 return features & ~MACB_NETIF_LSO; 2610 } 2611 return features; 2612 } 2613 2614 static inline int macb_clear_csum(struct sk_buff *skb) 2615 { 2616 /* no change for packets without checksum offloading */ 2617 if (skb->ip_summed != CHECKSUM_PARTIAL) 2618 return 0; 2619 2620 /* make sure we can modify the header */ 2621 if (unlikely(skb_cow_head(skb, 0))) 2622 return -1; 2623 2624 /* initialize checksum field 2625 * This is required - at least for Zynq, which otherwise calculates 2626 * wrong UDP header checksums for UDP packets with UDP data len <=2 2627 */ 2628 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 2629 return 0; 2630 } 2631 2632 static bool macb_needs_sw_fcs(struct sk_buff *skb, struct net_device *netdev) 2633 { 2634 return netdev->features & NETIF_F_HW_CSUM && 2635 skb->ip_summed != CHECKSUM_PARTIAL && 2636 !skb_shinfo(skb)->gso_size && !ptp_one_step_sync(skb); 2637 } 2638 2639 /* Returns a negative errno, or the FCS bytes appended (0 or ETH_FCS_LEN). */ 2640 static int macb_pad_and_fcs(struct sk_buff **skb, bool add_fcs) 2641 { 2642 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || 2643 skb_is_nonlinear(*skb); 2644 int padlen = ETH_ZLEN - (*skb)->len; 2645 int tailroom = skb_tailroom(*skb); 2646 bool shared = skb_shared(*skb); 2647 struct sk_buff *nskb; 2648 u32 fcs; 2649 2650 if (!add_fcs) 2651 return 0; 2652 2653 if (padlen <= 0) { 2654 /* FCS could be appended to tailroom. */ 2655 if (!shared && !skb_is_nonlinear(*skb) && 2656 tailroom >= ETH_FCS_LEN) 2657 goto add_fcs; 2658 /* Reallocate with room for the FCS. */ 2659 padlen = ETH_FCS_LEN; 2660 } else { 2661 /* Add room for FCS. */ 2662 padlen += ETH_FCS_LEN; 2663 } 2664 2665 if (shared || cloned || tailroom < padlen) { 2666 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 2667 if (!nskb) 2668 return -ENOMEM; 2669 2670 dev_consume_skb_any(*skb); 2671 *skb = nskb; 2672 } 2673 2674 if (padlen > ETH_FCS_LEN) 2675 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 2676 2677 add_fcs: 2678 /* set FCS to packet */ 2679 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 2680 fcs = ~fcs; 2681 2682 skb_put_u8(*skb, fcs & 0xff); 2683 skb_put_u8(*skb, (fcs >> 8) & 0xff); 2684 skb_put_u8(*skb, (fcs >> 16) & 0xff); 2685 skb_put_u8(*skb, (fcs >> 24) & 0xff); 2686 2687 return ETH_FCS_LEN; 2688 } 2689 2690 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, 2691 struct net_device *netdev) 2692 { 2693 struct macb *bp = netdev_priv(netdev); 2694 unsigned int q = skb_get_queue_mapping(skb); 2695 unsigned int desc_cnt, nr_frags, frag_size, f; 2696 struct macb_queue *queue = &bp->queues[q]; 2697 struct macb_txq *txq = macb_txq(queue); 2698 netdev_tx_t ret = NETDEV_TX_OK; 2699 unsigned int hdrlen, tx_len; 2700 bool add_fcs, is_lso; 2701 unsigned long flags; 2702 int fcs_len; 2703 2704 add_fcs = macb_needs_sw_fcs(skb, netdev); 2705 is_lso = (skb_shinfo(skb)->gso_size != 0); 2706 tx_len = add_fcs ? max_t(unsigned int, skb->len, ETH_ZLEN) + 2707 ETH_FCS_LEN : skb->len; 2708 2709 if (is_lso) { 2710 /* length of headers */ 2711 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2712 /* only queue eth + ip headers separately for UDP */ 2713 hdrlen = skb_transport_offset(skb); 2714 else 2715 hdrlen = skb_tcp_all_headers(skb); 2716 if (skb_headlen(skb) < hdrlen) { 2717 netdev_err(bp->netdev, "Error - LSO headers fragmented!!!\n"); 2718 /* if this is required, would need to copy to single buffer */ 2719 return NETDEV_TX_BUSY; 2720 } 2721 } else if (add_fcs) { 2722 hdrlen = umin(tx_len, bp->max_tx_length); 2723 } else { 2724 hdrlen = umin(skb_headlen(skb), bp->max_tx_length); 2725 } 2726 2727 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 2728 netdev_vdbg(bp->netdev, 2729 "start_xmit: queue %u len %u head %p data %p tail %p end %p\n", 2730 q, skb->len, skb->head, skb->data, 2731 skb_tail_pointer(skb), skb_end_pointer(skb)); 2732 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 2733 skb->data, 16, true); 2734 #endif 2735 2736 /* Count how many TX buffer descriptors are needed to send this 2737 * socket buffer: skb fragments of jumbo frames may need to be 2738 * split into many buffer descriptors. 2739 */ 2740 if (add_fcs) { 2741 /* macb_pad_and_fcs() linearizes the skb before adding the FCS. */ 2742 desc_cnt = DIV_ROUND_UP(tx_len, bp->max_tx_length); 2743 nr_frags = 0; 2744 } else if (is_lso && (skb_headlen(skb) > hdrlen)) { 2745 /* extra header descriptor if also payload in first buffer */ 2746 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 2747 nr_frags = skb_shinfo(skb)->nr_frags; 2748 } else { 2749 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 2750 nr_frags = skb_shinfo(skb)->nr_frags; 2751 } 2752 for (f = 0; f < nr_frags; f++) { 2753 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 2754 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 2755 } 2756 2757 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 2758 2759 /* This is a hard error, log it. */ 2760 if (CIRC_SPACE(txq->head, txq->tail, 2761 bp->ctx->tx_ring_size) < desc_cnt) { 2762 netif_stop_subqueue(netdev, q); 2763 netdev_dbg(netdev, "txq head = %u, txq tail = %u\n", 2764 txq->head, txq->tail); 2765 ret = NETDEV_TX_BUSY; 2766 goto unlock; 2767 } 2768 2769 if (macb_clear_csum(skb)) { 2770 dev_kfree_skb_any(skb); 2771 goto unlock; 2772 } 2773 2774 fcs_len = macb_pad_and_fcs(&skb, add_fcs); 2775 if (fcs_len < 0) { 2776 dev_kfree_skb_any(skb); 2777 goto unlock; 2778 } 2779 2780 if (macb_dma_ptp(&bp->info) && 2781 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 2782 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2783 2784 /* Map socket buffer for DMA transfer */ 2785 if (macb_tx_map(bp, queue, skb, hdrlen, fcs_len)) { 2786 dev_kfree_skb_any(skb); 2787 goto unlock; 2788 } 2789 2790 /* Make newly initialized descriptor visible to hardware */ 2791 wmb(); 2792 skb_tx_timestamp(skb); 2793 netdev_tx_sent_queue(netdev_get_tx_queue(bp->netdev, q), 2794 skb->len); 2795 2796 spin_lock(&bp->lock); 2797 macb_tx_lpi_wake(bp); 2798 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 2799 spin_unlock(&bp->lock); 2800 2801 if (CIRC_SPACE(txq->head, txq->tail, bp->ctx->tx_ring_size) < 1) 2802 netif_stop_subqueue(netdev, q); 2803 2804 unlock: 2805 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 2806 2807 return ret; 2808 } 2809 2810 static unsigned int macb_rx_buffer_size(struct macb *bp, unsigned int mtu) 2811 { 2812 unsigned int size; 2813 2814 if (!macb_is_gem(&bp->info)) { 2815 size = MACB_RX_BUFFER_SIZE; 2816 } else { 2817 size = mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2818 size = MIN(size, RX_BUFFER_MAX); 2819 2820 if (size % RX_BUFFER_MULTIPLE) { 2821 netdev_dbg(bp->netdev, 2822 "RX buffer must be multiple of %d bytes, expanding\n", 2823 RX_BUFFER_MULTIPLE); 2824 size = roundup(size, RX_BUFFER_MULTIPLE); 2825 } 2826 } 2827 2828 netdev_dbg(bp->netdev, "mtu [%u] rx_buffer_size [%u]\n", mtu, size); 2829 return size; 2830 } 2831 2832 static void gem_free_rx_buffers(struct macb_context *ctx) 2833 { 2834 struct device *dev = &ctx->info->pdev->dev; 2835 struct macb_dma_desc *desc; 2836 struct macb_rxq *rxq; 2837 struct sk_buff *skb; 2838 dma_addr_t addr; 2839 unsigned int q; 2840 int i; 2841 2842 for (q = 0; q < ctx->info->num_queues; ++q) { 2843 rxq = &ctx->rxq[q]; 2844 2845 if (!rxq->skbuff) 2846 continue; 2847 2848 for (i = 0; i < ctx->rx_ring_size; i++) { 2849 skb = rxq->skbuff[i]; 2850 2851 if (!skb) 2852 continue; 2853 2854 desc = macb_rx_desc(ctx, q, i); 2855 addr = macb_get_addr(ctx, desc); 2856 2857 dma_unmap_single(dev, addr, ctx->rx_buffer_size, 2858 DMA_FROM_DEVICE); 2859 dev_kfree_skb_any(skb); 2860 skb = NULL; 2861 } 2862 2863 kfree(rxq->skbuff); 2864 rxq->skbuff = NULL; 2865 } 2866 } 2867 2868 static void macb_free_rx_buffers(struct macb_context *ctx) 2869 { 2870 struct device *dev = &ctx->info->pdev->dev; 2871 struct macb_rxq *rxq = &ctx->rxq[0]; 2872 2873 if (rxq->buffers) { 2874 dma_free_coherent(dev, 2875 ctx->rx_ring_size * ctx->rx_buffer_size, 2876 rxq->buffers, rxq->buffers_dma); 2877 rxq->buffers = NULL; 2878 } 2879 } 2880 2881 static unsigned int macb_tx_ring_size_per_queue(struct macb_context *ctx) 2882 { 2883 return macb_dma_desc_get_size(ctx->info) * ctx->tx_ring_size + 2884 ctx->info->tx_bd_rd_prefetch; 2885 } 2886 2887 static unsigned int macb_rx_ring_size_per_queue(struct macb_context *ctx) 2888 { 2889 return macb_dma_desc_get_size(ctx->info) * ctx->rx_ring_size + 2890 ctx->info->rx_bd_rd_prefetch; 2891 } 2892 2893 static void macb_free(struct macb_context *ctx) 2894 { 2895 struct device *dev = &ctx->info->pdev->dev; 2896 struct macb_txq *txq; 2897 struct macb_rxq *rxq; 2898 unsigned int q; 2899 size_t size; 2900 2901 ctx->info->macbgem_ops.mog_free_rx_buffers(ctx); 2902 2903 txq = &ctx->txq[0]; 2904 size = ctx->info->num_queues * macb_tx_ring_size_per_queue(ctx); 2905 dma_free_coherent(dev, size, txq->ring, txq->ring_dma); 2906 2907 rxq = &ctx->rxq[0]; 2908 size = ctx->info->num_queues * macb_rx_ring_size_per_queue(ctx); 2909 dma_free_coherent(dev, size, rxq->ring, rxq->ring_dma); 2910 2911 for (q = 0; q < ctx->info->num_queues; ++q) { 2912 txq = &ctx->txq[q]; 2913 rxq = &ctx->rxq[q]; 2914 2915 if (txq->skb) { 2916 unsigned int dropped = 0, tail; 2917 2918 for (tail = txq->tail; tail != txq->head; tail++) { 2919 if (txq->skb[macb_tx_ring_wrap(ctx, tail)].skb) 2920 dropped++; 2921 macb_tx_unmap(ctx->info, 2922 &txq->skb[macb_tx_ring_wrap(ctx, tail)], 2923 0); 2924 } 2925 2926 ctx->info->queue_stats[q]->tx_dropped += dropped; 2927 ctx->info->netdev->stats.tx_dropped += dropped; 2928 2929 kfree(txq->skb); 2930 txq->skb = NULL; 2931 } 2932 2933 txq->ring = NULL; 2934 rxq->ring = NULL; 2935 } 2936 } 2937 2938 static int gem_alloc_rx_buffers(struct macb_context *ctx) 2939 { 2940 struct macb_rxq *rxq; 2941 unsigned int q; 2942 int size; 2943 2944 for (q = 0; q < ctx->info->num_queues; ++q) { 2945 rxq = &ctx->rxq[q]; 2946 size = ctx->rx_ring_size * sizeof(struct sk_buff *); 2947 rxq->skbuff = kzalloc(size, GFP_KERNEL); 2948 if (!rxq->skbuff) 2949 return -ENOMEM; 2950 else 2951 netdev_dbg(ctx->info->netdev, 2952 "Allocated %d RX struct sk_buff entries at %p\n", 2953 ctx->rx_ring_size, rxq->skbuff); 2954 } 2955 return 0; 2956 } 2957 2958 static int macb_alloc_rx_buffers(struct macb_context *ctx) 2959 { 2960 struct device *dev = &ctx->info->pdev->dev; 2961 struct macb_rxq *rxq = &ctx->rxq[0]; 2962 int size; 2963 2964 size = ctx->rx_ring_size * ctx->rx_buffer_size; 2965 rxq->buffers = dma_alloc_coherent(dev, size, 2966 &rxq->buffers_dma, GFP_KERNEL); 2967 if (!rxq->buffers) 2968 return -ENOMEM; 2969 2970 netdev_dbg(ctx->info->netdev, 2971 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2972 size, (unsigned long)rxq->buffers_dma, rxq->buffers); 2973 return 0; 2974 } 2975 2976 static int macb_alloc(struct macb_context *ctx) 2977 { 2978 unsigned int num_queues = ctx->info->num_queues; 2979 struct device *dev = &ctx->info->pdev->dev; 2980 dma_addr_t tx_dma, rx_dma; 2981 struct macb_txq *txq; 2982 struct macb_rxq *rxq; 2983 unsigned int q; 2984 void *tx, *rx; 2985 size_t size; 2986 2987 /* 2988 * Upper 32-bits of Tx/Rx DMA descriptor for each queues much match! 2989 * We cannot enforce this guarantee, the best we can do is do a single 2990 * allocation and hope it will land into alloc_pages() that guarantees 2991 * natural alignment of physical addresses. 2992 */ 2993 2994 size = num_queues * macb_tx_ring_size_per_queue(ctx); 2995 tx = dma_alloc_coherent(dev, size, &tx_dma, GFP_KERNEL); 2996 if (!tx) 2997 goto out_err; 2998 /* Record the buffer so that the error path frees it. */ 2999 txq = &ctx->txq[0]; 3000 txq->ring = tx; 3001 txq->ring_dma = tx_dma; 3002 if (upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) 3003 goto out_err; 3004 netdev_dbg(ctx->info->netdev, 3005 "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n", 3006 size, num_queues, (unsigned long)tx_dma, tx); 3007 3008 size = num_queues * macb_rx_ring_size_per_queue(ctx); 3009 rx = dma_alloc_coherent(dev, size, &rx_dma, GFP_KERNEL); 3010 if (!rx) 3011 goto out_err; 3012 /* Record the buffer so that the error path frees it. */ 3013 rxq = &ctx->rxq[0]; 3014 rxq->ring = rx; 3015 rxq->ring_dma = rx_dma; 3016 if (upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) 3017 goto out_err; 3018 netdev_dbg(ctx->info->netdev, 3019 "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n", 3020 size, num_queues, (unsigned long)rx_dma, rx); 3021 3022 for (q = 0; q < num_queues; ++q) { 3023 txq = &ctx->txq[q]; 3024 rxq = &ctx->rxq[q]; 3025 3026 txq->ring = tx + macb_tx_ring_size_per_queue(ctx) * q; 3027 txq->ring_dma = tx_dma + macb_tx_ring_size_per_queue(ctx) * q; 3028 3029 rxq->ring = rx + macb_rx_ring_size_per_queue(ctx) * q; 3030 rxq->ring_dma = rx_dma + macb_rx_ring_size_per_queue(ctx) * q; 3031 3032 size = ctx->tx_ring_size * sizeof(struct macb_tx_skb); 3033 txq->skb = kmalloc(size, GFP_KERNEL); 3034 if (!txq->skb) 3035 goto out_err; 3036 } 3037 if (ctx->info->macbgem_ops.mog_alloc_rx_buffers(ctx)) 3038 goto out_err; 3039 3040 return 0; 3041 3042 out_err: 3043 macb_free(ctx); 3044 return -ENOMEM; 3045 } 3046 3047 static struct macb_context *macb_context_alloc(struct macb *bp, 3048 unsigned int mtu, 3049 unsigned int rx_ring_size, 3050 unsigned int tx_ring_size) 3051 { 3052 struct macb_context *ctx; 3053 int err; 3054 3055 ctx = kzalloc_obj(*ctx); 3056 if (!ctx) 3057 return ERR_PTR(-ENOMEM); 3058 3059 ctx->info = &bp->info; 3060 ctx->rx_buffer_size = macb_rx_buffer_size(bp, mtu); 3061 ctx->rx_ring_size = rx_ring_size; 3062 ctx->tx_ring_size = tx_ring_size; 3063 3064 err = macb_alloc(ctx); 3065 if (err) { 3066 netdev_err(bp->netdev, 3067 "Unable to allocate DMA memory (error %d)\n", err); 3068 goto free_ctx; 3069 } 3070 3071 err = bp->macbgem_ops.mog_init_rings(ctx); 3072 if (err) 3073 goto free_rings; 3074 3075 return ctx; 3076 3077 free_rings: 3078 macb_free(ctx); 3079 free_ctx: 3080 kfree(ctx); 3081 return ERR_PTR(err); 3082 } 3083 3084 static int gem_init_rx_ring(struct macb_context *ctx, unsigned int q) 3085 { 3086 struct macb_rxq *rxq = &ctx->rxq[q]; 3087 unsigned int i; 3088 3089 rxq->tail = 0; 3090 rxq->prepared_head = 0; 3091 3092 for (i = 0; i < ctx->rx_ring_size; i++) 3093 macb_rx_desc(ctx, q, i)->addr |= MACB_BIT(RX_USED); 3094 3095 return gem_rx_refill(ctx, q); 3096 } 3097 3098 static int gem_init_rings(struct macb_context *ctx) 3099 { 3100 struct macb_dma_desc *desc = NULL; 3101 struct macb_txq *txq; 3102 int last_err = 0; 3103 unsigned int q; 3104 int err; 3105 int i; 3106 3107 for (q = 0; q < ctx->info->num_queues; ++q) { 3108 txq = &ctx->txq[q]; 3109 for (i = 0; i < ctx->tx_ring_size; i++) { 3110 desc = macb_tx_desc(ctx, q, i); 3111 macb_set_addr(ctx->info, desc, 0); 3112 desc->ctrl = MACB_BIT(TX_USED); 3113 } 3114 desc->ctrl |= MACB_BIT(TX_WRAP); 3115 txq->head = 0; 3116 txq->tail = 0; 3117 3118 err = gem_init_rx_ring(ctx, q); 3119 if (err) 3120 last_err = err; 3121 } 3122 3123 return last_err; 3124 } 3125 3126 static int macb_init_rings(struct macb_context *ctx) 3127 { 3128 struct macb_txq *txq = &ctx->txq[0]; 3129 struct macb_dma_desc *desc = NULL; 3130 int i; 3131 3132 macb_init_rx_ring(ctx, 0); 3133 3134 for (i = 0; i < ctx->tx_ring_size; i++) { 3135 desc = macb_tx_desc(ctx, 0, i); 3136 macb_set_addr(ctx->info, desc, 0); 3137 desc->ctrl = MACB_BIT(TX_USED); 3138 } 3139 txq->head = 0; 3140 txq->tail = 0; 3141 desc->ctrl |= MACB_BIT(TX_WRAP); 3142 3143 return 0; 3144 } 3145 3146 static void macb_reset_hw(struct macb *bp) 3147 { 3148 struct macb_queue *queue; 3149 unsigned int q; 3150 u32 ctrl = macb_readl(bp, NCR); 3151 3152 /* Disable RX and TX (XXX: Should we halt the transmission 3153 * more gracefully?) 3154 */ 3155 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 3156 3157 /* Clear the stats registers (XXX: Update stats first?) */ 3158 ctrl |= MACB_BIT(CLRSTAT); 3159 3160 macb_writel(bp, NCR, ctrl); 3161 3162 /* Clear all status flags */ 3163 macb_writel(bp, TSR, -1); 3164 macb_writel(bp, RSR, -1); 3165 3166 /* Disable RX partial store and forward and reset watermark value */ 3167 gem_writel(bp, PBUFRXCUT, 0); 3168 3169 /* Disable all interrupts */ 3170 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3171 queue_writel(queue, IDR, -1); 3172 queue_readl(queue, ISR); 3173 macb_queue_isr_clear(bp, queue, -1); 3174 } 3175 } 3176 3177 static u32 gem_mdc_clk_div(struct macb *bp) 3178 { 3179 u32 config; 3180 unsigned long pclk_hz = clk_get_rate(bp->pclk); 3181 3182 if (pclk_hz <= 20000000) 3183 config = GEM_BF(CLK, GEM_CLK_DIV8); 3184 else if (pclk_hz <= 40000000) 3185 config = GEM_BF(CLK, GEM_CLK_DIV16); 3186 else if (pclk_hz <= 80000000) 3187 config = GEM_BF(CLK, GEM_CLK_DIV32); 3188 else if (pclk_hz <= 120000000) 3189 config = GEM_BF(CLK, GEM_CLK_DIV48); 3190 else if (pclk_hz <= 160000000) 3191 config = GEM_BF(CLK, GEM_CLK_DIV64); 3192 else if (pclk_hz <= 240000000) 3193 config = GEM_BF(CLK, GEM_CLK_DIV96); 3194 else if (pclk_hz <= 320000000) 3195 config = GEM_BF(CLK, GEM_CLK_DIV128); 3196 else 3197 config = GEM_BF(CLK, GEM_CLK_DIV224); 3198 3199 return config; 3200 } 3201 3202 static u32 macb_mdc_clk_div(struct macb *bp) 3203 { 3204 u32 config; 3205 unsigned long pclk_hz; 3206 3207 if (macb_is_gem(&bp->info)) 3208 return gem_mdc_clk_div(bp); 3209 3210 pclk_hz = clk_get_rate(bp->pclk); 3211 if (pclk_hz <= 20000000) 3212 config = MACB_BF(CLK, MACB_CLK_DIV8); 3213 else if (pclk_hz <= 40000000) 3214 config = MACB_BF(CLK, MACB_CLK_DIV16); 3215 else if (pclk_hz <= 80000000) 3216 config = MACB_BF(CLK, MACB_CLK_DIV32); 3217 else 3218 config = MACB_BF(CLK, MACB_CLK_DIV64); 3219 3220 return config; 3221 } 3222 3223 /* Get the DMA bus width field of the network configuration register that we 3224 * should program. We find the width from decoding the design configuration 3225 * register to find the maximum supported data bus width. 3226 */ 3227 static u32 macb_dbw(struct macb *bp) 3228 { 3229 if (!macb_is_gem(&bp->info)) 3230 return 0; 3231 3232 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 3233 case 4: 3234 return GEM_BF(DBW, GEM_DBW128); 3235 case 2: 3236 return GEM_BF(DBW, GEM_DBW64); 3237 case 1: 3238 default: 3239 return GEM_BF(DBW, GEM_DBW32); 3240 } 3241 } 3242 3243 /* Configure the receive DMA engine 3244 * - use the correct receive buffer size 3245 * - set best burst length for DMA operations 3246 * (if not supported by FIFO, it will fallback to default) 3247 * - set both rx/tx packet buffers to full memory size 3248 * These are configurable parameters for GEM. 3249 */ 3250 static void macb_configure_dma(struct macb *bp) 3251 { 3252 struct macb_queue *queue; 3253 u32 buffer_size; 3254 unsigned int q; 3255 u32 dmacfg; 3256 3257 buffer_size = bp->ctx->rx_buffer_size / RX_BUFFER_MULTIPLE; 3258 if (macb_is_gem(&bp->info)) { 3259 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 3260 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3261 if (q) 3262 queue_writel(queue, RBQS, buffer_size); 3263 else 3264 dmacfg |= GEM_BF(RXBS, buffer_size); 3265 } 3266 if (bp->dma_burst_length) 3267 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 3268 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 3269 dmacfg &= ~GEM_BIT(ENDIA_PKT); 3270 3271 if (bp->native_io) 3272 dmacfg &= ~GEM_BIT(ENDIA_DESC); 3273 else 3274 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 3275 3276 if (bp->netdev->features & NETIF_F_HW_CSUM) 3277 dmacfg |= GEM_BIT(TXCOEN); 3278 else 3279 dmacfg &= ~GEM_BIT(TXCOEN); 3280 3281 dmacfg &= ~GEM_BIT(ADDR64); 3282 if (macb_dma64(&bp->info)) 3283 dmacfg |= GEM_BIT(ADDR64); 3284 if (macb_dma_ptp(&bp->info)) 3285 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 3286 netdev_dbg(bp->netdev, "Cadence configure DMA with 0x%08x\n", 3287 dmacfg); 3288 gem_writel(bp, DMACFG, dmacfg); 3289 } 3290 } 3291 3292 static void macb_context_swap_start(struct macb *bp) 3293 { 3294 struct macb_queue *queue; 3295 unsigned long flags; 3296 unsigned int q; 3297 u32 ctrl; 3298 3299 mutex_lock(&bp->mac_cfg_lock); 3300 3301 macb_quiesce_start(bp); 3302 3303 /* Can finally disable software Tx; need to wait until napi_tx and 3304 * tx_error_task cannot be scheduled as either might wakeup Tx. 3305 */ 3306 netif_tx_disable(bp->netdev); 3307 3308 /* Now that everything is stopped, clear DQL. */ 3309 for (q = 0; q < bp->num_queues; ++q) 3310 netdev_tx_reset_queue(netdev_get_tx_queue(bp->netdev, q)); 3311 3312 /* Safe to call outside bp->lock because bp->irq_quiesced ensures the IRQ 3313 * handling is a no-op and all BH features are disabled. 3314 * 3315 * Whether it fails or not we'll disable TE/RE next. 3316 * We were just trying to be nice. 3317 */ 3318 macb_halt_tx(bp); 3319 3320 spin_lock_irqsave(&bp->lock, flags); 3321 3322 ctrl = macb_readl(bp, NCR); 3323 macb_writel(bp, NCR, ctrl & ~(MACB_BIT(RE) | MACB_BIT(TE))); 3324 3325 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3326 queue_writel(queue, IDR, -1); 3327 queue_readl(queue, ISR); 3328 macb_queue_isr_clear(bp, queue, -1); 3329 } 3330 3331 macb_writel(bp, TSR, -1); 3332 macb_writel(bp, RSR, -1); 3333 3334 spin_unlock_irqrestore(&bp->lock, flags); 3335 } 3336 3337 static void macb_context_swap_end(struct macb *bp, 3338 struct macb_context *new_ctx) 3339 { 3340 struct macb_context *old_ctx; 3341 struct macb_queue *queue; 3342 unsigned long flags; 3343 unsigned int q; 3344 u32 ctrl; 3345 3346 lockdep_assert_held(&bp->mac_cfg_lock); 3347 3348 /* Swap contexts & give buffer pointers to HW. */ 3349 3350 old_ctx = bp->ctx; 3351 WRITE_ONCE(bp->ctx, new_ctx); 3352 wmb(); /* ensure IRQ enabled below read the new context */ 3353 macb_init_buffers(bp); 3354 3355 /* Start NAPI, HW Tx/Rx and software Tx. */ 3356 3357 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3358 napi_enable(&queue->napi_rx); 3359 napi_enable(&queue->napi_tx); 3360 } 3361 3362 spin_lock_irqsave(&bp->lock, flags); 3363 3364 /* Re-arm normal interrupt processing before enabling IRQs. */ 3365 bp->irq_quiesced = false; 3366 3367 macb_configure_dma(bp); 3368 3369 if (bp->link_up) { 3370 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3371 queue_writel(queue, IER, 3372 bp->rx_intr_mask | 3373 MACB_TX_INT_FLAGS | 3374 MACB_BIT(HRESP)); 3375 } 3376 3377 ctrl = macb_readl(bp, NCR); 3378 macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); 3379 } 3380 3381 spin_unlock_irqrestore(&bp->lock, flags); 3382 3383 if (bp->link_up) { 3384 netif_tx_wake_all_queues(bp->netdev); 3385 3386 if (bp->eee_active) 3387 macb_txp_lpi_initial_defer(bp); 3388 } 3389 3390 mutex_unlock(&bp->mac_cfg_lock); 3391 3392 /* Free old context. */ 3393 3394 macb_free(old_ctx); 3395 kfree(old_ctx); 3396 } 3397 3398 static void macb_init_hw(struct macb *bp) 3399 { 3400 u32 config; 3401 3402 macb_reset_hw(bp); 3403 macb_set_hwaddr(bp); 3404 3405 config = macb_mdc_clk_div(bp); 3406 /* Make eth data aligned. 3407 * If RSC capable, that offset is ignored by HW. 3408 */ 3409 if (!(bp->caps & MACB_CAPS_RSC)) 3410 config |= MACB_BF(RBOF, NET_IP_ALIGN); 3411 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 3412 if (bp->caps & MACB_CAPS_JUMBO) 3413 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 3414 else 3415 config |= MACB_BIT(BIG); /* Receive oversized frames */ 3416 if (bp->netdev->flags & IFF_PROMISC) 3417 config |= MACB_BIT(CAF); /* Copy All Frames */ 3418 else if (macb_is_gem(&bp->info) && 3419 bp->netdev->features & NETIF_F_RXCSUM) 3420 config |= GEM_BIT(RXCOEN); 3421 if (!(bp->netdev->flags & IFF_BROADCAST)) 3422 config |= MACB_BIT(NBC); /* No BroadCast */ 3423 config |= macb_dbw(bp); 3424 macb_writel(bp, NCFGR, config); 3425 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 3426 gem_writel(bp, JML, bp->jumbo_max_len); 3427 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 3428 if (bp->caps & MACB_CAPS_JUMBO) 3429 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 3430 3431 macb_configure_dma(bp); 3432 3433 /* Enable RX partial store and forward and set watermark */ 3434 if (bp->rx_watermark) 3435 gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU))); 3436 } 3437 3438 /* The hash address register is 64 bits long and takes up two 3439 * locations in the memory map. The least significant bits are stored 3440 * in EMAC_HSL and the most significant bits in EMAC_HSH. 3441 * 3442 * The unicast hash enable and the multicast hash enable bits in the 3443 * network configuration register enable the reception of hash matched 3444 * frames. The destination address is reduced to a 6 bit index into 3445 * the 64 bit hash register using the following hash function. The 3446 * hash function is an exclusive or of every sixth bit of the 3447 * destination address. 3448 * 3449 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 3450 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 3451 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 3452 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 3453 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 3454 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 3455 * 3456 * da[0] represents the least significant bit of the first byte 3457 * received, that is, the multicast/unicast indicator, and da[47] 3458 * represents the most significant bit of the last byte received. If 3459 * the hash index, hi[n], points to a bit that is set in the hash 3460 * register then the frame will be matched according to whether the 3461 * frame is multicast or unicast. A multicast match will be signalled 3462 * if the multicast hash enable bit is set, da[0] is 1 and the hash 3463 * index points to a bit set in the hash register. A unicast match 3464 * will be signalled if the unicast hash enable bit is set, da[0] is 0 3465 * and the hash index points to a bit set in the hash register. To 3466 * receive all multicast frames, the hash register should be set with 3467 * all ones and the multicast hash enable bit should be set in the 3468 * network configuration register. 3469 */ 3470 3471 static inline int hash_bit_value(int bitnr, __u8 *addr) 3472 { 3473 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 3474 return 1; 3475 return 0; 3476 } 3477 3478 /* Return the hash index value for the specified address. */ 3479 static int hash_get_index(__u8 *addr) 3480 { 3481 int i, j, bitval; 3482 int hash_index = 0; 3483 3484 for (j = 0; j < 6; j++) { 3485 for (i = 0, bitval = 0; i < 8; i++) 3486 bitval ^= hash_bit_value(i * 6 + j, addr); 3487 3488 hash_index |= (bitval << j); 3489 } 3490 3491 return hash_index; 3492 } 3493 3494 /* Add multicast addresses to the internal multicast-hash table. */ 3495 static void macb_sethashtable(struct net_device *netdev) 3496 { 3497 struct netdev_hw_addr *ha; 3498 unsigned long mc_filter[2]; 3499 unsigned int bitnr; 3500 struct macb *bp = netdev_priv(netdev); 3501 3502 mc_filter[0] = 0; 3503 mc_filter[1] = 0; 3504 3505 netdev_for_each_mc_addr(ha, netdev) { 3506 bitnr = hash_get_index(ha->addr); 3507 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 3508 } 3509 3510 macb_or_gem_writel(bp, HRB, mc_filter[0]); 3511 macb_or_gem_writel(bp, HRT, mc_filter[1]); 3512 } 3513 3514 /* Enable/Disable promiscuous and multicast modes. */ 3515 static void macb_set_rx_mode(struct net_device *netdev) 3516 { 3517 unsigned long cfg; 3518 struct macb *bp = netdev_priv(netdev); 3519 3520 cfg = macb_readl(bp, NCFGR); 3521 3522 if (netdev->flags & IFF_PROMISC) { 3523 /* Enable promiscuous mode */ 3524 cfg |= MACB_BIT(CAF); 3525 3526 /* Disable RX checksum offload */ 3527 if (macb_is_gem(&bp->info)) 3528 cfg &= ~GEM_BIT(RXCOEN); 3529 } else { 3530 /* Disable promiscuous mode */ 3531 cfg &= ~MACB_BIT(CAF); 3532 3533 /* Enable RX checksum offload only if requested */ 3534 if (macb_is_gem(&bp->info) && netdev->features & NETIF_F_RXCSUM) 3535 cfg |= GEM_BIT(RXCOEN); 3536 } 3537 3538 if (netdev->flags & IFF_ALLMULTI) { 3539 /* Enable all multicast mode */ 3540 macb_or_gem_writel(bp, HRB, -1); 3541 macb_or_gem_writel(bp, HRT, -1); 3542 cfg |= MACB_BIT(NCFGR_MTI); 3543 } else if (!netdev_mc_empty(netdev)) { 3544 /* Enable specific multicasts */ 3545 macb_sethashtable(netdev); 3546 cfg |= MACB_BIT(NCFGR_MTI); 3547 } else if (netdev->flags & (~IFF_ALLMULTI)) { 3548 /* Disable all multicast mode */ 3549 macb_or_gem_writel(bp, HRB, 0); 3550 macb_or_gem_writel(bp, HRT, 0); 3551 cfg &= ~MACB_BIT(NCFGR_MTI); 3552 } 3553 3554 macb_writel(bp, NCFGR, cfg); 3555 } 3556 3557 static int macb_open(struct net_device *netdev) 3558 { 3559 struct macb *bp = netdev_priv(netdev); 3560 struct macb_context *ctx; 3561 int err; 3562 3563 netdev_dbg(bp->netdev, "open\n"); 3564 3565 err = pm_runtime_resume_and_get(&bp->pdev->dev); 3566 if (err < 0) 3567 return err; 3568 3569 ctx = macb_context_alloc(bp, netdev->mtu, 3570 bp->configured_rx_ring_size, 3571 bp->configured_tx_ring_size); 3572 if (IS_ERR(ctx)) { 3573 err = PTR_ERR(ctx); 3574 bp->ctx = NULL; 3575 goto pm_exit; 3576 } 3577 3578 WRITE_ONCE(bp->ctx, ctx); 3579 wmb(); /* ensure IRQ enabled at quiesce end read the new context */ 3580 3581 macb_init_buffers(bp); 3582 3583 macb_quiesce_end(bp); 3584 3585 macb_init_hw(bp); 3586 3587 err = phy_set_mode_ext(bp->phy, PHY_MODE_ETHERNET, bp->phy_interface); 3588 if (err) 3589 goto reset_hw; 3590 3591 err = phy_power_on(bp->phy); 3592 if (err) 3593 goto reset_hw; 3594 3595 err = macb_phylink_connect(bp); 3596 if (err) 3597 goto phy_off; 3598 3599 netif_tx_start_all_queues(netdev); 3600 3601 if (bp->ptp_info) 3602 bp->ptp_info->ptp_init(netdev); 3603 3604 return 0; 3605 3606 phy_off: 3607 phy_power_off(bp->phy); 3608 3609 reset_hw: 3610 /* The netdev stays down: quiesce and drain, as macb_close() does. */ 3611 macb_quiesce_start(bp); 3612 3613 macb_reset_hw(bp); 3614 macb_free(bp->ctx); 3615 kfree(bp->ctx); 3616 bp->ctx = NULL; 3617 pm_exit: 3618 pm_runtime_put_sync(&bp->pdev->dev); 3619 return err; 3620 } 3621 3622 static int macb_close(struct net_device *netdev) 3623 { 3624 struct macb *bp = netdev_priv(netdev); 3625 unsigned long flags; 3626 unsigned int q; 3627 3628 macb_quiesce_start(bp); 3629 3630 /* Drain the BH contexts before stopping the queues: NAPI completion 3631 * and tx_error_task wake them up. 3632 */ 3633 netif_tx_stop_all_queues(netdev); 3634 for (q = 0; q < bp->num_queues; ++q) 3635 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, q)); 3636 3637 phylink_stop(bp->phylink); 3638 phylink_disconnect_phy(bp->phylink); 3639 3640 phy_power_off(bp->phy); 3641 3642 spin_lock_irqsave(&bp->lock, flags); 3643 macb_reset_hw(bp); 3644 netif_carrier_off(netdev); 3645 spin_unlock_irqrestore(&bp->lock, flags); 3646 3647 macb_free(bp->ctx); 3648 kfree(bp->ctx); 3649 bp->ctx = NULL; 3650 3651 if (bp->ptp_info) 3652 bp->ptp_info->ptp_remove(netdev); 3653 3654 pm_runtime_put(&bp->pdev->dev); 3655 3656 return 0; 3657 } 3658 3659 static int macb_change_mtu(struct net_device *netdev, int new_mtu) 3660 { 3661 struct macb *bp = netdev_priv(netdev); 3662 bool running = netif_running(netdev); 3663 struct macb_context *new_ctx; 3664 3665 if (running) { 3666 /* Context swapping is not supported for AT91. */ 3667 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) 3668 return -EBUSY; 3669 3670 new_ctx = macb_context_alloc(bp, new_mtu, 3671 bp->configured_rx_ring_size, 3672 bp->configured_tx_ring_size); 3673 if (IS_ERR(new_ctx)) 3674 return PTR_ERR(new_ctx); 3675 3676 macb_context_swap_start(bp); 3677 } 3678 3679 WRITE_ONCE(netdev->mtu, new_mtu); 3680 3681 if (running) 3682 macb_context_swap_end(bp, new_ctx); 3683 3684 return 0; 3685 } 3686 3687 static int macb_set_mac_addr(struct net_device *netdev, void *addr) 3688 { 3689 int err; 3690 3691 err = eth_mac_addr(netdev, addr); 3692 if (err < 0) 3693 return err; 3694 3695 macb_set_hwaddr(netdev_priv(netdev)); 3696 return 0; 3697 } 3698 3699 static void gem_update_stats(struct macb *bp) 3700 { 3701 struct macb_queue *queue; 3702 unsigned int i, q, idx; 3703 unsigned long *stat; 3704 3705 u64 *p = &bp->hw_stats.gem.tx_octets; 3706 3707 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 3708 u32 offset = gem_statistics[i].offset; 3709 u64 val = bp->macb_reg_readl(bp, offset); 3710 3711 bp->ethtool_stats[i] += val; 3712 *p += val; 3713 3714 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 3715 /* Add GEM_OCTTXH, GEM_OCTRXH */ 3716 val = bp->macb_reg_readl(bp, offset + 4); 3717 bp->ethtool_stats[i] += ((u64)val) << 32; 3718 *p += ((u64)val) << 32; 3719 } 3720 } 3721 3722 idx = GEM_STATS_LEN; 3723 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 3724 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 3725 bp->ethtool_stats[idx++] = *stat; 3726 } 3727 3728 static void gem_get_stats(struct macb *bp, struct rtnl_link_stats64 *nstat) 3729 { 3730 struct gem_stats *hwstat = &bp->hw_stats.gem; 3731 3732 spin_lock_irq(&bp->stats_lock); 3733 if (netif_running(bp->netdev)) 3734 gem_update_stats(bp); 3735 3736 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 3737 hwstat->rx_alignment_errors + 3738 hwstat->rx_resource_errors + 3739 hwstat->rx_overruns + 3740 hwstat->rx_oversize_frames + 3741 hwstat->rx_jabbers + 3742 hwstat->rx_undersized_frames + 3743 hwstat->rx_length_field_frame_errors); 3744 nstat->tx_errors = (hwstat->tx_late_collisions + 3745 hwstat->tx_excessive_collisions + 3746 hwstat->tx_underrun + 3747 hwstat->tx_carrier_sense_errors); 3748 nstat->multicast = hwstat->rx_multicast_frames; 3749 nstat->collisions = (hwstat->tx_single_collision_frames + 3750 hwstat->tx_multiple_collision_frames + 3751 hwstat->tx_excessive_collisions); 3752 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 3753 hwstat->rx_jabbers + 3754 hwstat->rx_undersized_frames + 3755 hwstat->rx_length_field_frame_errors); 3756 nstat->rx_over_errors = hwstat->rx_resource_errors; 3757 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 3758 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 3759 nstat->rx_fifo_errors = hwstat->rx_overruns; 3760 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 3761 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 3762 nstat->tx_fifo_errors = hwstat->tx_underrun; 3763 spin_unlock_irq(&bp->stats_lock); 3764 } 3765 3766 static void gem_get_ethtool_stats(struct net_device *netdev, 3767 struct ethtool_stats *stats, u64 *data) 3768 { 3769 struct macb *bp = netdev_priv(netdev); 3770 3771 spin_lock_irq(&bp->stats_lock); 3772 gem_update_stats(bp); 3773 memcpy(data, &bp->ethtool_stats, sizeof(u64) 3774 * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); 3775 spin_unlock_irq(&bp->stats_lock); 3776 } 3777 3778 static int gem_get_sset_count(struct net_device *netdev, int sset) 3779 { 3780 struct macb *bp = netdev_priv(netdev); 3781 3782 switch (sset) { 3783 case ETH_SS_STATS: 3784 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 3785 default: 3786 return -EOPNOTSUPP; 3787 } 3788 } 3789 3790 static void gem_get_ethtool_strings(struct net_device *netdev, u32 sset, u8 *p) 3791 { 3792 struct macb *bp = netdev_priv(netdev); 3793 struct macb_queue *queue; 3794 unsigned int i; 3795 unsigned int q; 3796 3797 switch (sset) { 3798 case ETH_SS_STATS: 3799 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 3800 memcpy(p, gem_statistics[i].stat_string, 3801 ETH_GSTRING_LEN); 3802 3803 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3804 for (i = 0; i < QUEUE_STATS_LEN; i++) 3805 ethtool_sprintf(&p, "q%u_%s", q, queue_statistics[i].stat_string); 3806 } 3807 break; 3808 } 3809 } 3810 3811 static void macb_get_stats(struct net_device *netdev, 3812 struct rtnl_link_stats64 *nstat) 3813 { 3814 struct macb *bp = netdev_priv(netdev); 3815 struct macb_stats *hwstat = &bp->hw_stats.macb; 3816 3817 netdev_stats_to_stats64(nstat, &bp->netdev->stats); 3818 if (macb_is_gem(&bp->info)) { 3819 gem_get_stats(bp, nstat); 3820 return; 3821 } 3822 3823 /* read stats from hardware */ 3824 spin_lock_irq(&bp->stats_lock); 3825 macb_update_stats(bp); 3826 3827 /* Convert HW stats into netdevice stats */ 3828 nstat->rx_errors = (hwstat->rx_fcs_errors + 3829 hwstat->rx_align_errors + 3830 hwstat->rx_resource_errors + 3831 hwstat->rx_overruns + 3832 hwstat->rx_oversize_pkts + 3833 hwstat->rx_jabbers + 3834 hwstat->rx_undersize_pkts + 3835 hwstat->rx_length_mismatch); 3836 nstat->tx_errors = (hwstat->tx_late_cols + 3837 hwstat->tx_excessive_cols + 3838 hwstat->tx_underruns + 3839 hwstat->tx_carrier_errors + 3840 hwstat->sqe_test_errors); 3841 nstat->collisions = (hwstat->tx_single_cols + 3842 hwstat->tx_multiple_cols + 3843 hwstat->tx_excessive_cols); 3844 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 3845 hwstat->rx_jabbers + 3846 hwstat->rx_undersize_pkts + 3847 hwstat->rx_length_mismatch); 3848 nstat->rx_over_errors = hwstat->rx_resource_errors + 3849 hwstat->rx_overruns; 3850 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 3851 nstat->rx_frame_errors = hwstat->rx_align_errors; 3852 nstat->rx_fifo_errors = hwstat->rx_overruns; 3853 /* XXX: What does "missed" mean? */ 3854 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 3855 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 3856 nstat->tx_fifo_errors = hwstat->tx_underruns; 3857 /* Don't know about heartbeat or window errors... */ 3858 spin_unlock_irq(&bp->stats_lock); 3859 } 3860 3861 static void macb_get_pause_stats(struct net_device *netdev, 3862 struct ethtool_pause_stats *pause_stats) 3863 { 3864 struct macb *bp = netdev_priv(netdev); 3865 struct macb_stats *hwstat = &bp->hw_stats.macb; 3866 3867 spin_lock_irq(&bp->stats_lock); 3868 macb_update_stats(bp); 3869 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; 3870 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; 3871 spin_unlock_irq(&bp->stats_lock); 3872 } 3873 3874 static void gem_get_pause_stats(struct net_device *netdev, 3875 struct ethtool_pause_stats *pause_stats) 3876 { 3877 struct macb *bp = netdev_priv(netdev); 3878 struct gem_stats *hwstat = &bp->hw_stats.gem; 3879 3880 spin_lock_irq(&bp->stats_lock); 3881 gem_update_stats(bp); 3882 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; 3883 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; 3884 spin_unlock_irq(&bp->stats_lock); 3885 } 3886 3887 static void macb_get_eth_mac_stats(struct net_device *netdev, 3888 struct ethtool_eth_mac_stats *mac_stats) 3889 { 3890 struct macb *bp = netdev_priv(netdev); 3891 struct macb_stats *hwstat = &bp->hw_stats.macb; 3892 3893 spin_lock_irq(&bp->stats_lock); 3894 macb_update_stats(bp); 3895 mac_stats->FramesTransmittedOK = hwstat->tx_ok; 3896 mac_stats->SingleCollisionFrames = hwstat->tx_single_cols; 3897 mac_stats->MultipleCollisionFrames = hwstat->tx_multiple_cols; 3898 mac_stats->FramesReceivedOK = hwstat->rx_ok; 3899 mac_stats->FrameCheckSequenceErrors = hwstat->rx_fcs_errors; 3900 mac_stats->AlignmentErrors = hwstat->rx_align_errors; 3901 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred; 3902 mac_stats->LateCollisions = hwstat->tx_late_cols; 3903 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_cols; 3904 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underruns; 3905 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_errors; 3906 mac_stats->FramesLostDueToIntMACRcvError = hwstat->rx_overruns; 3907 mac_stats->InRangeLengthErrors = hwstat->rx_length_mismatch; 3908 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_pkts; 3909 spin_unlock_irq(&bp->stats_lock); 3910 } 3911 3912 static void gem_get_eth_mac_stats(struct net_device *netdev, 3913 struct ethtool_eth_mac_stats *mac_stats) 3914 { 3915 struct macb *bp = netdev_priv(netdev); 3916 struct gem_stats *hwstat = &bp->hw_stats.gem; 3917 3918 spin_lock_irq(&bp->stats_lock); 3919 gem_update_stats(bp); 3920 mac_stats->FramesTransmittedOK = hwstat->tx_frames; 3921 mac_stats->SingleCollisionFrames = hwstat->tx_single_collision_frames; 3922 mac_stats->MultipleCollisionFrames = 3923 hwstat->tx_multiple_collision_frames; 3924 mac_stats->FramesReceivedOK = hwstat->rx_frames; 3925 mac_stats->FrameCheckSequenceErrors = 3926 hwstat->rx_frame_check_sequence_errors; 3927 mac_stats->AlignmentErrors = hwstat->rx_alignment_errors; 3928 mac_stats->OctetsTransmittedOK = hwstat->tx_octets; 3929 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred_frames; 3930 mac_stats->LateCollisions = hwstat->tx_late_collisions; 3931 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_collisions; 3932 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underrun; 3933 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_sense_errors; 3934 mac_stats->OctetsReceivedOK = hwstat->rx_octets; 3935 mac_stats->MulticastFramesXmittedOK = hwstat->tx_multicast_frames; 3936 mac_stats->BroadcastFramesXmittedOK = hwstat->tx_broadcast_frames; 3937 mac_stats->MulticastFramesReceivedOK = hwstat->rx_multicast_frames; 3938 mac_stats->BroadcastFramesReceivedOK = hwstat->rx_broadcast_frames; 3939 mac_stats->InRangeLengthErrors = hwstat->rx_length_field_frame_errors; 3940 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_frames; 3941 spin_unlock_irq(&bp->stats_lock); 3942 } 3943 3944 /* TODO: Report SQE test errors when added to phy_stats */ 3945 static void macb_get_eth_phy_stats(struct net_device *netdev, 3946 struct ethtool_eth_phy_stats *phy_stats) 3947 { 3948 struct macb *bp = netdev_priv(netdev); 3949 struct macb_stats *hwstat = &bp->hw_stats.macb; 3950 3951 spin_lock_irq(&bp->stats_lock); 3952 macb_update_stats(bp); 3953 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; 3954 spin_unlock_irq(&bp->stats_lock); 3955 } 3956 3957 static void gem_get_eth_phy_stats(struct net_device *netdev, 3958 struct ethtool_eth_phy_stats *phy_stats) 3959 { 3960 struct macb *bp = netdev_priv(netdev); 3961 struct gem_stats *hwstat = &bp->hw_stats.gem; 3962 3963 spin_lock_irq(&bp->stats_lock); 3964 gem_update_stats(bp); 3965 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; 3966 spin_unlock_irq(&bp->stats_lock); 3967 } 3968 3969 static void macb_get_rmon_stats(struct net_device *netdev, 3970 struct ethtool_rmon_stats *rmon_stats, 3971 const struct ethtool_rmon_hist_range **ranges) 3972 { 3973 struct macb *bp = netdev_priv(netdev); 3974 struct macb_stats *hwstat = &bp->hw_stats.macb; 3975 3976 spin_lock_irq(&bp->stats_lock); 3977 macb_update_stats(bp); 3978 rmon_stats->undersize_pkts = hwstat->rx_undersize_pkts; 3979 rmon_stats->oversize_pkts = hwstat->rx_oversize_pkts; 3980 rmon_stats->jabbers = hwstat->rx_jabbers; 3981 spin_unlock_irq(&bp->stats_lock); 3982 } 3983 3984 static const struct ethtool_rmon_hist_range gem_rmon_ranges[] = { 3985 { 64, 64 }, 3986 { 65, 127 }, 3987 { 128, 255 }, 3988 { 256, 511 }, 3989 { 512, 1023 }, 3990 { 1024, 1518 }, 3991 { 1519, 16384 }, 3992 { }, 3993 }; 3994 3995 static void gem_get_rmon_stats(struct net_device *netdev, 3996 struct ethtool_rmon_stats *rmon_stats, 3997 const struct ethtool_rmon_hist_range **ranges) 3998 { 3999 struct macb *bp = netdev_priv(netdev); 4000 struct gem_stats *hwstat = &bp->hw_stats.gem; 4001 4002 spin_lock_irq(&bp->stats_lock); 4003 gem_update_stats(bp); 4004 rmon_stats->undersize_pkts = hwstat->rx_undersized_frames; 4005 rmon_stats->oversize_pkts = hwstat->rx_oversize_frames; 4006 rmon_stats->jabbers = hwstat->rx_jabbers; 4007 rmon_stats->hist[0] = hwstat->rx_64_byte_frames; 4008 rmon_stats->hist[1] = hwstat->rx_65_127_byte_frames; 4009 rmon_stats->hist[2] = hwstat->rx_128_255_byte_frames; 4010 rmon_stats->hist[3] = hwstat->rx_256_511_byte_frames; 4011 rmon_stats->hist[4] = hwstat->rx_512_1023_byte_frames; 4012 rmon_stats->hist[5] = hwstat->rx_1024_1518_byte_frames; 4013 rmon_stats->hist[6] = hwstat->rx_greater_than_1518_byte_frames; 4014 rmon_stats->hist_tx[0] = hwstat->tx_64_byte_frames; 4015 rmon_stats->hist_tx[1] = hwstat->tx_65_127_byte_frames; 4016 rmon_stats->hist_tx[2] = hwstat->tx_128_255_byte_frames; 4017 rmon_stats->hist_tx[3] = hwstat->tx_256_511_byte_frames; 4018 rmon_stats->hist_tx[4] = hwstat->tx_512_1023_byte_frames; 4019 rmon_stats->hist_tx[5] = hwstat->tx_1024_1518_byte_frames; 4020 rmon_stats->hist_tx[6] = hwstat->tx_greater_than_1518_byte_frames; 4021 spin_unlock_irq(&bp->stats_lock); 4022 *ranges = gem_rmon_ranges; 4023 } 4024 4025 static int macb_get_regs_len(struct net_device *netdev) 4026 { 4027 return MACB_GREGS_NBR * sizeof(u32); 4028 } 4029 4030 static void macb_get_regs(struct net_device *netdev, struct ethtool_regs *regs, 4031 void *p) 4032 { 4033 dma_addr_t tx_dma_tail = 0, tx_dma_head = 0; 4034 struct macb *bp = netdev_priv(netdev); 4035 unsigned int tail = 0, head = 0; 4036 struct macb_txq *txq; 4037 u32 *regs_buff = p; 4038 4039 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 4040 | MACB_GREGS_VERSION; 4041 4042 if (bp->ctx) { 4043 txq = &bp->ctx->txq[0]; 4044 tail = macb_tx_ring_wrap(bp->ctx, txq->tail); 4045 head = macb_tx_ring_wrap(bp->ctx, txq->head); 4046 tx_dma_tail = macb_tx_dma(&bp->queues[0], tail); 4047 tx_dma_head = macb_tx_dma(&bp->queues[0], head); 4048 } 4049 4050 regs_buff[0] = macb_readl(bp, NCR); 4051 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 4052 regs_buff[2] = macb_readl(bp, NSR); 4053 regs_buff[3] = macb_readl(bp, TSR); 4054 regs_buff[4] = macb_readl(bp, RBQP); 4055 regs_buff[5] = macb_readl(bp, TBQP); 4056 regs_buff[6] = macb_readl(bp, RSR); 4057 regs_buff[7] = macb_readl(bp, IMR); 4058 4059 regs_buff[8] = tail; 4060 regs_buff[9] = head; 4061 regs_buff[10] = tx_dma_tail; 4062 regs_buff[11] = tx_dma_head; 4063 4064 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4065 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 4066 if (macb_is_gem(&bp->info)) 4067 regs_buff[13] = gem_readl(bp, DMACFG); 4068 } 4069 4070 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 4071 { 4072 struct macb *bp = netdev_priv(netdev); 4073 4074 phylink_ethtool_get_wol(bp->phylink, wol); 4075 wol->supported |= (WAKE_MAGIC | WAKE_ARP); 4076 4077 /* Add macb wolopts to phy wolopts */ 4078 wol->wolopts |= bp->wolopts; 4079 } 4080 4081 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 4082 { 4083 struct macb *bp = netdev_priv(netdev); 4084 int ret; 4085 4086 /* Pass the order to phylink layer */ 4087 ret = phylink_ethtool_set_wol(bp->phylink, wol); 4088 /* Don't manage WoL on MAC, if PHY set_wol() fails */ 4089 if (ret && ret != -EOPNOTSUPP) 4090 return ret; 4091 4092 bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0; 4093 bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0; 4094 bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0; 4095 4096 device_set_wakeup_enable(&bp->pdev->dev, bp->wol); 4097 4098 return 0; 4099 } 4100 4101 static int macb_get_link_ksettings(struct net_device *netdev, 4102 struct ethtool_link_ksettings *kset) 4103 { 4104 struct macb *bp = netdev_priv(netdev); 4105 4106 return phylink_ethtool_ksettings_get(bp->phylink, kset); 4107 } 4108 4109 static int macb_set_link_ksettings(struct net_device *netdev, 4110 const struct ethtool_link_ksettings *kset) 4111 { 4112 struct macb *bp = netdev_priv(netdev); 4113 4114 return phylink_ethtool_ksettings_set(bp->phylink, kset); 4115 } 4116 4117 static void macb_get_ringparam(struct net_device *netdev, 4118 struct ethtool_ringparam *ring, 4119 struct kernel_ethtool_ringparam *kernel_ring, 4120 struct netlink_ext_ack *extack) 4121 { 4122 struct macb *bp = netdev_priv(netdev); 4123 4124 ring->rx_max_pending = MAX_RX_RING_SIZE; 4125 ring->tx_max_pending = MAX_TX_RING_SIZE; 4126 4127 ring->rx_pending = bp->configured_rx_ring_size; 4128 ring->tx_pending = bp->configured_tx_ring_size; 4129 } 4130 4131 static int macb_set_ringparam(struct net_device *netdev, 4132 struct ethtool_ringparam *ring, 4133 struct kernel_ethtool_ringparam *kernel_ring, 4134 struct netlink_ext_ack *extack) 4135 { 4136 unsigned int new_rx_size, new_tx_size; 4137 struct macb *bp = netdev_priv(netdev); 4138 bool running = netif_running(netdev); 4139 struct macb_context *new_ctx; 4140 4141 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) 4142 return -EOPNOTSUPP; 4143 4144 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 4145 return -EINVAL; 4146 4147 new_rx_size = clamp_t(u32, ring->rx_pending, 4148 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 4149 new_rx_size = roundup_pow_of_two(new_rx_size); 4150 4151 new_tx_size = clamp_t(u32, ring->tx_pending, 4152 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 4153 new_tx_size = roundup_pow_of_two(new_tx_size); 4154 4155 if (new_tx_size == bp->configured_tx_ring_size && 4156 new_rx_size == bp->configured_rx_ring_size) { 4157 /* nothing to do */ 4158 return 0; 4159 } 4160 4161 if (running) { 4162 new_ctx = macb_context_alloc(bp, netdev->mtu, 4163 new_rx_size, new_tx_size); 4164 if (IS_ERR(new_ctx)) 4165 return PTR_ERR(new_ctx); 4166 4167 macb_context_swap_start(bp); 4168 } 4169 4170 bp->configured_rx_ring_size = new_rx_size; 4171 bp->configured_tx_ring_size = new_tx_size; 4172 4173 if (running) 4174 macb_context_swap_end(bp, new_ctx); 4175 4176 return 0; 4177 } 4178 4179 #ifdef CONFIG_MACB_USE_HWSTAMP 4180 static unsigned int gem_get_tsu_rate(struct macb *bp) 4181 { 4182 struct clk *tsu_clk; 4183 unsigned int tsu_rate; 4184 4185 if (!IS_ERR_OR_NULL(bp->tsu_clk)) { 4186 tsu_rate = clk_get_rate(bp->tsu_clk); 4187 } else { 4188 tsu_clk = bp->pclk; 4189 tsu_rate = clk_get_rate(tsu_clk); 4190 dev_warn(&bp->pdev->dev, "devicetree missing tsu_clk, using pclk as fallback\n"); 4191 } 4192 4193 return tsu_rate; 4194 } 4195 4196 static s32 gem_get_ptp_max_adj(void) 4197 { 4198 return 64000000; 4199 } 4200 4201 static int gem_get_ts_info(struct net_device *netdev, 4202 struct kernel_ethtool_ts_info *info) 4203 { 4204 struct macb *bp = netdev_priv(netdev); 4205 4206 if (!macb_dma_ptp(&bp->info)) { 4207 ethtool_op_get_ts_info(netdev, info); 4208 return 0; 4209 } 4210 4211 info->so_timestamping = 4212 SOF_TIMESTAMPING_TX_SOFTWARE | 4213 SOF_TIMESTAMPING_TX_HARDWARE | 4214 SOF_TIMESTAMPING_RX_HARDWARE | 4215 SOF_TIMESTAMPING_RAW_HARDWARE; 4216 info->tx_types = 4217 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 4218 (1 << HWTSTAMP_TX_OFF) | 4219 (1 << HWTSTAMP_TX_ON); 4220 info->rx_filters = 4221 (1 << HWTSTAMP_FILTER_NONE) | 4222 (1 << HWTSTAMP_FILTER_ALL); 4223 4224 if (bp->ptp_clock) 4225 info->phc_index = ptp_clock_index(bp->ptp_clock); 4226 4227 return 0; 4228 } 4229 4230 static struct macb_ptp_info gem_ptp_info = { 4231 .ptp_init = gem_ptp_init, 4232 .ptp_remove = gem_ptp_remove, 4233 .get_ptp_max_adj = gem_get_ptp_max_adj, 4234 .get_tsu_rate = gem_get_tsu_rate, 4235 .get_ts_info = gem_get_ts_info, 4236 .get_hwtst = gem_get_hwtst, 4237 .set_hwtst = gem_set_hwtst, 4238 }; 4239 #endif 4240 4241 static int macb_get_ts_info(struct net_device *netdev, 4242 struct kernel_ethtool_ts_info *info) 4243 { 4244 struct macb *bp = netdev_priv(netdev); 4245 4246 if (bp->ptp_info) 4247 return bp->ptp_info->get_ts_info(netdev, info); 4248 4249 return ethtool_op_get_ts_info(netdev, info); 4250 } 4251 4252 static void gem_enable_flow_filters(struct macb *bp, bool enable) 4253 { 4254 struct net_device *netdev = bp->netdev; 4255 struct ethtool_rx_fs_item *item; 4256 u32 t2_scr; 4257 int num_t2_scr; 4258 4259 if (!(netdev->features & NETIF_F_NTUPLE)) 4260 return; 4261 4262 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 4263 4264 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4265 struct ethtool_rx_flow_spec *fs = &item->fs; 4266 struct ethtool_tcpip4_spec *tp4sp_m; 4267 4268 if (fs->location >= num_t2_scr) 4269 continue; 4270 4271 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 4272 4273 /* enable/disable screener regs for the flow entry */ 4274 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 4275 4276 /* only enable fields with no masking */ 4277 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 4278 4279 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 4280 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 4281 else 4282 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 4283 4284 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 4285 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 4286 else 4287 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 4288 4289 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 4290 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 4291 else 4292 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 4293 4294 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 4295 } 4296 } 4297 4298 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 4299 { 4300 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 4301 uint16_t index = fs->location; 4302 u32 w0, w1, t2_scr; 4303 bool cmp_a = false; 4304 bool cmp_b = false; 4305 bool cmp_c = false; 4306 4307 if (!macb_is_gem(&bp->info)) 4308 return; 4309 4310 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 4311 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 4312 4313 /* ignore field if any masking set */ 4314 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 4315 /* 1st compare reg - IP source address */ 4316 w0 = 0; 4317 w1 = 0; 4318 w0 = tp4sp_v->ip4src; 4319 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 4320 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 4321 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 4322 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 4323 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 4324 cmp_a = true; 4325 } 4326 4327 /* ignore field if any masking set */ 4328 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 4329 /* 2nd compare reg - IP destination address */ 4330 w0 = 0; 4331 w1 = 0; 4332 w0 = tp4sp_v->ip4dst; 4333 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 4334 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 4335 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 4336 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 4337 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 4338 cmp_b = true; 4339 } 4340 4341 /* ignore both port fields if masking set in both */ 4342 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 4343 /* 3rd compare reg - source port, destination port */ 4344 w0 = 0; 4345 w1 = 0; 4346 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 4347 if (tp4sp_m->psrc == tp4sp_m->pdst) { 4348 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 4349 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 4350 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 4351 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 4352 } else { 4353 /* only one port definition */ 4354 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 4355 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 4356 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 4357 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 4358 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 4359 } else { /* dst port */ 4360 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 4361 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 4362 } 4363 } 4364 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 4365 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 4366 cmp_c = true; 4367 } 4368 4369 t2_scr = 0; 4370 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 4371 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 4372 if (cmp_a) 4373 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 4374 if (cmp_b) 4375 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 4376 if (cmp_c) 4377 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 4378 gem_writel_n(bp, SCRT2, index, t2_scr); 4379 } 4380 4381 static int gem_add_flow_filter(struct net_device *netdev, 4382 struct ethtool_rxnfc *cmd) 4383 { 4384 struct macb *bp = netdev_priv(netdev); 4385 struct ethtool_rx_flow_spec *fs = &cmd->fs; 4386 struct ethtool_rx_fs_item *item, *newfs; 4387 unsigned long flags; 4388 int ret = -EINVAL; 4389 bool added = false; 4390 4391 newfs = kmalloc_obj(*newfs); 4392 if (newfs == NULL) 4393 return -ENOMEM; 4394 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 4395 4396 netdev_dbg(netdev, 4397 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 4398 fs->flow_type, (int)fs->ring_cookie, fs->location, 4399 htonl(fs->h_u.tcp_ip4_spec.ip4src), 4400 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 4401 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), 4402 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); 4403 4404 spin_lock_irqsave(&bp->rx_fs_lock, flags); 4405 4406 /* find correct place to add in list */ 4407 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4408 if (item->fs.location > newfs->fs.location) { 4409 list_add_tail(&newfs->list, &item->list); 4410 added = true; 4411 break; 4412 } else if (item->fs.location == fs->location) { 4413 netdev_err(netdev, "Rule not added: location %d not free!\n", 4414 fs->location); 4415 ret = -EBUSY; 4416 goto err; 4417 } 4418 } 4419 if (!added) 4420 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 4421 4422 gem_prog_cmp_regs(bp, fs); 4423 bp->rx_fs_list.count++; 4424 /* enable filtering if NTUPLE on */ 4425 gem_enable_flow_filters(bp, 1); 4426 4427 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4428 return 0; 4429 4430 err: 4431 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4432 kfree(newfs); 4433 return ret; 4434 } 4435 4436 static int gem_del_flow_filter(struct net_device *netdev, 4437 struct ethtool_rxnfc *cmd) 4438 { 4439 struct macb *bp = netdev_priv(netdev); 4440 struct ethtool_rx_fs_item *item; 4441 struct ethtool_rx_flow_spec *fs; 4442 unsigned long flags; 4443 4444 spin_lock_irqsave(&bp->rx_fs_lock, flags); 4445 4446 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4447 if (item->fs.location == cmd->fs.location) { 4448 /* disable screener regs for the flow entry */ 4449 fs = &(item->fs); 4450 netdev_dbg(netdev, 4451 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 4452 fs->flow_type, (int)fs->ring_cookie, fs->location, 4453 htonl(fs->h_u.tcp_ip4_spec.ip4src), 4454 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 4455 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), 4456 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); 4457 4458 gem_writel_n(bp, SCRT2, fs->location, 0); 4459 4460 list_del(&item->list); 4461 bp->rx_fs_list.count--; 4462 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4463 kfree(item); 4464 return 0; 4465 } 4466 } 4467 4468 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 4469 return -EINVAL; 4470 } 4471 4472 static int gem_get_flow_entry(struct net_device *netdev, 4473 struct ethtool_rxnfc *cmd) 4474 { 4475 struct macb *bp = netdev_priv(netdev); 4476 struct ethtool_rx_fs_item *item; 4477 4478 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4479 if (item->fs.location == cmd->fs.location) { 4480 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 4481 return 0; 4482 } 4483 } 4484 return -EINVAL; 4485 } 4486 4487 static int gem_get_all_flow_entries(struct net_device *netdev, 4488 struct ethtool_rxnfc *cmd, u32 *rule_locs) 4489 { 4490 struct macb *bp = netdev_priv(netdev); 4491 struct ethtool_rx_fs_item *item; 4492 uint32_t cnt = 0; 4493 4494 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 4495 if (cnt == cmd->rule_cnt) 4496 return -EMSGSIZE; 4497 rule_locs[cnt] = item->fs.location; 4498 cnt++; 4499 } 4500 cmd->data = bp->max_tuples; 4501 cmd->rule_cnt = cnt; 4502 4503 return 0; 4504 } 4505 4506 static u32 gem_get_rx_ring_count(struct net_device *netdev) 4507 { 4508 struct macb *bp = netdev_priv(netdev); 4509 4510 return bp->num_queues; 4511 } 4512 4513 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 4514 u32 *rule_locs) 4515 { 4516 struct macb *bp = netdev_priv(netdev); 4517 int ret = 0; 4518 4519 switch (cmd->cmd) { 4520 case ETHTOOL_GRXCLSRLCNT: 4521 cmd->rule_cnt = bp->rx_fs_list.count; 4522 break; 4523 case ETHTOOL_GRXCLSRULE: 4524 ret = gem_get_flow_entry(netdev, cmd); 4525 break; 4526 case ETHTOOL_GRXCLSRLALL: 4527 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 4528 break; 4529 default: 4530 netdev_err(netdev, 4531 "Command parameter %d is not supported\n", cmd->cmd); 4532 ret = -EOPNOTSUPP; 4533 } 4534 4535 return ret; 4536 } 4537 4538 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 4539 { 4540 struct macb *bp = netdev_priv(netdev); 4541 int ret; 4542 4543 if (!(netdev->hw_features & NETIF_F_NTUPLE)) 4544 return -EOPNOTSUPP; 4545 4546 switch (cmd->cmd) { 4547 case ETHTOOL_SRXCLSRLINS: 4548 if ((cmd->fs.location >= bp->max_tuples) 4549 || (cmd->fs.ring_cookie >= bp->num_queues)) { 4550 ret = -EINVAL; 4551 break; 4552 } 4553 ret = gem_add_flow_filter(netdev, cmd); 4554 break; 4555 case ETHTOOL_SRXCLSRLDEL: 4556 ret = gem_del_flow_filter(netdev, cmd); 4557 break; 4558 default: 4559 netdev_err(netdev, 4560 "Command parameter %d is not supported\n", cmd->cmd); 4561 ret = -EOPNOTSUPP; 4562 } 4563 4564 return ret; 4565 } 4566 4567 static const struct ethtool_ops macb_ethtool_ops = { 4568 .get_regs_len = macb_get_regs_len, 4569 .get_regs = macb_get_regs, 4570 .get_link = ethtool_op_get_link, 4571 .get_ts_info = ethtool_op_get_ts_info, 4572 .get_pause_stats = macb_get_pause_stats, 4573 .get_eth_mac_stats = macb_get_eth_mac_stats, 4574 .get_eth_phy_stats = macb_get_eth_phy_stats, 4575 .get_rmon_stats = macb_get_rmon_stats, 4576 .get_wol = macb_get_wol, 4577 .set_wol = macb_set_wol, 4578 .get_link_ksettings = macb_get_link_ksettings, 4579 .set_link_ksettings = macb_set_link_ksettings, 4580 .get_ringparam = macb_get_ringparam, 4581 .set_ringparam = macb_set_ringparam, 4582 }; 4583 4584 static int macb_get_eee(struct net_device *netdev, struct ethtool_keee *eee) 4585 { 4586 struct macb *bp = netdev_priv(netdev); 4587 4588 return phylink_ethtool_get_eee(bp->phylink, eee); 4589 } 4590 4591 static int macb_set_eee(struct net_device *netdev, struct ethtool_keee *eee) 4592 { 4593 struct macb *bp = netdev_priv(netdev); 4594 4595 return phylink_ethtool_set_eee(bp->phylink, eee); 4596 } 4597 4598 static const struct ethtool_ops gem_ethtool_ops = { 4599 .get_regs_len = macb_get_regs_len, 4600 .get_regs = macb_get_regs, 4601 .get_wol = macb_get_wol, 4602 .set_wol = macb_set_wol, 4603 .get_link = ethtool_op_get_link, 4604 .get_ts_info = macb_get_ts_info, 4605 .get_ethtool_stats = gem_get_ethtool_stats, 4606 .get_strings = gem_get_ethtool_strings, 4607 .get_sset_count = gem_get_sset_count, 4608 .get_pause_stats = gem_get_pause_stats, 4609 .get_eth_mac_stats = gem_get_eth_mac_stats, 4610 .get_eth_phy_stats = gem_get_eth_phy_stats, 4611 .get_rmon_stats = gem_get_rmon_stats, 4612 .get_link_ksettings = macb_get_link_ksettings, 4613 .set_link_ksettings = macb_set_link_ksettings, 4614 .get_ringparam = macb_get_ringparam, 4615 .set_ringparam = macb_set_ringparam, 4616 .get_rxnfc = gem_get_rxnfc, 4617 .set_rxnfc = gem_set_rxnfc, 4618 .get_rx_ring_count = gem_get_rx_ring_count, 4619 .nway_reset = phy_ethtool_nway_reset, 4620 .get_eee = macb_get_eee, 4621 .set_eee = macb_set_eee, 4622 }; 4623 4624 static int macb_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd) 4625 { 4626 struct macb *bp = netdev_priv(netdev); 4627 4628 if (!netif_running(netdev)) 4629 return -EINVAL; 4630 4631 return phylink_mii_ioctl(bp->phylink, rq, cmd); 4632 } 4633 4634 static int macb_hwtstamp_get(struct net_device *netdev, 4635 struct kernel_hwtstamp_config *cfg) 4636 { 4637 struct macb *bp = netdev_priv(netdev); 4638 4639 if (!netif_running(netdev)) 4640 return -EINVAL; 4641 4642 if (!bp->ptp_info) 4643 return -EOPNOTSUPP; 4644 4645 return bp->ptp_info->get_hwtst(netdev, cfg); 4646 } 4647 4648 static int macb_hwtstamp_set(struct net_device *netdev, 4649 struct kernel_hwtstamp_config *cfg, 4650 struct netlink_ext_ack *extack) 4651 { 4652 struct macb *bp = netdev_priv(netdev); 4653 4654 if (!netif_running(netdev)) 4655 return -EINVAL; 4656 4657 if (!bp->ptp_info) 4658 return -EOPNOTSUPP; 4659 4660 return bp->ptp_info->set_hwtst(netdev, cfg, extack); 4661 } 4662 4663 static inline void macb_set_txcsum_feature(struct macb *bp, 4664 netdev_features_t features) 4665 { 4666 u32 val; 4667 4668 if (!macb_is_gem(&bp->info)) 4669 return; 4670 4671 val = gem_readl(bp, DMACFG); 4672 if (features & NETIF_F_HW_CSUM) 4673 val |= GEM_BIT(TXCOEN); 4674 else 4675 val &= ~GEM_BIT(TXCOEN); 4676 4677 gem_writel(bp, DMACFG, val); 4678 } 4679 4680 static inline void macb_set_rxcsum_feature(struct macb *bp, 4681 netdev_features_t features) 4682 { 4683 struct net_device *netdev = bp->netdev; 4684 u32 val; 4685 4686 if (!macb_is_gem(&bp->info)) 4687 return; 4688 4689 val = gem_readl(bp, NCFGR); 4690 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 4691 val |= GEM_BIT(RXCOEN); 4692 else 4693 val &= ~GEM_BIT(RXCOEN); 4694 4695 gem_writel(bp, NCFGR, val); 4696 } 4697 4698 static inline void macb_set_rxflow_feature(struct macb *bp, 4699 netdev_features_t features) 4700 { 4701 if (!macb_is_gem(&bp->info)) 4702 return; 4703 4704 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 4705 } 4706 4707 static int macb_set_features(struct net_device *netdev, 4708 netdev_features_t features) 4709 { 4710 struct macb *bp = netdev_priv(netdev); 4711 netdev_features_t changed = features ^ netdev->features; 4712 4713 /* TX checksum offload */ 4714 if (changed & NETIF_F_HW_CSUM) 4715 macb_set_txcsum_feature(bp, features); 4716 4717 /* RX checksum offload */ 4718 if (changed & NETIF_F_RXCSUM) 4719 macb_set_rxcsum_feature(bp, features); 4720 4721 /* RX Flow Filters */ 4722 if (changed & NETIF_F_NTUPLE) 4723 macb_set_rxflow_feature(bp, features); 4724 4725 return 0; 4726 } 4727 4728 static void macb_restore_features(struct macb *bp) 4729 { 4730 struct net_device *netdev = bp->netdev; 4731 netdev_features_t features = netdev->features; 4732 struct ethtool_rx_fs_item *item; 4733 4734 /* TX checksum offload */ 4735 macb_set_txcsum_feature(bp, features); 4736 4737 /* RX checksum offload */ 4738 macb_set_rxcsum_feature(bp, features); 4739 4740 /* RX Flow Filters */ 4741 list_for_each_entry(item, &bp->rx_fs_list.list, list) 4742 gem_prog_cmp_regs(bp, &item->fs); 4743 4744 macb_set_rxflow_feature(bp, features); 4745 } 4746 4747 static int macb_taprio_setup_replace(struct net_device *netdev, 4748 struct tc_taprio_qopt_offload *conf) 4749 { 4750 u64 total_on_time = 0, start_time_sec = 0, start_time = conf->base_time; 4751 u32 configured_queues = 0, speed = 0, start_time_nsec; 4752 struct macb_queue_enst_config *enst_queue; 4753 struct ethtool_link_ksettings kset = {}; 4754 struct macb *bp = netdev_priv(netdev); 4755 struct tc_taprio_sched_entry *entry; 4756 struct macb_queue *queue; 4757 u32 queue_mask; 4758 u8 queue_id; 4759 size_t i, q; 4760 int err; 4761 4762 if (conf->num_entries > bp->num_queues) { 4763 netdev_err(netdev, "Too many TAPRIO entries: %zu > %d queues\n", 4764 conf->num_entries, bp->num_queues); 4765 return -EINVAL; 4766 } 4767 4768 if (conf->base_time < 0) { 4769 netdev_err(netdev, "Invalid base_time: must be 0 or positive, got %lld\n", 4770 conf->base_time); 4771 return -ERANGE; 4772 } 4773 4774 /* Get the current link speed */ 4775 err = phylink_ethtool_ksettings_get(bp->phylink, &kset); 4776 if (unlikely(err)) { 4777 netdev_err(netdev, "Failed to get link settings: %d\n", err); 4778 return err; 4779 } 4780 4781 speed = kset.base.speed; 4782 if (unlikely(speed == SPEED_UNKNOWN || !speed)) { 4783 netdev_err(netdev, "Invalid speed %d, link-down?\n", speed); 4784 return -EINVAL; 4785 } 4786 4787 enst_queue = kzalloc_objs(*enst_queue, bp->num_queues); 4788 if (unlikely(!enst_queue)) 4789 return -ENOMEM; 4790 4791 /* Pre-validate all entries before making any hardware changes */ 4792 for (i = 0; i < conf->num_entries; i++) { 4793 entry = &conf->entries[i]; 4794 4795 if (entry->command != TC_TAPRIO_CMD_SET_GATES) { 4796 netdev_err(netdev, "Entry %zu: unsupported command %d\n", 4797 i, entry->command); 4798 err = -EOPNOTSUPP; 4799 goto cleanup; 4800 } 4801 4802 /* Validate gate_mask: must be nonzero, single queue, and within range */ 4803 if (!is_power_of_2(entry->gate_mask)) { 4804 netdev_err(netdev, "Entry %zu: gate_mask 0x%x is not a power of 2 (only one queue per entry allowed)\n", 4805 i, entry->gate_mask); 4806 err = -EINVAL; 4807 goto cleanup; 4808 } 4809 4810 /* gate_mask must not select queues outside the valid queues */ 4811 queue_id = order_base_2(entry->gate_mask); 4812 if (queue_id >= bp->num_queues) { 4813 netdev_err(netdev, "Entry %zu: gate_mask 0x%x exceeds queue range (max_queues=%d)\n", 4814 i, entry->gate_mask, bp->num_queues); 4815 err = -EINVAL; 4816 goto cleanup; 4817 } 4818 4819 /* Check for start time limits */ 4820 start_time_sec = start_time; 4821 start_time_nsec = do_div(start_time_sec, NSEC_PER_SEC); 4822 if (start_time_sec > GENMASK(GEM_START_TIME_SEC_SIZE - 1, 0)) { 4823 netdev_err(netdev, "Entry %zu: Start time %llu s exceeds hardware limit\n", 4824 i, start_time_sec); 4825 err = -ERANGE; 4826 goto cleanup; 4827 } 4828 4829 /* Check for on time limit */ 4830 if (entry->interval > enst_max_hw_interval(speed)) { 4831 netdev_err(netdev, "Entry %zu: interval %u ns exceeds hardware limit %llu ns\n", 4832 i, entry->interval, enst_max_hw_interval(speed)); 4833 err = -ERANGE; 4834 goto cleanup; 4835 } 4836 4837 /* Check for off time limit*/ 4838 if ((conf->cycle_time - entry->interval) > enst_max_hw_interval(speed)) { 4839 netdev_err(netdev, "Entry %zu: off_time %llu ns exceeds hardware limit %llu ns\n", 4840 i, conf->cycle_time - entry->interval, 4841 enst_max_hw_interval(speed)); 4842 err = -ERANGE; 4843 goto cleanup; 4844 } 4845 4846 enst_queue[queue_id].start_time_mask = 4847 (start_time_sec << GEM_START_TIME_SEC_OFFSET) | 4848 start_time_nsec; 4849 enst_queue[queue_id].on_time_bytes = 4850 enst_ns_to_hw_units(entry->interval, speed); 4851 enst_queue[queue_id].off_time_bytes = 4852 enst_ns_to_hw_units(conf->cycle_time - entry->interval, speed); 4853 4854 configured_queues |= entry->gate_mask; 4855 total_on_time += entry->interval; 4856 start_time += entry->interval; 4857 } 4858 4859 /* Check total interval doesn't exceed cycle time */ 4860 if (total_on_time > conf->cycle_time) { 4861 netdev_err(netdev, "Total ON %llu ns exceeds cycle time %llu ns\n", 4862 total_on_time, conf->cycle_time); 4863 err = -EINVAL; 4864 goto cleanup; 4865 } 4866 4867 netdev_dbg(netdev, "TAPRIO setup: %zu entries, base_time=%lld ns, cycle_time=%llu ns\n", 4868 conf->num_entries, conf->base_time, conf->cycle_time); 4869 4870 /* All validations passed - proceed with hardware configuration */ 4871 scoped_guard(spinlock_irqsave, &bp->lock) { 4872 /* Disable ENST queues if running before configuring */ 4873 queue_mask = BIT_U32(bp->num_queues) - 1; 4874 gem_writel(bp, ENST_CONTROL, 4875 queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); 4876 4877 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 4878 /* Configure queue timing registers */ 4879 queue_writel(queue, ENST_START_TIME, 4880 enst_queue[q].start_time_mask); 4881 queue_writel(queue, ENST_ON_TIME, 4882 enst_queue[q].on_time_bytes); 4883 queue_writel(queue, ENST_OFF_TIME, 4884 enst_queue[q].off_time_bytes); 4885 } 4886 4887 /* Enable ENST for all configured queues in one write */ 4888 gem_writel(bp, ENST_CONTROL, configured_queues); 4889 } 4890 4891 netdev_info(netdev, "TAPRIO configuration completed successfully: %zu entries, %d queues configured\n", 4892 conf->num_entries, hweight32(configured_queues)); 4893 4894 cleanup: 4895 kfree(enst_queue); 4896 return err; 4897 } 4898 4899 static void macb_taprio_destroy(struct net_device *netdev) 4900 { 4901 struct macb *bp = netdev_priv(netdev); 4902 struct macb_queue *queue; 4903 u32 queue_mask; 4904 unsigned int q; 4905 4906 netdev_reset_tc(netdev); 4907 queue_mask = BIT_U32(bp->num_queues) - 1; 4908 4909 scoped_guard(spinlock_irqsave, &bp->lock) { 4910 /* Single disable command for all queues */ 4911 gem_writel(bp, ENST_CONTROL, 4912 queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); 4913 4914 /* Clear all queue ENST registers in batch */ 4915 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 4916 queue_writel(queue, ENST_START_TIME, 0); 4917 queue_writel(queue, ENST_ON_TIME, 0); 4918 queue_writel(queue, ENST_OFF_TIME, 0); 4919 } 4920 } 4921 netdev_info(netdev, "TAPRIO destroy: All gates disabled\n"); 4922 } 4923 4924 static int macb_setup_taprio(struct net_device *netdev, 4925 struct tc_taprio_qopt_offload *taprio) 4926 { 4927 struct macb *bp = netdev_priv(netdev); 4928 int err = 0; 4929 4930 if (unlikely(!(netdev->hw_features & NETIF_F_HW_TC))) 4931 return -EOPNOTSUPP; 4932 4933 /* Check if Device is in runtime suspend */ 4934 if (unlikely(pm_runtime_suspended(&bp->pdev->dev))) { 4935 netdev_err(netdev, "Device is in runtime suspend\n"); 4936 return -EOPNOTSUPP; 4937 } 4938 4939 switch (taprio->cmd) { 4940 case TAPRIO_CMD_REPLACE: 4941 err = macb_taprio_setup_replace(netdev, taprio); 4942 break; 4943 case TAPRIO_CMD_DESTROY: 4944 macb_taprio_destroy(netdev); 4945 break; 4946 default: 4947 err = -EOPNOTSUPP; 4948 } 4949 4950 return err; 4951 } 4952 4953 static int macb_setup_tc(struct net_device *netdev, enum tc_setup_type type, 4954 void *type_data) 4955 { 4956 if (!netdev || !type_data) 4957 return -EINVAL; 4958 4959 switch (type) { 4960 case TC_SETUP_QDISC_TAPRIO: 4961 return macb_setup_taprio(netdev, type_data); 4962 default: 4963 return -EOPNOTSUPP; 4964 } 4965 } 4966 4967 static void macb_tx_timeout(struct net_device *netdev, unsigned int q) 4968 { 4969 struct macb *bp = netdev_priv(netdev); 4970 4971 macb_tx_restart(&bp->queues[q]); 4972 } 4973 4974 static const struct net_device_ops macb_netdev_ops = { 4975 .ndo_open = macb_open, 4976 .ndo_stop = macb_close, 4977 .ndo_start_xmit = macb_start_xmit, 4978 .ndo_set_rx_mode = macb_set_rx_mode, 4979 .ndo_get_stats64 = macb_get_stats, 4980 .ndo_eth_ioctl = macb_ioctl, 4981 .ndo_validate_addr = eth_validate_addr, 4982 .ndo_change_mtu = macb_change_mtu, 4983 .ndo_set_mac_address = macb_set_mac_addr, 4984 #ifdef CONFIG_NET_POLL_CONTROLLER 4985 .ndo_poll_controller = macb_poll_controller, 4986 #endif 4987 .ndo_set_features = macb_set_features, 4988 .ndo_features_check = macb_features_check, 4989 .ndo_hwtstamp_set = macb_hwtstamp_set, 4990 .ndo_hwtstamp_get = macb_hwtstamp_get, 4991 .ndo_setup_tc = macb_setup_tc, 4992 .ndo_tx_timeout = macb_tx_timeout, 4993 }; 4994 4995 /* Configure peripheral capabilities according to device tree 4996 * and integration options used 4997 */ 4998 static void macb_configure_caps(struct macb *bp, 4999 const struct macb_config *dt_conf) 5000 { 5001 u32 dcfg; 5002 5003 bp->caps = dt_conf->caps; 5004 5005 if (!dt_conf->usrio) 5006 bp->caps |= MACB_CAPS_USRIO_DISABLED; 5007 5008 if (hw_is_gem(bp->regs, bp->native_io)) { 5009 bp->caps |= MACB_CAPS_MACB_IS_GEM; 5010 5011 dcfg = gem_readl(bp, DCFG1); 5012 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 5013 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 5014 if (GEM_BFEXT(NO_PCS, dcfg) == 0) 5015 bp->caps |= MACB_CAPS_PCS; 5016 if (!(dcfg & GEM_BIT(USERIO))) 5017 bp->caps |= MACB_CAPS_USRIO_DISABLED; 5018 dcfg = gem_readl(bp, DCFG12); 5019 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) 5020 bp->caps |= MACB_CAPS_HIGH_SPEED; 5021 dcfg = gem_readl(bp, DCFG2); 5022 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 5023 bp->caps |= MACB_CAPS_FIFO_MODE; 5024 if (GEM_BFEXT(PBUF_RSC, gem_readl(bp, DCFG6))) 5025 bp->caps |= MACB_CAPS_RSC; 5026 if (gem_has_ptp(&bp->info)) { 5027 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 5028 dev_err(&bp->pdev->dev, 5029 "GEM doesn't support hardware ptp.\n"); 5030 else { 5031 #ifdef CONFIG_MACB_USE_HWSTAMP 5032 bp->caps |= MACB_CAPS_DMA_PTP; 5033 bp->ptp_info = &gem_ptp_info; 5034 #endif 5035 } 5036 } 5037 } 5038 5039 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 5040 } 5041 5042 static int macb_probe_queues(struct device *dev, void __iomem *mem, bool native_io) 5043 { 5044 /* BIT(0) is never set but queue 0 always exists. */ 5045 unsigned int queue_mask = 0x1; 5046 5047 /* Use hw_is_gem() as MACB_CAPS_MACB_IS_GEM is not yet positioned. */ 5048 if (hw_is_gem(mem, native_io)) { 5049 if (native_io) 5050 queue_mask |= __raw_readl(mem + GEM_DCFG6) & 0xFF; 5051 else 5052 queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xFF; 5053 5054 if (fls(queue_mask) != ffz(queue_mask)) { 5055 dev_err(dev, "queue mask %#x has a hole\n", queue_mask); 5056 return -EINVAL; 5057 } 5058 } 5059 5060 return hweight32(queue_mask); 5061 } 5062 5063 static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, 5064 struct clk *rx_clk, struct clk *tsu_clk) 5065 { 5066 struct clk_bulk_data clks[] = { 5067 { .clk = tsu_clk, }, 5068 { .clk = rx_clk, }, 5069 { .clk = pclk, }, 5070 { .clk = hclk, }, 5071 { .clk = tx_clk }, 5072 }; 5073 5074 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); 5075 } 5076 5077 static int macb_clk_init_dflt(struct platform_device *pdev, struct clk **pclk, 5078 struct clk **hclk, struct clk **tx_clk, 5079 struct clk **rx_clk, struct clk **tsu_clk) 5080 { 5081 struct macb_platform_data *pdata; 5082 int err; 5083 5084 pdata = dev_get_platdata(&pdev->dev); 5085 if (pdata) { 5086 *pclk = pdata->pclk; 5087 *hclk = pdata->hclk; 5088 } else { 5089 *pclk = devm_clk_get(&pdev->dev, "pclk"); 5090 *hclk = devm_clk_get(&pdev->dev, "hclk"); 5091 } 5092 5093 if (IS_ERR_OR_NULL(*pclk)) 5094 return dev_err_probe(&pdev->dev, 5095 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, 5096 "failed to get pclk\n"); 5097 5098 if (IS_ERR_OR_NULL(*hclk)) 5099 return dev_err_probe(&pdev->dev, 5100 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, 5101 "failed to get hclk\n"); 5102 5103 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 5104 if (IS_ERR(*tx_clk)) 5105 return PTR_ERR(*tx_clk); 5106 5107 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 5108 if (IS_ERR(*rx_clk)) 5109 return PTR_ERR(*rx_clk); 5110 5111 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 5112 if (IS_ERR(*tsu_clk)) 5113 return PTR_ERR(*tsu_clk); 5114 5115 err = clk_prepare_enable(*pclk); 5116 if (err) { 5117 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 5118 return err; 5119 } 5120 5121 err = clk_prepare_enable(*hclk); 5122 if (err) { 5123 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 5124 goto err_disable_pclk; 5125 } 5126 5127 err = clk_prepare_enable(*tx_clk); 5128 if (err) { 5129 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 5130 goto err_disable_hclk; 5131 } 5132 5133 err = clk_prepare_enable(*rx_clk); 5134 if (err) { 5135 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 5136 goto err_disable_txclk; 5137 } 5138 5139 err = clk_prepare_enable(*tsu_clk); 5140 if (err) { 5141 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 5142 goto err_disable_rxclk; 5143 } 5144 5145 return 0; 5146 5147 err_disable_rxclk: 5148 clk_disable_unprepare(*rx_clk); 5149 5150 err_disable_txclk: 5151 clk_disable_unprepare(*tx_clk); 5152 5153 err_disable_hclk: 5154 clk_disable_unprepare(*hclk); 5155 5156 err_disable_pclk: 5157 clk_disable_unprepare(*pclk); 5158 5159 return err; 5160 } 5161 5162 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 5163 struct clk **hclk, struct clk **tx_clk, 5164 struct clk **rx_clk, struct clk **tsu_clk, 5165 const struct macb_config *config) 5166 { 5167 if (config->clk_init) 5168 return config->clk_init(pdev, pclk, hclk, tx_clk, rx_clk, 5169 tsu_clk); 5170 else 5171 return macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, 5172 tsu_clk); 5173 } 5174 5175 static int macb_init_dflt(struct platform_device *pdev) 5176 { 5177 struct net_device *netdev = platform_get_drvdata(pdev); 5178 unsigned int hw_q, q; 5179 struct macb *bp = netdev_priv(netdev); 5180 struct macb_queue *queue; 5181 int err; 5182 u32 val, reg; 5183 5184 /* No locking needed because the IRQs are not requested yet. The 5185 * flag is cleared by macb_open() and re-armed by macb_close(). 5186 */ 5187 bp->irq_quiesced = true; 5188 5189 /* set the queue register mapping once for all: queue0 has a special 5190 * register mapping but we don't want to test the queue index then 5191 * compute the corresponding register offset at run time. 5192 */ 5193 for (hw_q = 0, q = 0; hw_q < bp->num_queues; ++hw_q) { 5194 queue = &bp->queues[q]; 5195 queue->bp = bp; 5196 spin_lock_init(&queue->tx_ptr_lock); 5197 netif_napi_add(netdev, &queue->napi_rx, macb_rx_poll); 5198 netif_napi_add_tx(netdev, &queue->napi_tx, macb_tx_poll); 5199 if (hw_q) { 5200 queue->ISR = GEM_ISR(hw_q - 1); 5201 queue->IER = GEM_IER(hw_q - 1); 5202 queue->IDR = GEM_IDR(hw_q - 1); 5203 queue->IMR = GEM_IMR(hw_q - 1); 5204 queue->TBQP = GEM_TBQP(hw_q - 1); 5205 queue->RBQP = GEM_RBQP(hw_q - 1); 5206 queue->RBQS = GEM_RBQS(hw_q - 1); 5207 } else { 5208 /* queue0 uses legacy registers */ 5209 queue->ISR = MACB_ISR; 5210 queue->IER = MACB_IER; 5211 queue->IDR = MACB_IDR; 5212 queue->IMR = MACB_IMR; 5213 queue->TBQP = MACB_TBQP; 5214 queue->RBQP = MACB_RBQP; 5215 } 5216 5217 queue->ENST_START_TIME = GEM_ENST_START_TIME(hw_q); 5218 queue->ENST_ON_TIME = GEM_ENST_ON_TIME(hw_q); 5219 queue->ENST_OFF_TIME = GEM_ENST_OFF_TIME(hw_q); 5220 5221 /* get irq: here we use the linux queue index, not the hardware 5222 * queue index. the queue irq definitions in the device tree 5223 * must remove the optional gaps that could exist in the 5224 * hardware queue mask. 5225 */ 5226 queue->irq = platform_get_irq(pdev, q); 5227 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 5228 IRQF_SHARED, netdev->name, queue); 5229 if (err) { 5230 dev_err(&pdev->dev, 5231 "Unable to request IRQ %d (error %d)\n", 5232 queue->irq, err); 5233 return err; 5234 } 5235 5236 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 5237 q++; 5238 } 5239 5240 netdev->netdev_ops = &macb_netdev_ops; 5241 5242 /* setup appropriated routines according to adapter type */ 5243 if (macb_is_gem(&bp->info)) { 5244 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 5245 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 5246 bp->macbgem_ops.mog_init_rings = gem_init_rings; 5247 bp->macbgem_ops.mog_rx = gem_rx; 5248 netdev->ethtool_ops = &gem_ethtool_ops; 5249 } else { 5250 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 5251 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 5252 bp->macbgem_ops.mog_init_rings = macb_init_rings; 5253 bp->macbgem_ops.mog_rx = macb_rx; 5254 netdev->ethtool_ops = &macb_ethtool_ops; 5255 } 5256 5257 netdev_sw_irq_coalesce_default_on(netdev); 5258 5259 netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 5260 5261 /* Set features */ 5262 netdev->hw_features = NETIF_F_SG; 5263 5264 /* Check LSO capability; runtime detection can be overridden by a cap 5265 * flag if the hardware is known to be buggy 5266 */ 5267 if (!(bp->caps & MACB_CAPS_NO_LSO) && 5268 GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 5269 netdev->hw_features |= MACB_NETIF_LSO; 5270 5271 /* Checksum offload is only available on gem with packet buffer */ 5272 if (macb_is_gem(&bp->info) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 5273 netdev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 5274 if (bp->caps & MACB_CAPS_SG_DISABLED) 5275 netdev->hw_features &= ~NETIF_F_SG; 5276 /* Enable HW_TC if hardware supports QBV */ 5277 if (bp->caps & MACB_CAPS_QBV) 5278 netdev->hw_features |= NETIF_F_HW_TC; 5279 5280 netdev->features = netdev->hw_features; 5281 5282 /* Check RX Flow Filters support. 5283 * Max Rx flows set by availability of screeners & compare regs: 5284 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 5285 */ 5286 reg = gem_readl(bp, DCFG8); 5287 bp->max_tuples = umin((GEM_BFEXT(SCR2CMP, reg) / 3), 5288 GEM_BFEXT(T2SCR, reg)); 5289 INIT_LIST_HEAD(&bp->rx_fs_list.list); 5290 if (bp->max_tuples > 0) { 5291 /* also needs one ethtype match to check IPv4 */ 5292 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 5293 /* program this reg now */ 5294 reg = 0; 5295 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 5296 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 5297 /* Filtering is supported in hw but don't enable it in kernel now */ 5298 netdev->hw_features |= NETIF_F_NTUPLE; 5299 /* init Rx flow definitions */ 5300 bp->rx_fs_list.count = 0; 5301 spin_lock_init(&bp->rx_fs_lock); 5302 } else 5303 bp->max_tuples = 0; 5304 } 5305 5306 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 5307 val = 0; 5308 if (bp->caps & MACB_CAPS_USRIO_HAS_MII) { 5309 if (phy_interface_mode_is_rgmii(bp->phy_interface)) 5310 val = bp->usrio->rgmii; 5311 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 5312 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 5313 val = bp->usrio->rmii; 5314 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 5315 val = bp->usrio->mii; 5316 } 5317 5318 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 5319 val |= bp->usrio->clken; 5320 5321 if (bp->caps & MACB_CAPS_USRIO_HAS_REFCLK_SOURCE) { 5322 const char *prop; 5323 bool refclk_ext; 5324 int ret; 5325 5326 /* Default to whatever was set in the match data for 5327 * this device. There's two properties for refclk 5328 * control, but the boolean one is deprecated so is 5329 * a lower priority to check, no device should have 5330 * both. 5331 */ 5332 refclk_ext = bp->usrio->refclk_default_external; 5333 5334 ret = of_property_read_string(pdev->dev.of_node, 5335 "cdns,refclk-source", &prop); 5336 if (!ret) { 5337 if (!strcmp(prop, "external")) 5338 refclk_ext = true; 5339 else 5340 refclk_ext = false; 5341 } else { 5342 ret = of_property_read_bool(pdev->dev.of_node, 5343 "cdns,refclk-ext"); 5344 if (ret) 5345 refclk_ext = true; 5346 } 5347 5348 if (refclk_ext) 5349 val |= bp->usrio->refclk; 5350 } 5351 5352 if (bp->caps & MACB_CAPS_USRIO_HAS_TSUCLK_SOURCE) 5353 val |= bp->usrio->tsu_source; 5354 5355 macb_or_gem_writel(bp, USRIO, val); 5356 } 5357 5358 /* Set MII management clock divider */ 5359 val = macb_mdc_clk_div(bp); 5360 val |= macb_dbw(bp); 5361 if (bp->phy_interface == PHY_INTERFACE_MODE_1000BASEX) 5362 val |= GEM_BIT(PCSSEL); 5363 else if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 5364 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 5365 macb_writel(bp, NCFGR, val); 5366 5367 return 0; 5368 } 5369 5370 static int macb_init(struct platform_device *pdev, 5371 const struct macb_config *config) 5372 { 5373 if (config->init) 5374 return config->init(pdev); 5375 else 5376 return macb_init_dflt(pdev); 5377 } 5378 5379 static const struct macb_usrio_config at91_default_usrio = { 5380 .mii = MACB_BIT(MII), 5381 .rmii = MACB_BIT(RMII), 5382 .rgmii = GEM_BIT(RGMII), 5383 .clken = MACB_BIT(CLKEN), 5384 }; 5385 5386 /* 1518 rounded up */ 5387 #define AT91ETHER_MAX_RBUFF_SZ 0x600 5388 /* max number of receive buffers */ 5389 #define AT91ETHER_MAX_RX_DESCR 9 5390 5391 static struct sifive_fu540_macb_mgmt *mgmt; 5392 5393 static int at91ether_alloc_coherent(struct macb *bp) 5394 { 5395 struct macb_rxq *rxq = &bp->ctx->rxq[0]; 5396 5397 rxq->ring = dma_alloc_coherent(&bp->pdev->dev, 5398 (AT91ETHER_MAX_RX_DESCR * 5399 macb_dma_desc_get_size(&bp->info)), 5400 &rxq->ring_dma, GFP_KERNEL); 5401 if (!rxq->ring) 5402 return -ENOMEM; 5403 5404 rxq->buffers = dma_alloc_coherent(&bp->pdev->dev, 5405 AT91ETHER_MAX_RX_DESCR * 5406 AT91ETHER_MAX_RBUFF_SZ, 5407 &rxq->buffers_dma, 5408 GFP_KERNEL); 5409 if (!rxq->buffers) { 5410 dma_free_coherent(&bp->pdev->dev, 5411 AT91ETHER_MAX_RX_DESCR * 5412 macb_dma_desc_get_size(&bp->info), 5413 rxq->ring, rxq->ring_dma); 5414 rxq->ring = NULL; 5415 return -ENOMEM; 5416 } 5417 5418 return 0; 5419 } 5420 5421 static void at91ether_free_coherent(struct macb *bp) 5422 { 5423 struct macb_rxq *rxq = &bp->ctx->rxq[0]; 5424 5425 if (rxq->ring) { 5426 dma_free_coherent(&bp->pdev->dev, 5427 AT91ETHER_MAX_RX_DESCR * 5428 macb_dma_desc_get_size(&bp->info), 5429 rxq->ring, rxq->ring_dma); 5430 rxq->ring = NULL; 5431 } 5432 5433 if (rxq->buffers) { 5434 dma_free_coherent(&bp->pdev->dev, 5435 AT91ETHER_MAX_RX_DESCR * 5436 AT91ETHER_MAX_RBUFF_SZ, 5437 rxq->buffers, rxq->buffers_dma); 5438 rxq->buffers = NULL; 5439 } 5440 } 5441 5442 /* Initialize and start the Receiver and Transmit subsystems */ 5443 static int at91ether_start(struct macb *bp) 5444 { 5445 struct macb_rxq *rxq = &bp->ctx->rxq[0]; 5446 struct macb_dma_desc *desc; 5447 dma_addr_t addr; 5448 u32 ctl; 5449 int i, ret; 5450 5451 ret = at91ether_alloc_coherent(bp); 5452 if (ret) 5453 return ret; 5454 5455 addr = rxq->buffers_dma; 5456 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 5457 desc = macb_rx_desc(bp->ctx, 0, i); 5458 macb_set_addr(&bp->info, desc, addr); 5459 desc->ctrl = 0; 5460 addr += AT91ETHER_MAX_RBUFF_SZ; 5461 } 5462 5463 /* Set the Wrap bit on the last descriptor */ 5464 desc->addr |= MACB_BIT(RX_WRAP); 5465 5466 /* Reset buffer index */ 5467 rxq->tail = 0; 5468 5469 /* Program address of descriptor list in Rx Buffer Queue register */ 5470 macb_writel(bp, RBQP, rxq->ring_dma); 5471 5472 /* Enable Receive and Transmit */ 5473 ctl = macb_readl(bp, NCR); 5474 macb_writel(bp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 5475 5476 /* Enable MAC interrupts */ 5477 macb_writel(bp, IER, MACB_BIT(RCOMP) | 5478 MACB_BIT(RXUBR) | 5479 MACB_BIT(ISR_TUND) | 5480 MACB_BIT(ISR_RLE) | 5481 MACB_BIT(TCOMP) | 5482 MACB_BIT(ISR_ROVR) | 5483 MACB_BIT(HRESP)); 5484 5485 return 0; 5486 } 5487 5488 static void at91ether_stop(struct macb *bp) 5489 { 5490 u32 ctl; 5491 5492 /* Disable MAC interrupts */ 5493 macb_writel(bp, IDR, MACB_BIT(RCOMP) | 5494 MACB_BIT(RXUBR) | 5495 MACB_BIT(ISR_TUND) | 5496 MACB_BIT(ISR_RLE) | 5497 MACB_BIT(TCOMP) | 5498 MACB_BIT(ISR_ROVR) | 5499 MACB_BIT(HRESP)); 5500 5501 /* Disable Receiver and Transmitter */ 5502 ctl = macb_readl(bp, NCR); 5503 macb_writel(bp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 5504 5505 synchronize_irq(bp->netdev->irq); 5506 5507 /* Free resources. */ 5508 at91ether_free_coherent(bp); 5509 } 5510 5511 /* Open the ethernet interface */ 5512 static int at91ether_open(struct net_device *netdev) 5513 { 5514 struct macb *bp = netdev_priv(netdev); 5515 u32 ctl; 5516 int ret; 5517 5518 ret = pm_runtime_resume_and_get(&bp->pdev->dev); 5519 if (ret < 0) 5520 return ret; 5521 5522 WRITE_ONCE(bp->ctx, kzalloc_obj(*bp->ctx)); 5523 wmb(); /* ensure IRQ enabled below read the new context */ 5524 if (!bp->ctx) { 5525 ret = -ENOMEM; 5526 goto pm_exit; 5527 } 5528 bp->ctx->info = &bp->info; 5529 5530 /* Clear internal statistics */ 5531 ctl = macb_readl(bp, NCR); 5532 macb_writel(bp, NCR, ctl | MACB_BIT(CLRSTAT)); 5533 5534 macb_set_hwaddr(bp); 5535 5536 ret = at91ether_start(bp); 5537 if (ret) 5538 goto free_ctx; 5539 5540 ret = macb_phylink_connect(bp); 5541 if (ret) 5542 goto stop; 5543 5544 netif_start_queue(netdev); 5545 5546 return 0; 5547 5548 stop: 5549 at91ether_stop(bp); 5550 free_ctx: 5551 kfree(bp->ctx); 5552 bp->ctx = NULL; 5553 pm_exit: 5554 pm_runtime_put_sync(&bp->pdev->dev); 5555 return ret; 5556 } 5557 5558 /* Close the interface */ 5559 static int at91ether_close(struct net_device *netdev) 5560 { 5561 struct macb *bp = netdev_priv(netdev); 5562 5563 netif_stop_queue(netdev); 5564 5565 phylink_stop(bp->phylink); 5566 phylink_disconnect_phy(bp->phylink); 5567 5568 at91ether_stop(bp); 5569 5570 kfree(bp->ctx); 5571 bp->ctx = NULL; 5572 5573 pm_runtime_put(&bp->pdev->dev); 5574 5575 return 0; 5576 } 5577 5578 /* Transmit packet */ 5579 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 5580 struct net_device *netdev) 5581 { 5582 struct macb *bp = netdev_priv(netdev); 5583 struct device *dev = &bp->pdev->dev; 5584 5585 if (macb_readl(bp, TSR) & MACB_BIT(RM9200_BNQ)) { 5586 int desc = 0; 5587 5588 netif_stop_queue(netdev); 5589 5590 /* Store packet information (to free when Tx completed) */ 5591 bp->rm9200_txq[desc].skb = skb; 5592 bp->rm9200_txq[desc].size = skb->len; 5593 bp->rm9200_txq[desc].mapping = dma_map_single(dev, skb->data, 5594 skb->len, 5595 DMA_TO_DEVICE); 5596 if (dma_mapping_error(dev, bp->rm9200_txq[desc].mapping)) { 5597 dev_kfree_skb_any(skb); 5598 netdev->stats.tx_dropped++; 5599 netdev_err(netdev, "%s: DMA mapping error\n", __func__); 5600 return NETDEV_TX_OK; 5601 } 5602 5603 /* Set address of the data in the Transmit Address register */ 5604 macb_writel(bp, TAR, bp->rm9200_txq[desc].mapping); 5605 /* Set length of the packet in the Transmit Control register */ 5606 macb_writel(bp, TCR, skb->len); 5607 5608 } else { 5609 netdev_err(netdev, "%s called, but device is busy!\n", 5610 __func__); 5611 return NETDEV_TX_BUSY; 5612 } 5613 5614 return NETDEV_TX_OK; 5615 } 5616 5617 /* Extract received frame from buffer descriptors and sent to upper layers. 5618 * (Called from interrupt context) 5619 */ 5620 static void at91ether_rx(struct net_device *netdev) 5621 { 5622 struct macb *bp = netdev_priv(netdev); 5623 struct macb_rxq *rxq = &bp->ctx->rxq[0]; 5624 struct macb_dma_desc *desc; 5625 unsigned char *p_recv; 5626 struct sk_buff *skb; 5627 unsigned int pktlen; 5628 5629 desc = macb_rx_desc(bp->ctx, 0, rxq->tail); 5630 while (desc->addr & MACB_BIT(RX_USED)) { 5631 p_recv = rxq->buffers + rxq->tail * AT91ETHER_MAX_RBUFF_SZ; 5632 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 5633 skb = netdev_alloc_skb(netdev, pktlen + 2); 5634 if (skb) { 5635 skb_reserve(skb, 2); 5636 skb_put_data(skb, p_recv, pktlen); 5637 5638 skb->protocol = eth_type_trans(skb, netdev); 5639 netdev->stats.rx_packets++; 5640 netdev->stats.rx_bytes += pktlen; 5641 netif_rx(skb); 5642 } else { 5643 netdev->stats.rx_dropped++; 5644 } 5645 5646 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 5647 netdev->stats.multicast++; 5648 5649 /* reset ownership bit */ 5650 desc->addr &= ~MACB_BIT(RX_USED); 5651 5652 /* wrap after last buffer */ 5653 if (rxq->tail == AT91ETHER_MAX_RX_DESCR - 1) 5654 rxq->tail = 0; 5655 else 5656 rxq->tail++; 5657 5658 desc = macb_rx_desc(bp->ctx, 0, rxq->tail); 5659 } 5660 } 5661 5662 /* MAC interrupt handler */ 5663 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 5664 { 5665 struct net_device *netdev = dev_id; 5666 struct macb *bp = netdev_priv(netdev); 5667 u32 intstatus, ctl; 5668 unsigned int desc; 5669 5670 /* MAC Interrupt Status register indicates what interrupts are pending. 5671 * It is automatically cleared once read. 5672 */ 5673 intstatus = macb_readl(bp, ISR); 5674 5675 /* Receive complete */ 5676 if (intstatus & MACB_BIT(RCOMP)) 5677 at91ether_rx(netdev); 5678 5679 /* Transmit complete */ 5680 if (intstatus & MACB_BIT(TCOMP)) { 5681 /* The TCOM bit is set even if the transmission failed */ 5682 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 5683 netdev->stats.tx_errors++; 5684 5685 desc = 0; 5686 if (bp->rm9200_txq[desc].skb) { 5687 dev_consume_skb_irq(bp->rm9200_txq[desc].skb); 5688 bp->rm9200_txq[desc].skb = NULL; 5689 dma_unmap_single(&bp->pdev->dev, 5690 bp->rm9200_txq[desc].mapping, 5691 bp->rm9200_txq[desc].size, 5692 DMA_TO_DEVICE); 5693 netdev->stats.tx_packets++; 5694 netdev->stats.tx_bytes += bp->rm9200_txq[desc].size; 5695 } 5696 netif_wake_queue(netdev); 5697 } 5698 5699 /* Work-around for EMAC Errata section 41.3.1 */ 5700 if (intstatus & MACB_BIT(RXUBR)) { 5701 ctl = macb_readl(bp, NCR); 5702 macb_writel(bp, NCR, ctl & ~MACB_BIT(RE)); 5703 wmb(); 5704 macb_writel(bp, NCR, ctl | MACB_BIT(RE)); 5705 } 5706 5707 if (intstatus & MACB_BIT(ISR_ROVR)) 5708 netdev_err(netdev, "ROVR error\n"); 5709 5710 return IRQ_HANDLED; 5711 } 5712 5713 #ifdef CONFIG_NET_POLL_CONTROLLER 5714 static void at91ether_poll_controller(struct net_device *netdev) 5715 { 5716 unsigned long flags; 5717 5718 local_irq_save(flags); 5719 at91ether_interrupt(netdev->irq, netdev); 5720 local_irq_restore(flags); 5721 } 5722 #endif 5723 5724 static const struct net_device_ops at91ether_netdev_ops = { 5725 .ndo_open = at91ether_open, 5726 .ndo_stop = at91ether_close, 5727 .ndo_start_xmit = at91ether_start_xmit, 5728 .ndo_get_stats64 = macb_get_stats, 5729 .ndo_set_rx_mode = macb_set_rx_mode, 5730 .ndo_set_mac_address = eth_mac_addr, 5731 .ndo_eth_ioctl = macb_ioctl, 5732 .ndo_validate_addr = eth_validate_addr, 5733 #ifdef CONFIG_NET_POLL_CONTROLLER 5734 .ndo_poll_controller = at91ether_poll_controller, 5735 #endif 5736 .ndo_hwtstamp_set = macb_hwtstamp_set, 5737 .ndo_hwtstamp_get = macb_hwtstamp_get, 5738 }; 5739 5740 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 5741 struct clk **hclk, struct clk **tx_clk, 5742 struct clk **rx_clk, struct clk **tsu_clk) 5743 { 5744 int err; 5745 5746 *hclk = NULL; 5747 *tx_clk = NULL; 5748 *rx_clk = NULL; 5749 *tsu_clk = NULL; 5750 5751 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 5752 if (IS_ERR(*pclk)) 5753 return PTR_ERR(*pclk); 5754 5755 err = clk_prepare_enable(*pclk); 5756 if (err) { 5757 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 5758 return err; 5759 } 5760 5761 return 0; 5762 } 5763 5764 static int at91ether_init(struct platform_device *pdev) 5765 { 5766 struct net_device *netdev = platform_get_drvdata(pdev); 5767 struct macb *bp = netdev_priv(netdev); 5768 int err; 5769 5770 bp->queues[0].bp = bp; 5771 5772 netdev->netdev_ops = &at91ether_netdev_ops; 5773 netdev->ethtool_ops = &macb_ethtool_ops; 5774 5775 err = devm_request_irq(&pdev->dev, netdev->irq, at91ether_interrupt, 5776 0, netdev->name, netdev); 5777 if (err) 5778 return err; 5779 5780 macb_writel(bp, NCR, 0); 5781 5782 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); 5783 5784 return 0; 5785 } 5786 5787 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 5788 unsigned long parent_rate) 5789 { 5790 return mgmt->rate; 5791 } 5792 5793 static int fu540_macb_tx_determine_rate(struct clk_hw *hw, 5794 struct clk_rate_request *req) 5795 { 5796 if (WARN_ON(req->rate < 2500000)) 5797 req->rate = 2500000; 5798 else if (req->rate == 2500000) 5799 req->rate = 2500000; 5800 else if (WARN_ON(req->rate < 13750000)) 5801 req->rate = 2500000; 5802 else if (WARN_ON(req->rate < 25000000)) 5803 req->rate = 25000000; 5804 else if (req->rate == 25000000) 5805 req->rate = 25000000; 5806 else if (WARN_ON(req->rate < 75000000)) 5807 req->rate = 25000000; 5808 else if (WARN_ON(req->rate < 125000000)) 5809 req->rate = 125000000; 5810 else if (req->rate == 125000000) 5811 req->rate = 125000000; 5812 else if (WARN_ON(req->rate > 125000000)) 5813 req->rate = 125000000; 5814 else 5815 req->rate = 125000000; 5816 5817 return 0; 5818 } 5819 5820 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 5821 unsigned long parent_rate) 5822 { 5823 struct clk_rate_request req; 5824 int ret; 5825 5826 clk_hw_init_rate_request(hw, &req, rate); 5827 ret = fu540_macb_tx_determine_rate(hw, &req); 5828 if (ret != 0) 5829 return ret; 5830 5831 if (req.rate != 125000000) 5832 iowrite32(1, mgmt->reg); 5833 else 5834 iowrite32(0, mgmt->reg); 5835 mgmt->rate = rate; 5836 5837 return 0; 5838 } 5839 5840 static const struct clk_ops fu540_c000_ops = { 5841 .recalc_rate = fu540_macb_tx_recalc_rate, 5842 .determine_rate = fu540_macb_tx_determine_rate, 5843 .set_rate = fu540_macb_tx_set_rate, 5844 }; 5845 5846 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 5847 struct clk **hclk, struct clk **tx_clk, 5848 struct clk **rx_clk, struct clk **tsu_clk) 5849 { 5850 struct clk_init_data init = {}; 5851 int err = 0; 5852 5853 err = macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 5854 if (err) 5855 return err; 5856 5857 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 5858 if (!mgmt) { 5859 err = -ENOMEM; 5860 goto err_disable_clks; 5861 } 5862 5863 init.name = "sifive-gemgxl-mgmt"; 5864 init.ops = &fu540_c000_ops; 5865 init.flags = 0; 5866 init.num_parents = 0; 5867 5868 mgmt->rate = 0; 5869 mgmt->hw.init = &init; 5870 5871 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); 5872 if (IS_ERR(*tx_clk)) { 5873 err = PTR_ERR(*tx_clk); 5874 goto err_disable_clks; 5875 } 5876 5877 err = clk_prepare_enable(*tx_clk); 5878 if (err) { 5879 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 5880 *tx_clk = NULL; 5881 goto err_disable_clks; 5882 } else { 5883 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 5884 } 5885 5886 return 0; 5887 5888 err_disable_clks: 5889 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); 5890 5891 return err; 5892 } 5893 5894 static int fu540_c000_init(struct platform_device *pdev) 5895 { 5896 mgmt->reg = devm_platform_ioremap_resource(pdev, 1); 5897 if (IS_ERR(mgmt->reg)) 5898 return PTR_ERR(mgmt->reg); 5899 5900 return macb_init_dflt(pdev); 5901 } 5902 5903 static int init_reset_optional(struct platform_device *pdev) 5904 { 5905 struct net_device *netdev = platform_get_drvdata(pdev); 5906 struct macb *bp = netdev_priv(netdev); 5907 int ret; 5908 5909 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 5910 /* Ensure PHY device used in SGMII mode is ready */ 5911 bp->phy = devm_phy_optional_get(&pdev->dev, NULL); 5912 5913 if (IS_ERR(bp->phy)) 5914 return dev_err_probe(&pdev->dev, PTR_ERR(bp->phy), 5915 "failed to get SGMII PHY\n"); 5916 5917 ret = phy_init(bp->phy); 5918 if (ret) 5919 return dev_err_probe(&pdev->dev, ret, 5920 "failed to init SGMII PHY\n"); 5921 5922 ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); 5923 if (!ret) { 5924 u32 pm_info[2]; 5925 5926 ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", 5927 pm_info, ARRAY_SIZE(pm_info)); 5928 if (ret) { 5929 dev_err(&pdev->dev, "Failed to read power management information\n"); 5930 goto err_out_phy_exit; 5931 } 5932 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); 5933 if (ret) 5934 goto err_out_phy_exit; 5935 5936 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); 5937 if (ret) 5938 goto err_out_phy_exit; 5939 } 5940 5941 } 5942 5943 /* Fully reset controller at hardware level if mapped in device tree */ 5944 ret = device_reset_optional(&pdev->dev); 5945 if (ret) { 5946 phy_exit(bp->phy); 5947 return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); 5948 } 5949 5950 ret = macb_init_dflt(pdev); 5951 5952 err_out_phy_exit: 5953 if (ret) 5954 phy_exit(bp->phy); 5955 5956 return ret; 5957 } 5958 5959 static int eyeq5_init(struct platform_device *pdev) 5960 { 5961 struct net_device *netdev = platform_get_drvdata(pdev); 5962 struct macb *bp = netdev_priv(netdev); 5963 struct device *dev = &pdev->dev; 5964 int ret; 5965 5966 bp->phy = devm_phy_get(dev, NULL); 5967 if (IS_ERR(bp->phy)) 5968 return dev_err_probe(dev, PTR_ERR(bp->phy), 5969 "failed to get PHY\n"); 5970 5971 ret = phy_init(bp->phy); 5972 if (ret) 5973 return dev_err_probe(dev, ret, "failed to init PHY\n"); 5974 5975 ret = macb_init_dflt(pdev); 5976 if (ret) 5977 phy_exit(bp->phy); 5978 return ret; 5979 } 5980 5981 static int macb_alloc_tieoff(struct macb *bp) 5982 { 5983 /* Tieoff is a workaround in case HW cannot disable queues, for PM. */ 5984 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) 5985 return 0; 5986 5987 bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev, 5988 macb_dma_desc_get_size(&bp->info), 5989 &bp->rx_ring_tieoff_dma, 5990 GFP_KERNEL); 5991 if (!bp->rx_ring_tieoff) 5992 return -ENOMEM; 5993 5994 macb_set_addr(&bp->info, bp->rx_ring_tieoff, 5995 MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED)); 5996 5997 bp->rx_ring_tieoff->ctrl = 0; 5998 5999 return 0; 6000 } 6001 6002 static void macb_free_tieoff(struct macb *bp) 6003 { 6004 if (!bp->rx_ring_tieoff) 6005 return; 6006 6007 dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(&bp->info), 6008 bp->rx_ring_tieoff, 6009 bp->rx_ring_tieoff_dma); 6010 bp->rx_ring_tieoff = NULL; 6011 } 6012 6013 static const struct macb_usrio_config mpfs_usrio = { 6014 .tsu_source = 0, 6015 }; 6016 6017 static const struct macb_usrio_config sama7g5_gem_usrio = { 6018 .mii = 0, 6019 .rmii = 1, 6020 .rgmii = 2, 6021 .refclk = BIT(2), 6022 .refclk_default_external = false, 6023 .hdfctlen = BIT(6), 6024 }; 6025 6026 static const struct macb_usrio_config sama7g5_emac_usrio = { 6027 .mii = 0, 6028 .rmii = 1, 6029 .rgmii = 2, 6030 .refclk = BIT(2), 6031 .refclk_default_external = true, 6032 .hdfctlen = BIT(6), 6033 }; 6034 6035 static const struct macb_config fu540_c000_config = { 6036 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 6037 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_USRIO_HAS_MII, 6038 .dma_burst_length = 16, 6039 .clk_init = fu540_c000_clk_init, 6040 .init = fu540_c000_init, 6041 .jumbo_max_len = 10240, 6042 .usrio = &at91_default_usrio, 6043 }; 6044 6045 static const struct macb_config at91sam9260_config = { 6046 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 6047 MACB_CAPS_USRIO_HAS_MII, 6048 .usrio = &at91_default_usrio, 6049 }; 6050 6051 static const struct macb_config sama5d3macb_config = { 6052 .caps = MACB_CAPS_SG_DISABLED | 6053 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 6054 MACB_CAPS_USRIO_HAS_MII, 6055 .usrio = &at91_default_usrio, 6056 }; 6057 6058 static const struct macb_config pc302gem_config = { 6059 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | 6060 MACB_CAPS_USRIO_HAS_MII, 6061 .dma_burst_length = 16, 6062 .usrio = &at91_default_usrio, 6063 }; 6064 6065 static const struct macb_config sama5d2_config = { 6066 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO | 6067 MACB_CAPS_USRIO_HAS_MII, 6068 .dma_burst_length = 16, 6069 .jumbo_max_len = 10240, 6070 .usrio = &at91_default_usrio, 6071 }; 6072 6073 static const struct macb_config sama5d29_config = { 6074 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP | 6075 MACB_CAPS_USRIO_HAS_MII, 6076 .dma_burst_length = 16, 6077 .usrio = &at91_default_usrio, 6078 }; 6079 6080 static const struct macb_config sama5d3_config = { 6081 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | 6082 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO | 6083 MACB_CAPS_USRIO_HAS_MII, 6084 .dma_burst_length = 16, 6085 .jumbo_max_len = 10240, 6086 .usrio = &at91_default_usrio, 6087 }; 6088 6089 static const struct macb_config sama5d4_config = { 6090 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 6091 MACB_CAPS_USRIO_HAS_MII, 6092 .dma_burst_length = 4, 6093 .usrio = &at91_default_usrio, 6094 }; 6095 6096 static const struct macb_config emac_config = { 6097 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC | 6098 MACB_CAPS_USRIO_HAS_MII, 6099 .clk_init = at91ether_clk_init, 6100 .init = at91ether_init, 6101 .usrio = &at91_default_usrio, 6102 }; 6103 6104 static const struct macb_config np4_config = { 6105 .caps = MACB_CAPS_USRIO_DISABLED, 6106 }; 6107 6108 static const struct macb_config zynqmp_config = { 6109 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 6110 MACB_CAPS_JUMBO | 6111 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | 6112 MACB_CAPS_USRIO_HAS_MII, 6113 .dma_burst_length = 16, 6114 .init = init_reset_optional, 6115 .jumbo_max_len = 10240, 6116 .usrio = &at91_default_usrio, 6117 }; 6118 6119 static const struct macb_config zynq_config = { 6120 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 6121 MACB_CAPS_NEEDS_RSTONUBR | 6122 MACB_CAPS_USRIO_HAS_MII, 6123 .dma_burst_length = 16, 6124 .usrio = &at91_default_usrio, 6125 }; 6126 6127 static const struct macb_config mpfs_config = { 6128 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 6129 MACB_CAPS_JUMBO | 6130 MACB_CAPS_GEM_HAS_PTP | 6131 MACB_CAPS_USRIO_HAS_TSUCLK_SOURCE, 6132 .dma_burst_length = 16, 6133 .init = init_reset_optional, 6134 .usrio = &mpfs_usrio, 6135 .max_tx_length = 4040, /* Cadence Erratum 1686 */ 6136 .jumbo_max_len = 4040, 6137 }; 6138 6139 static const struct macb_config sama7g5_gem_config = { 6140 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 6141 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 6142 MACB_CAPS_USRIO_HAS_REFCLK_SOURCE | 6143 MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP | 6144 MACB_CAPS_USRIO_HAS_MII, 6145 .dma_burst_length = 16, 6146 .usrio = &sama7g5_gem_usrio, 6147 }; 6148 6149 static const struct macb_config sama7g5_emac_config = { 6150 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 6151 MACB_CAPS_MIIONRGMII | 6152 MACB_CAPS_USRIO_HAS_REFCLK_SOURCE | 6153 MACB_CAPS_GEM_HAS_PTP | 6154 MACB_CAPS_USRIO_HAS_MII, 6155 .dma_burst_length = 16, 6156 .usrio = &sama7g5_emac_usrio, 6157 }; 6158 6159 static const struct macb_config versal_config = { 6160 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 6161 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | 6162 MACB_CAPS_NEED_TSUCLK | MACB_CAPS_QUEUE_DISABLE | 6163 MACB_CAPS_QBV | 6164 MACB_CAPS_USRIO_HAS_MII, 6165 .dma_burst_length = 16, 6166 .init = init_reset_optional, 6167 .jumbo_max_len = 10240, 6168 .usrio = &at91_default_usrio, 6169 }; 6170 6171 static const struct macb_config eyeq5_config = { 6172 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 6173 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_QUEUE_DISABLE | 6174 MACB_CAPS_NO_LSO | MACB_CAPS_EEE, 6175 .dma_burst_length = 16, 6176 .init = eyeq5_init, 6177 .jumbo_max_len = 10240, 6178 }; 6179 6180 static const struct macb_config raspberrypi_rp1_config = { 6181 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 6182 MACB_CAPS_JUMBO | 6183 MACB_CAPS_GEM_HAS_PTP | 6184 MACB_CAPS_EEE | 6185 MACB_CAPS_USRIO_HAS_MII, 6186 .dma_burst_length = 16, 6187 .usrio = &at91_default_usrio, 6188 .jumbo_max_len = 10240, 6189 }; 6190 6191 static const struct macb_config pic64hpsc_config = { 6192 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 6193 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_USRIO_DISABLED, 6194 .dma_burst_length = 16, 6195 .init = init_reset_optional, 6196 .jumbo_max_len = 16383, 6197 }; 6198 6199 static const struct of_device_id macb_dt_ids[] = { 6200 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 6201 { .compatible = "cdns,macb" }, 6202 { .compatible = "cdns,np4-macb", .data = &np4_config }, 6203 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 6204 { .compatible = "cdns,gem", .data = &pc302gem_config }, 6205 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 6206 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 6207 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, 6208 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 6209 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 6210 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 6211 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 6212 { .compatible = "cdns,emac", .data = &emac_config }, 6213 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */ 6214 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ 6215 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 6216 { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, 6217 { .compatible = "microchip,pic64hpsc-gem", .data = &pic64hpsc_config}, 6218 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, 6219 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, 6220 { .compatible = "mobileye,eyeq5-gem", .data = &eyeq5_config }, 6221 { .compatible = "raspberrypi,rp1-gem", .data = &raspberrypi_rp1_config }, 6222 { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config}, 6223 { .compatible = "xlnx,zynq-gem", .data = &zynq_config }, 6224 { .compatible = "xlnx,versal-gem", .data = &versal_config}, 6225 { /* sentinel */ } 6226 }; 6227 MODULE_DEVICE_TABLE(of, macb_dt_ids); 6228 6229 static const struct macb_config default_gem_config = { 6230 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 6231 MACB_CAPS_JUMBO | 6232 MACB_CAPS_GEM_HAS_PTP, 6233 .dma_burst_length = 16, 6234 .usrio = &at91_default_usrio, 6235 .jumbo_max_len = 10240, 6236 }; 6237 6238 static int macb_probe(struct platform_device *pdev) 6239 { 6240 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 6241 struct device_node *np = pdev->dev.of_node; 6242 const struct macb_config *macb_config; 6243 struct clk *tsu_clk = NULL; 6244 phy_interface_t interface; 6245 struct net_device *netdev; 6246 struct resource *regs; 6247 u32 wtrmrk_rst_val; 6248 void __iomem *mem; 6249 struct macb *bp; 6250 unsigned int q; 6251 int num_queues; 6252 bool native_io; 6253 int err, val; 6254 6255 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); 6256 if (IS_ERR(mem)) 6257 return PTR_ERR(mem); 6258 6259 macb_config = of_device_get_match_data(&pdev->dev); 6260 if (!macb_config) 6261 macb_config = &default_gem_config; 6262 6263 err = macb_clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk, 6264 macb_config); 6265 if (err) 6266 return err; 6267 6268 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 6269 pm_runtime_use_autosuspend(&pdev->dev); 6270 pm_runtime_get_noresume(&pdev->dev); 6271 pm_runtime_set_active(&pdev->dev); 6272 pm_runtime_enable(&pdev->dev); 6273 native_io = hw_is_native_io(mem); 6274 6275 num_queues = macb_probe_queues(&pdev->dev, mem, native_io); 6276 if (num_queues < 0) { 6277 err = num_queues; 6278 goto err_disable_clocks; 6279 } 6280 6281 netdev = alloc_etherdev_mq(sizeof(*bp), num_queues); 6282 if (!netdev) { 6283 err = -ENOMEM; 6284 goto err_disable_clocks; 6285 } 6286 6287 netdev->base_addr = regs->start; 6288 6289 SET_NETDEV_DEV(netdev, &pdev->dev); 6290 6291 bp = netdev_priv(netdev); 6292 bp->pdev = pdev; 6293 bp->netdev = netdev; 6294 bp->regs = mem; 6295 bp->native_io = native_io; 6296 for (q = 0; q < MACB_MAX_QUEUES; q++) 6297 bp->info.queue_stats[q] = &bp->queues[q].stats; 6298 if (native_io) { 6299 bp->macb_reg_readl = hw_readl_native; 6300 bp->macb_reg_writel = hw_writel_native; 6301 } else { 6302 bp->macb_reg_readl = hw_readl; 6303 bp->macb_reg_writel = hw_writel; 6304 } 6305 bp->num_queues = num_queues; 6306 bp->dma_burst_length = macb_config->dma_burst_length; 6307 bp->pclk = pclk; 6308 bp->hclk = hclk; 6309 bp->tx_clk = tx_clk; 6310 bp->rx_clk = rx_clk; 6311 bp->tsu_clk = tsu_clk; 6312 bp->jumbo_max_len = macb_config->jumbo_max_len; 6313 6314 if (!hw_is_gem(bp->regs, bp->native_io)) 6315 bp->max_tx_length = MACB_MAX_TX_LEN; 6316 else if (macb_config->max_tx_length) 6317 bp->max_tx_length = macb_config->max_tx_length; 6318 else 6319 bp->max_tx_length = GEM_MAX_TX_LEN; 6320 6321 bp->wol = 0; 6322 device_set_wakeup_capable(&pdev->dev, 1); 6323 6324 bp->usrio = macb_config->usrio; 6325 6326 if (of_property_read_bool(bp->pdev->dev.of_node, "cdns,timer-adjust") && 6327 IS_ENABLED(CONFIG_MACB_USE_HWSTAMP)) { 6328 dev_err(&pdev->dev, "Timer adjust mode is not supported\n"); 6329 err = -EINVAL; 6330 goto err_out_free_netdev; 6331 } 6332 6333 /* By default we set to partial store and forward mode for zynqmp. 6334 * Disable if not set in devicetree. 6335 */ 6336 if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) { 6337 err = of_property_read_u32(bp->pdev->dev.of_node, 6338 "cdns,rx-watermark", 6339 &bp->rx_watermark); 6340 6341 if (!err) { 6342 /* Disable partial store and forward in case of error or 6343 * invalid watermark value 6344 */ 6345 wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1; 6346 if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) { 6347 dev_info(&bp->pdev->dev, "Invalid watermark value\n"); 6348 bp->rx_watermark = 0; 6349 } 6350 } 6351 } 6352 spin_lock_init(&bp->lock); 6353 spin_lock_init(&bp->stats_lock); 6354 spin_lock_init(&bp->tsu_clk_lock); 6355 mutex_init(&bp->mac_cfg_lock); 6356 6357 /* setup capabilities */ 6358 macb_configure_caps(bp, macb_config); 6359 6360 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { 6361 /* The name is lying on EMAC: sizes aren't configurable. */ 6362 bp->configured_rx_ring_size = AT91ETHER_MAX_RX_DESCR; 6363 bp->configured_tx_ring_size = 1; /* tiny ring buffer */ 6364 } else { 6365 bp->configured_rx_ring_size = DEFAULT_RX_RING_SIZE; 6366 bp->configured_tx_ring_size = DEFAULT_TX_RING_SIZE; 6367 } 6368 6369 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 6370 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 6371 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); 6372 if (err) { 6373 dev_err(&pdev->dev, "failed to set DMA mask\n"); 6374 goto err_out_free_netdev; 6375 } 6376 bp->caps |= MACB_CAPS_DMA_64B; 6377 } 6378 #endif 6379 platform_set_drvdata(pdev, netdev); 6380 6381 netdev->irq = platform_get_irq(pdev, 0); 6382 if (netdev->irq < 0) { 6383 err = netdev->irq; 6384 goto err_out_free_netdev; 6385 } 6386 6387 /* MTU range: 68 - 1518 or 10240 */ 6388 netdev->min_mtu = GEM_MTU_MIN_SIZE; 6389 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 6390 netdev->max_mtu = MIN(bp->jumbo_max_len, RX_BUFFER_MAX) - 6391 ETH_HLEN - ETH_FCS_LEN; 6392 else 6393 netdev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; 6394 6395 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 6396 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 6397 if (val) 6398 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 6399 macb_dma_desc_get_size(&bp->info); 6400 6401 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 6402 if (val) 6403 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 6404 macb_dma_desc_get_size(&bp->info); 6405 } 6406 6407 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 6408 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 6409 bp->rx_intr_mask |= MACB_BIT(RXUBR); 6410 6411 err = of_get_ethdev_address(np, bp->netdev); 6412 if (err == -EPROBE_DEFER) 6413 goto err_out_free_netdev; 6414 else if (err) 6415 macb_get_hwaddr(bp); 6416 6417 err = of_get_phy_mode(np, &interface); 6418 if (err) 6419 /* not found in DT, MII by default */ 6420 bp->phy_interface = PHY_INTERFACE_MODE_MII; 6421 else 6422 bp->phy_interface = interface; 6423 6424 /* IP specific init */ 6425 err = macb_init(pdev, macb_config); 6426 if (err) 6427 goto err_out_free_netdev; 6428 6429 err = macb_mii_init(bp); 6430 if (err) 6431 goto err_out_phy_exit; 6432 6433 netif_carrier_off(netdev); 6434 6435 err = macb_alloc_tieoff(bp); 6436 if (err) 6437 goto err_out_unregister_mdio; 6438 6439 INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task); 6440 INIT_DELAYED_WORK(&bp->tx_lpi_work, macb_tx_lpi_work_fn); 6441 6442 err = register_netdev(netdev); 6443 if (err) { 6444 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 6445 goto err_out_free_tieoff; 6446 } 6447 6448 netdev_info(netdev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 6449 macb_is_gem(&bp->info) ? "GEM" : "MACB", 6450 macb_readl(bp, MID), netdev->base_addr, netdev->irq, 6451 netdev->dev_addr); 6452 6453 pm_runtime_put_autosuspend(&bp->pdev->dev); 6454 6455 return 0; 6456 6457 err_out_free_tieoff: 6458 macb_free_tieoff(bp); 6459 6460 err_out_unregister_mdio: 6461 if (bp->mii_bus) { 6462 mdiobus_unregister(bp->mii_bus); 6463 mdiobus_free(bp->mii_bus); 6464 } 6465 phylink_destroy(bp->phylink); 6466 6467 err_out_phy_exit: 6468 phy_exit(bp->phy); 6469 6470 err_out_free_netdev: 6471 free_netdev(netdev); 6472 6473 err_disable_clocks: 6474 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); 6475 pm_runtime_disable(&pdev->dev); 6476 pm_runtime_set_suspended(&pdev->dev); 6477 pm_runtime_dont_use_autosuspend(&pdev->dev); 6478 6479 return err; 6480 } 6481 6482 static void macb_remove(struct platform_device *pdev) 6483 { 6484 struct net_device *netdev; 6485 struct macb *bp; 6486 6487 netdev = platform_get_drvdata(pdev); 6488 6489 if (netdev) { 6490 bp = netdev_priv(netdev); 6491 unregister_netdev(netdev); 6492 macb_free_tieoff(bp); 6493 phy_exit(bp->phy); 6494 if (bp->mii_bus) { 6495 mdiobus_unregister(bp->mii_bus); 6496 mdiobus_free(bp->mii_bus); 6497 } 6498 6499 device_set_wakeup_enable(&bp->pdev->dev, 0); 6500 cancel_delayed_work_sync(&bp->tx_lpi_work); 6501 cancel_work_sync(&bp->hresp_err_bh_work); 6502 pm_runtime_disable(&pdev->dev); 6503 pm_runtime_dont_use_autosuspend(&pdev->dev); 6504 pm_runtime_set_suspended(&pdev->dev); 6505 phylink_destroy(bp->phylink); 6506 free_netdev(netdev); 6507 } 6508 } 6509 6510 static int __maybe_unused macb_suspend(struct device *dev) 6511 { 6512 struct net_device *netdev = dev_get_drvdata(dev); 6513 struct macb *bp = netdev_priv(netdev); 6514 struct in_ifaddr *ifa = NULL; 6515 struct macb_queue *queue; 6516 struct in_device *idev; 6517 unsigned long flags; 6518 u32 tmp, ifa_local; 6519 unsigned int q; 6520 6521 if (!device_may_wakeup(&bp->netdev->dev)) 6522 phy_exit(bp->phy); 6523 6524 if (!netif_running(netdev)) 6525 return 0; 6526 6527 if (bp->wol & MACB_WOL_ENABLED) { 6528 if (bp->wolopts & WAKE_ARP) { 6529 /* Check for IP address in WOL ARP mode */ 6530 rcu_read_lock(); 6531 idev = __in_dev_get_rcu(bp->netdev); 6532 if (idev) 6533 ifa = rcu_dereference(idev->ifa_list); 6534 if (!ifa) { 6535 rcu_read_unlock(); 6536 netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); 6537 return -EOPNOTSUPP; 6538 } 6539 ifa_local = be32_to_cpu(ifa->ifa_local); 6540 rcu_read_unlock(); 6541 } 6542 6543 spin_lock_irqsave(&bp->lock, flags); 6544 6545 /* Disable Tx and Rx engines before disabling the queues, 6546 * this is mandatory as per the IP spec sheet 6547 */ 6548 tmp = macb_readl(bp, NCR); 6549 macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE))); 6550 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 6551 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) 6552 macb_writel(bp, RBQPH, 6553 upper_32_bits(bp->rx_ring_tieoff_dma)); 6554 #endif 6555 for (q = 0, queue = bp->queues; q < bp->num_queues; 6556 ++q, ++queue) { 6557 /* Disable RX queues */ 6558 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) { 6559 queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE)); 6560 } else { 6561 /* Tie off RX queues */ 6562 queue_writel(queue, RBQP, 6563 lower_32_bits(bp->rx_ring_tieoff_dma)); 6564 } 6565 /* Disable all interrupts */ 6566 queue_writel(queue, IDR, -1); 6567 queue_readl(queue, ISR); 6568 macb_queue_isr_clear(bp, queue, -1); 6569 } 6570 /* Enable Receive engine */ 6571 macb_writel(bp, NCR, tmp | MACB_BIT(RE)); 6572 /* Flush all status bits */ 6573 macb_writel(bp, TSR, -1); 6574 macb_writel(bp, RSR, -1); 6575 6576 tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0; 6577 if (bp->wolopts & WAKE_ARP) { 6578 tmp |= MACB_BIT(ARP); 6579 /* write IP address into register */ 6580 tmp |= MACB_BFEXT(IP, ifa_local); 6581 } 6582 6583 if (macb_is_gem(&bp->info)) { 6584 queue_writel(bp->queues, IER, GEM_BIT(WOL)); 6585 gem_writel(bp, WOL, tmp); 6586 } else { 6587 queue_writel(bp->queues, IER, MACB_BIT(WOL)); 6588 macb_writel(bp, WOL, tmp); 6589 } 6590 spin_unlock_irqrestore(&bp->lock, flags); 6591 6592 enable_irq_wake(bp->queues[0].irq); 6593 } 6594 6595 netif_device_detach(netdev); 6596 for (q = 0, queue = bp->queues; q < bp->num_queues; 6597 ++q, ++queue) { 6598 napi_disable(&queue->napi_rx); 6599 napi_disable(&queue->napi_tx); 6600 } 6601 6602 if (!(bp->wol & MACB_WOL_ENABLED)) { 6603 rtnl_lock(); 6604 phylink_stop(bp->phylink); 6605 rtnl_unlock(); 6606 spin_lock_irqsave(&bp->lock, flags); 6607 macb_reset_hw(bp); 6608 spin_unlock_irqrestore(&bp->lock, flags); 6609 } 6610 6611 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 6612 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 6613 6614 if (netdev->hw_features & NETIF_F_NTUPLE) 6615 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 6616 6617 if (bp->ptp_info) 6618 bp->ptp_info->ptp_remove(netdev); 6619 if (!device_may_wakeup(dev)) 6620 pm_runtime_force_suspend(dev); 6621 6622 return 0; 6623 } 6624 6625 static int __maybe_unused macb_resume(struct device *dev) 6626 { 6627 struct net_device *netdev = dev_get_drvdata(dev); 6628 struct macb *bp = netdev_priv(netdev); 6629 struct macb_context *ctx = bp->ctx; 6630 struct macb_queue *queue; 6631 unsigned long flags; 6632 unsigned int q; 6633 6634 if (!device_may_wakeup(&bp->netdev->dev)) 6635 phy_init(bp->phy); 6636 6637 if (!netif_running(netdev)) 6638 return 0; 6639 6640 if (!device_may_wakeup(dev)) 6641 pm_runtime_force_resume(dev); 6642 6643 if (bp->wol & MACB_WOL_ENABLED) { 6644 spin_lock_irqsave(&bp->lock, flags); 6645 /* Disable WoL */ 6646 if (macb_is_gem(&bp->info)) { 6647 queue_writel(bp->queues, IDR, GEM_BIT(WOL)); 6648 gem_writel(bp, WOL, 0); 6649 } else { 6650 queue_writel(bp->queues, IDR, MACB_BIT(WOL)); 6651 macb_writel(bp, WOL, 0); 6652 } 6653 /* Clear ISR on queue 0 */ 6654 queue_readl(bp->queues, ISR); 6655 macb_queue_isr_clear(bp, bp->queues, -1); 6656 spin_unlock_irqrestore(&bp->lock, flags); 6657 6658 disable_irq_wake(bp->queues[0].irq); 6659 6660 /* Now make sure we disable phy before moving 6661 * to common restore path 6662 */ 6663 rtnl_lock(); 6664 phylink_stop(bp->phylink); 6665 rtnl_unlock(); 6666 } 6667 6668 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 6669 macb_init_buffers(bp); 6670 6671 for (q = 0, queue = bp->queues; q < bp->num_queues; 6672 ++q, ++queue) { 6673 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 6674 if (macb_is_gem(&bp->info)) 6675 gem_init_rx_ring(ctx, q); 6676 else 6677 macb_init_rx_ring(ctx, q); 6678 } 6679 6680 napi_enable(&queue->napi_rx); 6681 napi_enable(&queue->napi_tx); 6682 } 6683 6684 if (netdev->hw_features & NETIF_F_NTUPLE) 6685 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 6686 6687 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 6688 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 6689 6690 macb_writel(bp, NCR, MACB_BIT(MPE)); 6691 macb_init_hw(bp); 6692 macb_set_rx_mode(netdev); 6693 macb_restore_features(bp); 6694 rtnl_lock(); 6695 6696 phylink_start(bp->phylink); 6697 rtnl_unlock(); 6698 6699 netif_device_attach(netdev); 6700 if (bp->ptp_info) 6701 bp->ptp_info->ptp_init(netdev); 6702 6703 return 0; 6704 } 6705 6706 static int __maybe_unused macb_runtime_suspend(struct device *dev) 6707 { 6708 struct net_device *netdev = dev_get_drvdata(dev); 6709 struct macb *bp = netdev_priv(netdev); 6710 6711 if (!(device_may_wakeup(dev))) 6712 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); 6713 else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) 6714 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); 6715 6716 return 0; 6717 } 6718 6719 static int __maybe_unused macb_runtime_resume(struct device *dev) 6720 { 6721 struct net_device *netdev = dev_get_drvdata(dev); 6722 struct macb *bp = netdev_priv(netdev); 6723 6724 if (!(device_may_wakeup(dev))) { 6725 clk_prepare_enable(bp->pclk); 6726 clk_prepare_enable(bp->hclk); 6727 clk_prepare_enable(bp->tx_clk); 6728 clk_prepare_enable(bp->rx_clk); 6729 clk_prepare_enable(bp->tsu_clk); 6730 } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { 6731 clk_prepare_enable(bp->tsu_clk); 6732 } 6733 6734 return 0; 6735 } 6736 6737 static void macb_shutdown(struct platform_device *pdev) 6738 { 6739 struct net_device *netdev = platform_get_drvdata(pdev); 6740 6741 rtnl_lock(); 6742 6743 if (netif_running(netdev)) 6744 dev_close(netdev); 6745 6746 netif_device_detach(netdev); 6747 6748 rtnl_unlock(); 6749 } 6750 6751 static const struct dev_pm_ops macb_pm_ops = { 6752 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 6753 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 6754 }; 6755 6756 static struct platform_driver macb_driver = { 6757 .probe = macb_probe, 6758 .remove = macb_remove, 6759 .driver = { 6760 .name = "macb", 6761 .of_match_table = macb_dt_ids, 6762 .pm = &macb_pm_ops, 6763 }, 6764 .shutdown = macb_shutdown, 6765 }; 6766 6767 module_platform_driver(macb_driver); 6768 6769 MODULE_LICENSE("GPL"); 6770 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 6771 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 6772 MODULE_ALIAS("platform:macb"); 6773