1 /* 2 * Broadcom GENET (Gigabit Ethernet) controller driver 3 * 4 * Copyright (c) 2014 Broadcom Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) "bcmgenet: " fmt 12 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/sched.h> 16 #include <linux/types.h> 17 #include <linux/fcntl.h> 18 #include <linux/interrupt.h> 19 #include <linux/string.h> 20 #include <linux/if_ether.h> 21 #include <linux/init.h> 22 #include <linux/errno.h> 23 #include <linux/delay.h> 24 #include <linux/platform_device.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/pm.h> 27 #include <linux/clk.h> 28 #include <linux/of.h> 29 #include <linux/of_address.h> 30 #include <linux/of_irq.h> 31 #include <linux/of_net.h> 32 #include <linux/of_platform.h> 33 #include <net/arp.h> 34 35 #include <linux/mii.h> 36 #include <linux/ethtool.h> 37 #include <linux/netdevice.h> 38 #include <linux/inetdevice.h> 39 #include <linux/etherdevice.h> 40 #include <linux/skbuff.h> 41 #include <linux/in.h> 42 #include <linux/ip.h> 43 #include <linux/ipv6.h> 44 #include <linux/phy.h> 45 #include <linux/platform_data/bcmgenet.h> 46 47 #include <asm/unaligned.h> 48 49 #include "bcmgenet.h" 50 51 /* Maximum number of hardware queues, downsized if needed */ 52 #define GENET_MAX_MQ_CNT 4 53 54 /* Default highest priority queue for multi queue support */ 55 #define GENET_Q0_PRIORITY 0 56 57 #define GENET_DEFAULT_BD_CNT \ 58 (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->bds_cnt) 59 60 #define RX_BUF_LENGTH 2048 61 #define SKB_ALIGNMENT 32 62 63 /* Tx/Rx DMA register offset, skip 256 descriptors */ 64 #define WORDS_PER_BD(p) (p->hw_params->words_per_bd) 65 #define DMA_DESC_SIZE (WORDS_PER_BD(priv) * sizeof(u32)) 66 67 #define GENET_TDMA_REG_OFF (priv->hw_params->tdma_offset + \ 68 TOTAL_DESC * DMA_DESC_SIZE) 69 70 #define GENET_RDMA_REG_OFF (priv->hw_params->rdma_offset + \ 71 TOTAL_DESC * DMA_DESC_SIZE) 72 73 static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv, 74 void __iomem *d, u32 value) 75 { 76 __raw_writel(value, d + DMA_DESC_LENGTH_STATUS); 77 } 78 79 static inline u32 dmadesc_get_length_status(struct bcmgenet_priv *priv, 80 void __iomem *d) 81 { 82 return __raw_readl(d + DMA_DESC_LENGTH_STATUS); 83 } 84 85 static inline void dmadesc_set_addr(struct bcmgenet_priv *priv, 86 void __iomem *d, 87 dma_addr_t addr) 88 { 89 __raw_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO); 90 91 /* Register writes to GISB bus can take couple hundred nanoseconds 92 * and are done for each packet, save these expensive writes unless 93 * the platform is explicitly configured for 64-bits/LPAE. 94 */ 95 #ifdef CONFIG_PHYS_ADDR_T_64BIT 96 if (priv->hw_params->flags & GENET_HAS_40BITS) 97 __raw_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI); 98 #endif 99 } 100 101 /* Combined address + length/status setter */ 102 static inline void dmadesc_set(struct bcmgenet_priv *priv, 103 void __iomem *d, dma_addr_t addr, u32 val) 104 { 105 dmadesc_set_length_status(priv, d, val); 106 dmadesc_set_addr(priv, d, addr); 107 } 108 109 static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv, 110 void __iomem *d) 111 { 112 dma_addr_t addr; 113 114 addr = __raw_readl(d + DMA_DESC_ADDRESS_LO); 115 116 /* Register writes to GISB bus can take couple hundred nanoseconds 117 * and are done for each packet, save these expensive writes unless 118 * the platform is explicitly configured for 64-bits/LPAE. 119 */ 120 #ifdef CONFIG_PHYS_ADDR_T_64BIT 121 if (priv->hw_params->flags & GENET_HAS_40BITS) 122 addr |= (u64)__raw_readl(d + DMA_DESC_ADDRESS_HI) << 32; 123 #endif 124 return addr; 125 } 126 127 #define GENET_VER_FMT "%1d.%1d EPHY: 0x%04x" 128 129 #define GENET_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \ 130 NETIF_MSG_LINK) 131 132 static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv) 133 { 134 if (GENET_IS_V1(priv)) 135 return bcmgenet_rbuf_readl(priv, RBUF_FLUSH_CTRL_V1); 136 else 137 return bcmgenet_sys_readl(priv, SYS_RBUF_FLUSH_CTRL); 138 } 139 140 static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val) 141 { 142 if (GENET_IS_V1(priv)) 143 bcmgenet_rbuf_writel(priv, val, RBUF_FLUSH_CTRL_V1); 144 else 145 bcmgenet_sys_writel(priv, val, SYS_RBUF_FLUSH_CTRL); 146 } 147 148 /* These macros are defined to deal with register map change 149 * between GENET1.1 and GENET2. Only those currently being used 150 * by driver are defined. 151 */ 152 static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv) 153 { 154 if (GENET_IS_V1(priv)) 155 return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1); 156 else 157 return __raw_readl(priv->base + 158 priv->hw_params->tbuf_offset + TBUF_CTRL); 159 } 160 161 static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val) 162 { 163 if (GENET_IS_V1(priv)) 164 bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1); 165 else 166 __raw_writel(val, priv->base + 167 priv->hw_params->tbuf_offset + TBUF_CTRL); 168 } 169 170 static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv) 171 { 172 if (GENET_IS_V1(priv)) 173 return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1); 174 else 175 return __raw_readl(priv->base + 176 priv->hw_params->tbuf_offset + TBUF_BP_MC); 177 } 178 179 static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val) 180 { 181 if (GENET_IS_V1(priv)) 182 bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1); 183 else 184 __raw_writel(val, priv->base + 185 priv->hw_params->tbuf_offset + TBUF_BP_MC); 186 } 187 188 /* RX/TX DMA register accessors */ 189 enum dma_reg { 190 DMA_RING_CFG = 0, 191 DMA_CTRL, 192 DMA_STATUS, 193 DMA_SCB_BURST_SIZE, 194 DMA_ARB_CTRL, 195 DMA_PRIORITY_0, 196 DMA_PRIORITY_1, 197 DMA_PRIORITY_2, 198 }; 199 200 static const u8 bcmgenet_dma_regs_v3plus[] = { 201 [DMA_RING_CFG] = 0x00, 202 [DMA_CTRL] = 0x04, 203 [DMA_STATUS] = 0x08, 204 [DMA_SCB_BURST_SIZE] = 0x0C, 205 [DMA_ARB_CTRL] = 0x2C, 206 [DMA_PRIORITY_0] = 0x30, 207 [DMA_PRIORITY_1] = 0x34, 208 [DMA_PRIORITY_2] = 0x38, 209 }; 210 211 static const u8 bcmgenet_dma_regs_v2[] = { 212 [DMA_RING_CFG] = 0x00, 213 [DMA_CTRL] = 0x04, 214 [DMA_STATUS] = 0x08, 215 [DMA_SCB_BURST_SIZE] = 0x0C, 216 [DMA_ARB_CTRL] = 0x30, 217 [DMA_PRIORITY_0] = 0x34, 218 [DMA_PRIORITY_1] = 0x38, 219 [DMA_PRIORITY_2] = 0x3C, 220 }; 221 222 static const u8 bcmgenet_dma_regs_v1[] = { 223 [DMA_CTRL] = 0x00, 224 [DMA_STATUS] = 0x04, 225 [DMA_SCB_BURST_SIZE] = 0x0C, 226 [DMA_ARB_CTRL] = 0x30, 227 [DMA_PRIORITY_0] = 0x34, 228 [DMA_PRIORITY_1] = 0x38, 229 [DMA_PRIORITY_2] = 0x3C, 230 }; 231 232 /* Set at runtime once bcmgenet version is known */ 233 static const u8 *bcmgenet_dma_regs; 234 235 static inline struct bcmgenet_priv *dev_to_priv(struct device *dev) 236 { 237 return netdev_priv(dev_get_drvdata(dev)); 238 } 239 240 static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv, 241 enum dma_reg r) 242 { 243 return __raw_readl(priv->base + GENET_TDMA_REG_OFF + 244 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]); 245 } 246 247 static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv, 248 u32 val, enum dma_reg r) 249 { 250 __raw_writel(val, priv->base + GENET_TDMA_REG_OFF + 251 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]); 252 } 253 254 static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv, 255 enum dma_reg r) 256 { 257 return __raw_readl(priv->base + GENET_RDMA_REG_OFF + 258 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]); 259 } 260 261 static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv, 262 u32 val, enum dma_reg r) 263 { 264 __raw_writel(val, priv->base + GENET_RDMA_REG_OFF + 265 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]); 266 } 267 268 /* RDMA/TDMA ring registers and accessors 269 * we merge the common fields and just prefix with T/D the registers 270 * having different meaning depending on the direction 271 */ 272 enum dma_ring_reg { 273 TDMA_READ_PTR = 0, 274 RDMA_WRITE_PTR = TDMA_READ_PTR, 275 TDMA_READ_PTR_HI, 276 RDMA_WRITE_PTR_HI = TDMA_READ_PTR_HI, 277 TDMA_CONS_INDEX, 278 RDMA_PROD_INDEX = TDMA_CONS_INDEX, 279 TDMA_PROD_INDEX, 280 RDMA_CONS_INDEX = TDMA_PROD_INDEX, 281 DMA_RING_BUF_SIZE, 282 DMA_START_ADDR, 283 DMA_START_ADDR_HI, 284 DMA_END_ADDR, 285 DMA_END_ADDR_HI, 286 DMA_MBUF_DONE_THRESH, 287 TDMA_FLOW_PERIOD, 288 RDMA_XON_XOFF_THRESH = TDMA_FLOW_PERIOD, 289 TDMA_WRITE_PTR, 290 RDMA_READ_PTR = TDMA_WRITE_PTR, 291 TDMA_WRITE_PTR_HI, 292 RDMA_READ_PTR_HI = TDMA_WRITE_PTR_HI 293 }; 294 295 /* GENET v4 supports 40-bits pointer addressing 296 * for obvious reasons the LO and HI word parts 297 * are contiguous, but this offsets the other 298 * registers. 299 */ 300 static const u8 genet_dma_ring_regs_v4[] = { 301 [TDMA_READ_PTR] = 0x00, 302 [TDMA_READ_PTR_HI] = 0x04, 303 [TDMA_CONS_INDEX] = 0x08, 304 [TDMA_PROD_INDEX] = 0x0C, 305 [DMA_RING_BUF_SIZE] = 0x10, 306 [DMA_START_ADDR] = 0x14, 307 [DMA_START_ADDR_HI] = 0x18, 308 [DMA_END_ADDR] = 0x1C, 309 [DMA_END_ADDR_HI] = 0x20, 310 [DMA_MBUF_DONE_THRESH] = 0x24, 311 [TDMA_FLOW_PERIOD] = 0x28, 312 [TDMA_WRITE_PTR] = 0x2C, 313 [TDMA_WRITE_PTR_HI] = 0x30, 314 }; 315 316 static const u8 genet_dma_ring_regs_v123[] = { 317 [TDMA_READ_PTR] = 0x00, 318 [TDMA_CONS_INDEX] = 0x04, 319 [TDMA_PROD_INDEX] = 0x08, 320 [DMA_RING_BUF_SIZE] = 0x0C, 321 [DMA_START_ADDR] = 0x10, 322 [DMA_END_ADDR] = 0x14, 323 [DMA_MBUF_DONE_THRESH] = 0x18, 324 [TDMA_FLOW_PERIOD] = 0x1C, 325 [TDMA_WRITE_PTR] = 0x20, 326 }; 327 328 /* Set at runtime once GENET version is known */ 329 static const u8 *genet_dma_ring_regs; 330 331 static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv, 332 unsigned int ring, 333 enum dma_ring_reg r) 334 { 335 return __raw_readl(priv->base + GENET_TDMA_REG_OFF + 336 (DMA_RING_SIZE * ring) + 337 genet_dma_ring_regs[r]); 338 } 339 340 static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv, 341 unsigned int ring, u32 val, 342 enum dma_ring_reg r) 343 { 344 __raw_writel(val, priv->base + GENET_TDMA_REG_OFF + 345 (DMA_RING_SIZE * ring) + 346 genet_dma_ring_regs[r]); 347 } 348 349 static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv, 350 unsigned int ring, 351 enum dma_ring_reg r) 352 { 353 return __raw_readl(priv->base + GENET_RDMA_REG_OFF + 354 (DMA_RING_SIZE * ring) + 355 genet_dma_ring_regs[r]); 356 } 357 358 static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv, 359 unsigned int ring, u32 val, 360 enum dma_ring_reg r) 361 { 362 __raw_writel(val, priv->base + GENET_RDMA_REG_OFF + 363 (DMA_RING_SIZE * ring) + 364 genet_dma_ring_regs[r]); 365 } 366 367 static int bcmgenet_get_settings(struct net_device *dev, 368 struct ethtool_cmd *cmd) 369 { 370 struct bcmgenet_priv *priv = netdev_priv(dev); 371 372 if (!netif_running(dev)) 373 return -EINVAL; 374 375 if (!priv->phydev) 376 return -ENODEV; 377 378 return phy_ethtool_gset(priv->phydev, cmd); 379 } 380 381 static int bcmgenet_set_settings(struct net_device *dev, 382 struct ethtool_cmd *cmd) 383 { 384 struct bcmgenet_priv *priv = netdev_priv(dev); 385 386 if (!netif_running(dev)) 387 return -EINVAL; 388 389 if (!priv->phydev) 390 return -ENODEV; 391 392 return phy_ethtool_sset(priv->phydev, cmd); 393 } 394 395 static int bcmgenet_set_rx_csum(struct net_device *dev, 396 netdev_features_t wanted) 397 { 398 struct bcmgenet_priv *priv = netdev_priv(dev); 399 u32 rbuf_chk_ctrl; 400 bool rx_csum_en; 401 402 rx_csum_en = !!(wanted & NETIF_F_RXCSUM); 403 404 rbuf_chk_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CHK_CTRL); 405 406 /* enable rx checksumming */ 407 if (rx_csum_en) 408 rbuf_chk_ctrl |= RBUF_RXCHK_EN; 409 else 410 rbuf_chk_ctrl &= ~RBUF_RXCHK_EN; 411 priv->desc_rxchk_en = rx_csum_en; 412 413 /* If UniMAC forwards CRC, we need to skip over it to get 414 * a valid CHK bit to be set in the per-packet status word 415 */ 416 if (rx_csum_en && priv->crc_fwd_en) 417 rbuf_chk_ctrl |= RBUF_SKIP_FCS; 418 else 419 rbuf_chk_ctrl &= ~RBUF_SKIP_FCS; 420 421 bcmgenet_rbuf_writel(priv, rbuf_chk_ctrl, RBUF_CHK_CTRL); 422 423 return 0; 424 } 425 426 static int bcmgenet_set_tx_csum(struct net_device *dev, 427 netdev_features_t wanted) 428 { 429 struct bcmgenet_priv *priv = netdev_priv(dev); 430 bool desc_64b_en; 431 u32 tbuf_ctrl, rbuf_ctrl; 432 433 tbuf_ctrl = bcmgenet_tbuf_ctrl_get(priv); 434 rbuf_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CTRL); 435 436 desc_64b_en = !!(wanted & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 437 438 /* enable 64 bytes descriptor in both directions (RBUF and TBUF) */ 439 if (desc_64b_en) { 440 tbuf_ctrl |= RBUF_64B_EN; 441 rbuf_ctrl |= RBUF_64B_EN; 442 } else { 443 tbuf_ctrl &= ~RBUF_64B_EN; 444 rbuf_ctrl &= ~RBUF_64B_EN; 445 } 446 priv->desc_64b_en = desc_64b_en; 447 448 bcmgenet_tbuf_ctrl_set(priv, tbuf_ctrl); 449 bcmgenet_rbuf_writel(priv, rbuf_ctrl, RBUF_CTRL); 450 451 return 0; 452 } 453 454 static int bcmgenet_set_features(struct net_device *dev, 455 netdev_features_t features) 456 { 457 netdev_features_t changed = features ^ dev->features; 458 netdev_features_t wanted = dev->wanted_features; 459 int ret = 0; 460 461 if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) 462 ret = bcmgenet_set_tx_csum(dev, wanted); 463 if (changed & (NETIF_F_RXCSUM)) 464 ret = bcmgenet_set_rx_csum(dev, wanted); 465 466 return ret; 467 } 468 469 static u32 bcmgenet_get_msglevel(struct net_device *dev) 470 { 471 struct bcmgenet_priv *priv = netdev_priv(dev); 472 473 return priv->msg_enable; 474 } 475 476 static void bcmgenet_set_msglevel(struct net_device *dev, u32 level) 477 { 478 struct bcmgenet_priv *priv = netdev_priv(dev); 479 480 priv->msg_enable = level; 481 } 482 483 /* standard ethtool support functions. */ 484 enum bcmgenet_stat_type { 485 BCMGENET_STAT_NETDEV = -1, 486 BCMGENET_STAT_MIB_RX, 487 BCMGENET_STAT_MIB_TX, 488 BCMGENET_STAT_RUNT, 489 BCMGENET_STAT_MISC, 490 BCMGENET_STAT_SOFT, 491 }; 492 493 struct bcmgenet_stats { 494 char stat_string[ETH_GSTRING_LEN]; 495 int stat_sizeof; 496 int stat_offset; 497 enum bcmgenet_stat_type type; 498 /* reg offset from UMAC base for misc counters */ 499 u16 reg_offset; 500 }; 501 502 #define STAT_NETDEV(m) { \ 503 .stat_string = __stringify(m), \ 504 .stat_sizeof = sizeof(((struct net_device_stats *)0)->m), \ 505 .stat_offset = offsetof(struct net_device_stats, m), \ 506 .type = BCMGENET_STAT_NETDEV, \ 507 } 508 509 #define STAT_GENET_MIB(str, m, _type) { \ 510 .stat_string = str, \ 511 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \ 512 .stat_offset = offsetof(struct bcmgenet_priv, m), \ 513 .type = _type, \ 514 } 515 516 #define STAT_GENET_MIB_RX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_RX) 517 #define STAT_GENET_MIB_TX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_TX) 518 #define STAT_GENET_RUNT(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_RUNT) 519 #define STAT_GENET_SOFT_MIB(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_SOFT) 520 521 #define STAT_GENET_MISC(str, m, offset) { \ 522 .stat_string = str, \ 523 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \ 524 .stat_offset = offsetof(struct bcmgenet_priv, m), \ 525 .type = BCMGENET_STAT_MISC, \ 526 .reg_offset = offset, \ 527 } 528 529 530 /* There is a 0xC gap between the end of RX and beginning of TX stats and then 531 * between the end of TX stats and the beginning of the RX RUNT 532 */ 533 #define BCMGENET_STAT_OFFSET 0xc 534 535 /* Hardware counters must be kept in sync because the order/offset 536 * is important here (order in structure declaration = order in hardware) 537 */ 538 static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = { 539 /* general stats */ 540 STAT_NETDEV(rx_packets), 541 STAT_NETDEV(tx_packets), 542 STAT_NETDEV(rx_bytes), 543 STAT_NETDEV(tx_bytes), 544 STAT_NETDEV(rx_errors), 545 STAT_NETDEV(tx_errors), 546 STAT_NETDEV(rx_dropped), 547 STAT_NETDEV(tx_dropped), 548 STAT_NETDEV(multicast), 549 /* UniMAC RSV counters */ 550 STAT_GENET_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64), 551 STAT_GENET_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127), 552 STAT_GENET_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255), 553 STAT_GENET_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511), 554 STAT_GENET_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023), 555 STAT_GENET_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518), 556 STAT_GENET_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv), 557 STAT_GENET_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047), 558 STAT_GENET_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095), 559 STAT_GENET_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216), 560 STAT_GENET_MIB_RX("rx_pkts", mib.rx.pkt), 561 STAT_GENET_MIB_RX("rx_bytes", mib.rx.bytes), 562 STAT_GENET_MIB_RX("rx_multicast", mib.rx.mca), 563 STAT_GENET_MIB_RX("rx_broadcast", mib.rx.bca), 564 STAT_GENET_MIB_RX("rx_fcs", mib.rx.fcs), 565 STAT_GENET_MIB_RX("rx_control", mib.rx.cf), 566 STAT_GENET_MIB_RX("rx_pause", mib.rx.pf), 567 STAT_GENET_MIB_RX("rx_unknown", mib.rx.uo), 568 STAT_GENET_MIB_RX("rx_align", mib.rx.aln), 569 STAT_GENET_MIB_RX("rx_outrange", mib.rx.flr), 570 STAT_GENET_MIB_RX("rx_code", mib.rx.cde), 571 STAT_GENET_MIB_RX("rx_carrier", mib.rx.fcr), 572 STAT_GENET_MIB_RX("rx_oversize", mib.rx.ovr), 573 STAT_GENET_MIB_RX("rx_jabber", mib.rx.jbr), 574 STAT_GENET_MIB_RX("rx_mtu_err", mib.rx.mtue), 575 STAT_GENET_MIB_RX("rx_good_pkts", mib.rx.pok), 576 STAT_GENET_MIB_RX("rx_unicast", mib.rx.uc), 577 STAT_GENET_MIB_RX("rx_ppp", mib.rx.ppp), 578 STAT_GENET_MIB_RX("rx_crc", mib.rx.rcrc), 579 /* UniMAC TSV counters */ 580 STAT_GENET_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64), 581 STAT_GENET_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127), 582 STAT_GENET_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255), 583 STAT_GENET_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511), 584 STAT_GENET_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023), 585 STAT_GENET_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518), 586 STAT_GENET_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv), 587 STAT_GENET_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047), 588 STAT_GENET_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095), 589 STAT_GENET_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216), 590 STAT_GENET_MIB_TX("tx_pkts", mib.tx.pkts), 591 STAT_GENET_MIB_TX("tx_multicast", mib.tx.mca), 592 STAT_GENET_MIB_TX("tx_broadcast", mib.tx.bca), 593 STAT_GENET_MIB_TX("tx_pause", mib.tx.pf), 594 STAT_GENET_MIB_TX("tx_control", mib.tx.cf), 595 STAT_GENET_MIB_TX("tx_fcs_err", mib.tx.fcs), 596 STAT_GENET_MIB_TX("tx_oversize", mib.tx.ovr), 597 STAT_GENET_MIB_TX("tx_defer", mib.tx.drf), 598 STAT_GENET_MIB_TX("tx_excess_defer", mib.tx.edf), 599 STAT_GENET_MIB_TX("tx_single_col", mib.tx.scl), 600 STAT_GENET_MIB_TX("tx_multi_col", mib.tx.mcl), 601 STAT_GENET_MIB_TX("tx_late_col", mib.tx.lcl), 602 STAT_GENET_MIB_TX("tx_excess_col", mib.tx.ecl), 603 STAT_GENET_MIB_TX("tx_frags", mib.tx.frg), 604 STAT_GENET_MIB_TX("tx_total_col", mib.tx.ncl), 605 STAT_GENET_MIB_TX("tx_jabber", mib.tx.jbr), 606 STAT_GENET_MIB_TX("tx_bytes", mib.tx.bytes), 607 STAT_GENET_MIB_TX("tx_good_pkts", mib.tx.pok), 608 STAT_GENET_MIB_TX("tx_unicast", mib.tx.uc), 609 /* UniMAC RUNT counters */ 610 STAT_GENET_RUNT("rx_runt_pkts", mib.rx_runt_cnt), 611 STAT_GENET_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs), 612 STAT_GENET_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align), 613 STAT_GENET_RUNT("rx_runt_bytes", mib.rx_runt_bytes), 614 /* Misc UniMAC counters */ 615 STAT_GENET_MISC("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt, 616 UMAC_RBUF_OVFL_CNT), 617 STAT_GENET_MISC("rbuf_err_cnt", mib.rbuf_err_cnt, UMAC_RBUF_ERR_CNT), 618 STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT), 619 STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed), 620 STAT_GENET_SOFT_MIB("rx_dma_failed", mib.rx_dma_failed), 621 STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed), 622 }; 623 624 #define BCMGENET_STATS_LEN ARRAY_SIZE(bcmgenet_gstrings_stats) 625 626 static void bcmgenet_get_drvinfo(struct net_device *dev, 627 struct ethtool_drvinfo *info) 628 { 629 strlcpy(info->driver, "bcmgenet", sizeof(info->driver)); 630 strlcpy(info->version, "v2.0", sizeof(info->version)); 631 info->n_stats = BCMGENET_STATS_LEN; 632 } 633 634 static int bcmgenet_get_sset_count(struct net_device *dev, int string_set) 635 { 636 switch (string_set) { 637 case ETH_SS_STATS: 638 return BCMGENET_STATS_LEN; 639 default: 640 return -EOPNOTSUPP; 641 } 642 } 643 644 static void bcmgenet_get_strings(struct net_device *dev, u32 stringset, 645 u8 *data) 646 { 647 int i; 648 649 switch (stringset) { 650 case ETH_SS_STATS: 651 for (i = 0; i < BCMGENET_STATS_LEN; i++) { 652 memcpy(data + i * ETH_GSTRING_LEN, 653 bcmgenet_gstrings_stats[i].stat_string, 654 ETH_GSTRING_LEN); 655 } 656 break; 657 } 658 } 659 660 static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv) 661 { 662 int i, j = 0; 663 664 for (i = 0; i < BCMGENET_STATS_LEN; i++) { 665 const struct bcmgenet_stats *s; 666 u8 offset = 0; 667 u32 val = 0; 668 char *p; 669 670 s = &bcmgenet_gstrings_stats[i]; 671 switch (s->type) { 672 case BCMGENET_STAT_NETDEV: 673 case BCMGENET_STAT_SOFT: 674 continue; 675 case BCMGENET_STAT_MIB_RX: 676 case BCMGENET_STAT_MIB_TX: 677 case BCMGENET_STAT_RUNT: 678 if (s->type != BCMGENET_STAT_MIB_RX) 679 offset = BCMGENET_STAT_OFFSET; 680 val = bcmgenet_umac_readl(priv, 681 UMAC_MIB_START + j + offset); 682 break; 683 case BCMGENET_STAT_MISC: 684 val = bcmgenet_umac_readl(priv, s->reg_offset); 685 /* clear if overflowed */ 686 if (val == ~0) 687 bcmgenet_umac_writel(priv, 0, s->reg_offset); 688 break; 689 } 690 691 j += s->stat_sizeof; 692 p = (char *)priv + s->stat_offset; 693 *(u32 *)p = val; 694 } 695 } 696 697 static void bcmgenet_get_ethtool_stats(struct net_device *dev, 698 struct ethtool_stats *stats, 699 u64 *data) 700 { 701 struct bcmgenet_priv *priv = netdev_priv(dev); 702 int i; 703 704 if (netif_running(dev)) 705 bcmgenet_update_mib_counters(priv); 706 707 for (i = 0; i < BCMGENET_STATS_LEN; i++) { 708 const struct bcmgenet_stats *s; 709 char *p; 710 711 s = &bcmgenet_gstrings_stats[i]; 712 if (s->type == BCMGENET_STAT_NETDEV) 713 p = (char *)&dev->stats; 714 else 715 p = (char *)priv; 716 p += s->stat_offset; 717 data[i] = *(u32 *)p; 718 } 719 } 720 721 static void bcmgenet_eee_enable_set(struct net_device *dev, bool enable) 722 { 723 struct bcmgenet_priv *priv = netdev_priv(dev); 724 u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL; 725 u32 reg; 726 727 if (enable && !priv->clk_eee_enabled) { 728 clk_prepare_enable(priv->clk_eee); 729 priv->clk_eee_enabled = true; 730 } 731 732 reg = bcmgenet_umac_readl(priv, UMAC_EEE_CTRL); 733 if (enable) 734 reg |= EEE_EN; 735 else 736 reg &= ~EEE_EN; 737 bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL); 738 739 /* Enable EEE and switch to a 27Mhz clock automatically */ 740 reg = __raw_readl(priv->base + off); 741 if (enable) 742 reg |= TBUF_EEE_EN | TBUF_PM_EN; 743 else 744 reg &= ~(TBUF_EEE_EN | TBUF_PM_EN); 745 __raw_writel(reg, priv->base + off); 746 747 /* Do the same for thing for RBUF */ 748 reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL); 749 if (enable) 750 reg |= RBUF_EEE_EN | RBUF_PM_EN; 751 else 752 reg &= ~(RBUF_EEE_EN | RBUF_PM_EN); 753 bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL); 754 755 if (!enable && priv->clk_eee_enabled) { 756 clk_disable_unprepare(priv->clk_eee); 757 priv->clk_eee_enabled = false; 758 } 759 760 priv->eee.eee_enabled = enable; 761 priv->eee.eee_active = enable; 762 } 763 764 static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e) 765 { 766 struct bcmgenet_priv *priv = netdev_priv(dev); 767 struct ethtool_eee *p = &priv->eee; 768 769 if (GENET_IS_V1(priv)) 770 return -EOPNOTSUPP; 771 772 e->eee_enabled = p->eee_enabled; 773 e->eee_active = p->eee_active; 774 e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER); 775 776 return phy_ethtool_get_eee(priv->phydev, e); 777 } 778 779 static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e) 780 { 781 struct bcmgenet_priv *priv = netdev_priv(dev); 782 struct ethtool_eee *p = &priv->eee; 783 int ret = 0; 784 785 if (GENET_IS_V1(priv)) 786 return -EOPNOTSUPP; 787 788 p->eee_enabled = e->eee_enabled; 789 790 if (!p->eee_enabled) { 791 bcmgenet_eee_enable_set(dev, false); 792 } else { 793 ret = phy_init_eee(priv->phydev, 0); 794 if (ret) { 795 netif_err(priv, hw, dev, "EEE initialization failed\n"); 796 return ret; 797 } 798 799 bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER); 800 bcmgenet_eee_enable_set(dev, true); 801 } 802 803 return phy_ethtool_set_eee(priv->phydev, e); 804 } 805 806 static int bcmgenet_nway_reset(struct net_device *dev) 807 { 808 struct bcmgenet_priv *priv = netdev_priv(dev); 809 810 return genphy_restart_aneg(priv->phydev); 811 } 812 813 /* standard ethtool support functions. */ 814 static struct ethtool_ops bcmgenet_ethtool_ops = { 815 .get_strings = bcmgenet_get_strings, 816 .get_sset_count = bcmgenet_get_sset_count, 817 .get_ethtool_stats = bcmgenet_get_ethtool_stats, 818 .get_settings = bcmgenet_get_settings, 819 .set_settings = bcmgenet_set_settings, 820 .get_drvinfo = bcmgenet_get_drvinfo, 821 .get_link = ethtool_op_get_link, 822 .get_msglevel = bcmgenet_get_msglevel, 823 .set_msglevel = bcmgenet_set_msglevel, 824 .get_wol = bcmgenet_get_wol, 825 .set_wol = bcmgenet_set_wol, 826 .get_eee = bcmgenet_get_eee, 827 .set_eee = bcmgenet_set_eee, 828 .nway_reset = bcmgenet_nway_reset, 829 }; 830 831 /* Power down the unimac, based on mode. */ 832 static void bcmgenet_power_down(struct bcmgenet_priv *priv, 833 enum bcmgenet_power_mode mode) 834 { 835 u32 reg; 836 837 switch (mode) { 838 case GENET_POWER_CABLE_SENSE: 839 phy_detach(priv->phydev); 840 break; 841 842 case GENET_POWER_WOL_MAGIC: 843 bcmgenet_wol_power_down_cfg(priv, mode); 844 break; 845 846 case GENET_POWER_PASSIVE: 847 /* Power down LED */ 848 if (priv->hw_params->flags & GENET_HAS_EXT) { 849 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); 850 reg |= (EXT_PWR_DOWN_PHY | 851 EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS); 852 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT); 853 } 854 break; 855 default: 856 break; 857 } 858 } 859 860 static void bcmgenet_power_up(struct bcmgenet_priv *priv, 861 enum bcmgenet_power_mode mode) 862 { 863 u32 reg; 864 865 if (!(priv->hw_params->flags & GENET_HAS_EXT)) 866 return; 867 868 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); 869 870 switch (mode) { 871 case GENET_POWER_PASSIVE: 872 reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_PHY | 873 EXT_PWR_DOWN_BIAS); 874 /* fallthrough */ 875 case GENET_POWER_CABLE_SENSE: 876 /* enable APD */ 877 reg |= EXT_PWR_DN_EN_LD; 878 break; 879 case GENET_POWER_WOL_MAGIC: 880 bcmgenet_wol_power_up_cfg(priv, mode); 881 return; 882 default: 883 break; 884 } 885 886 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT); 887 888 if (mode == GENET_POWER_PASSIVE) 889 bcmgenet_mii_reset(priv->dev); 890 } 891 892 /* ioctl handle special commands that are not present in ethtool. */ 893 static int bcmgenet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 894 { 895 struct bcmgenet_priv *priv = netdev_priv(dev); 896 int val = 0; 897 898 if (!netif_running(dev)) 899 return -EINVAL; 900 901 switch (cmd) { 902 case SIOCGMIIPHY: 903 case SIOCGMIIREG: 904 case SIOCSMIIREG: 905 if (!priv->phydev) 906 val = -ENODEV; 907 else 908 val = phy_mii_ioctl(priv->phydev, rq, cmd); 909 break; 910 911 default: 912 val = -EINVAL; 913 break; 914 } 915 916 return val; 917 } 918 919 static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv, 920 struct bcmgenet_tx_ring *ring) 921 { 922 struct enet_cb *tx_cb_ptr; 923 924 tx_cb_ptr = ring->cbs; 925 tx_cb_ptr += ring->write_ptr - ring->cb_ptr; 926 tx_cb_ptr->bd_addr = priv->tx_bds + ring->write_ptr * DMA_DESC_SIZE; 927 /* Advancing local write pointer */ 928 if (ring->write_ptr == ring->end_ptr) 929 ring->write_ptr = ring->cb_ptr; 930 else 931 ring->write_ptr++; 932 933 return tx_cb_ptr; 934 } 935 936 /* Simple helper to free a control block's resources */ 937 static void bcmgenet_free_cb(struct enet_cb *cb) 938 { 939 dev_kfree_skb_any(cb->skb); 940 cb->skb = NULL; 941 dma_unmap_addr_set(cb, dma_addr, 0); 942 } 943 944 static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_priv *priv, 945 struct bcmgenet_tx_ring *ring) 946 { 947 bcmgenet_intrl2_0_writel(priv, 948 UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE, 949 INTRL2_CPU_MASK_SET); 950 } 951 952 static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_priv *priv, 953 struct bcmgenet_tx_ring *ring) 954 { 955 bcmgenet_intrl2_0_writel(priv, 956 UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE, 957 INTRL2_CPU_MASK_CLEAR); 958 } 959 960 static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_priv *priv, 961 struct bcmgenet_tx_ring *ring) 962 { 963 bcmgenet_intrl2_1_writel(priv, (1 << ring->index), 964 INTRL2_CPU_MASK_CLEAR); 965 priv->int1_mask &= ~(1 << ring->index); 966 } 967 968 static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_priv *priv, 969 struct bcmgenet_tx_ring *ring) 970 { 971 bcmgenet_intrl2_1_writel(priv, (1 << ring->index), 972 INTRL2_CPU_MASK_SET); 973 priv->int1_mask |= (1 << ring->index); 974 } 975 976 /* Unlocked version of the reclaim routine */ 977 static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev, 978 struct bcmgenet_tx_ring *ring) 979 { 980 struct bcmgenet_priv *priv = netdev_priv(dev); 981 int last_tx_cn, last_c_index, num_tx_bds; 982 struct enet_cb *tx_cb_ptr; 983 struct netdev_queue *txq; 984 unsigned int pkts_compl = 0; 985 unsigned int bds_compl; 986 unsigned int c_index; 987 988 /* Compute how many buffers are transmitted since last xmit call */ 989 c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX); 990 txq = netdev_get_tx_queue(dev, ring->queue); 991 992 last_c_index = ring->c_index; 993 num_tx_bds = ring->size; 994 995 c_index &= (num_tx_bds - 1); 996 997 if (c_index >= last_c_index) 998 last_tx_cn = c_index - last_c_index; 999 else 1000 last_tx_cn = num_tx_bds - last_c_index + c_index; 1001 1002 netif_dbg(priv, tx_done, dev, 1003 "%s ring=%d index=%d last_tx_cn=%d last_index=%d\n", 1004 __func__, ring->index, 1005 c_index, last_tx_cn, last_c_index); 1006 1007 /* Reclaim transmitted buffers */ 1008 while (last_tx_cn-- > 0) { 1009 tx_cb_ptr = ring->cbs + last_c_index; 1010 bds_compl = 0; 1011 if (tx_cb_ptr->skb) { 1012 pkts_compl++; 1013 bds_compl = skb_shinfo(tx_cb_ptr->skb)->nr_frags + 1; 1014 dev->stats.tx_bytes += tx_cb_ptr->skb->len; 1015 dma_unmap_single(&dev->dev, 1016 dma_unmap_addr(tx_cb_ptr, dma_addr), 1017 tx_cb_ptr->skb->len, 1018 DMA_TO_DEVICE); 1019 bcmgenet_free_cb(tx_cb_ptr); 1020 } else if (dma_unmap_addr(tx_cb_ptr, dma_addr)) { 1021 dev->stats.tx_bytes += 1022 dma_unmap_len(tx_cb_ptr, dma_len); 1023 dma_unmap_page(&dev->dev, 1024 dma_unmap_addr(tx_cb_ptr, dma_addr), 1025 dma_unmap_len(tx_cb_ptr, dma_len), 1026 DMA_TO_DEVICE); 1027 dma_unmap_addr_set(tx_cb_ptr, dma_addr, 0); 1028 } 1029 dev->stats.tx_packets++; 1030 ring->free_bds += bds_compl; 1031 1032 last_c_index++; 1033 last_c_index &= (num_tx_bds - 1); 1034 } 1035 1036 if (ring->free_bds > (MAX_SKB_FRAGS + 1)) { 1037 if (netif_tx_queue_stopped(txq)) 1038 netif_tx_wake_queue(txq); 1039 } 1040 1041 ring->c_index = c_index; 1042 1043 return pkts_compl; 1044 } 1045 1046 static unsigned int bcmgenet_tx_reclaim(struct net_device *dev, 1047 struct bcmgenet_tx_ring *ring) 1048 { 1049 unsigned int released; 1050 unsigned long flags; 1051 1052 spin_lock_irqsave(&ring->lock, flags); 1053 released = __bcmgenet_tx_reclaim(dev, ring); 1054 spin_unlock_irqrestore(&ring->lock, flags); 1055 1056 return released; 1057 } 1058 1059 static int bcmgenet_tx_poll(struct napi_struct *napi, int budget) 1060 { 1061 struct bcmgenet_tx_ring *ring = 1062 container_of(napi, struct bcmgenet_tx_ring, napi); 1063 unsigned int work_done = 0; 1064 1065 work_done = bcmgenet_tx_reclaim(ring->priv->dev, ring); 1066 1067 if (work_done == 0) { 1068 napi_complete(napi); 1069 ring->int_enable(ring->priv, ring); 1070 1071 return 0; 1072 } 1073 1074 return budget; 1075 } 1076 1077 static void bcmgenet_tx_reclaim_all(struct net_device *dev) 1078 { 1079 struct bcmgenet_priv *priv = netdev_priv(dev); 1080 int i; 1081 1082 if (netif_is_multiqueue(dev)) { 1083 for (i = 0; i < priv->hw_params->tx_queues; i++) 1084 bcmgenet_tx_reclaim(dev, &priv->tx_rings[i]); 1085 } 1086 1087 bcmgenet_tx_reclaim(dev, &priv->tx_rings[DESC_INDEX]); 1088 } 1089 1090 /* Transmits a single SKB (either head of a fragment or a single SKB) 1091 * caller must hold priv->lock 1092 */ 1093 static int bcmgenet_xmit_single(struct net_device *dev, 1094 struct sk_buff *skb, 1095 u16 dma_desc_flags, 1096 struct bcmgenet_tx_ring *ring) 1097 { 1098 struct bcmgenet_priv *priv = netdev_priv(dev); 1099 struct device *kdev = &priv->pdev->dev; 1100 struct enet_cb *tx_cb_ptr; 1101 unsigned int skb_len; 1102 dma_addr_t mapping; 1103 u32 length_status; 1104 int ret; 1105 1106 tx_cb_ptr = bcmgenet_get_txcb(priv, ring); 1107 1108 if (unlikely(!tx_cb_ptr)) 1109 BUG(); 1110 1111 tx_cb_ptr->skb = skb; 1112 1113 skb_len = skb_headlen(skb) < ETH_ZLEN ? ETH_ZLEN : skb_headlen(skb); 1114 1115 mapping = dma_map_single(kdev, skb->data, skb_len, DMA_TO_DEVICE); 1116 ret = dma_mapping_error(kdev, mapping); 1117 if (ret) { 1118 priv->mib.tx_dma_failed++; 1119 netif_err(priv, tx_err, dev, "Tx DMA map failed\n"); 1120 dev_kfree_skb(skb); 1121 return ret; 1122 } 1123 1124 dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping); 1125 dma_unmap_len_set(tx_cb_ptr, dma_len, skb->len); 1126 length_status = (skb_len << DMA_BUFLENGTH_SHIFT) | dma_desc_flags | 1127 (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT) | 1128 DMA_TX_APPEND_CRC; 1129 1130 if (skb->ip_summed == CHECKSUM_PARTIAL) 1131 length_status |= DMA_TX_DO_CSUM; 1132 1133 dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, length_status); 1134 1135 /* Decrement total BD count and advance our write pointer */ 1136 ring->free_bds -= 1; 1137 ring->prod_index += 1; 1138 ring->prod_index &= DMA_P_INDEX_MASK; 1139 1140 return 0; 1141 } 1142 1143 /* Transmit a SKB fragment */ 1144 static int bcmgenet_xmit_frag(struct net_device *dev, 1145 skb_frag_t *frag, 1146 u16 dma_desc_flags, 1147 struct bcmgenet_tx_ring *ring) 1148 { 1149 struct bcmgenet_priv *priv = netdev_priv(dev); 1150 struct device *kdev = &priv->pdev->dev; 1151 struct enet_cb *tx_cb_ptr; 1152 dma_addr_t mapping; 1153 int ret; 1154 1155 tx_cb_ptr = bcmgenet_get_txcb(priv, ring); 1156 1157 if (unlikely(!tx_cb_ptr)) 1158 BUG(); 1159 tx_cb_ptr->skb = NULL; 1160 1161 mapping = skb_frag_dma_map(kdev, frag, 0, 1162 skb_frag_size(frag), DMA_TO_DEVICE); 1163 ret = dma_mapping_error(kdev, mapping); 1164 if (ret) { 1165 priv->mib.tx_dma_failed++; 1166 netif_err(priv, tx_err, dev, "%s: Tx DMA map failed\n", 1167 __func__); 1168 return ret; 1169 } 1170 1171 dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping); 1172 dma_unmap_len_set(tx_cb_ptr, dma_len, frag->size); 1173 1174 dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, 1175 (frag->size << DMA_BUFLENGTH_SHIFT) | dma_desc_flags | 1176 (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT)); 1177 1178 1179 ring->free_bds -= 1; 1180 ring->prod_index += 1; 1181 ring->prod_index &= DMA_P_INDEX_MASK; 1182 1183 return 0; 1184 } 1185 1186 /* Reallocate the SKB to put enough headroom in front of it and insert 1187 * the transmit checksum offsets in the descriptors 1188 */ 1189 static struct sk_buff *bcmgenet_put_tx_csum(struct net_device *dev, 1190 struct sk_buff *skb) 1191 { 1192 struct status_64 *status = NULL; 1193 struct sk_buff *new_skb; 1194 u16 offset; 1195 u8 ip_proto; 1196 u16 ip_ver; 1197 u32 tx_csum_info; 1198 1199 if (unlikely(skb_headroom(skb) < sizeof(*status))) { 1200 /* If 64 byte status block enabled, must make sure skb has 1201 * enough headroom for us to insert 64B status block. 1202 */ 1203 new_skb = skb_realloc_headroom(skb, sizeof(*status)); 1204 dev_kfree_skb(skb); 1205 if (!new_skb) { 1206 dev->stats.tx_errors++; 1207 dev->stats.tx_dropped++; 1208 return NULL; 1209 } 1210 skb = new_skb; 1211 } 1212 1213 skb_push(skb, sizeof(*status)); 1214 status = (struct status_64 *)skb->data; 1215 1216 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1217 ip_ver = htons(skb->protocol); 1218 switch (ip_ver) { 1219 case ETH_P_IP: 1220 ip_proto = ip_hdr(skb)->protocol; 1221 break; 1222 case ETH_P_IPV6: 1223 ip_proto = ipv6_hdr(skb)->nexthdr; 1224 break; 1225 default: 1226 return skb; 1227 } 1228 1229 offset = skb_checksum_start_offset(skb) - sizeof(*status); 1230 tx_csum_info = (offset << STATUS_TX_CSUM_START_SHIFT) | 1231 (offset + skb->csum_offset); 1232 1233 /* Set the length valid bit for TCP and UDP and just set 1234 * the special UDP flag for IPv4, else just set to 0. 1235 */ 1236 if (ip_proto == IPPROTO_TCP || ip_proto == IPPROTO_UDP) { 1237 tx_csum_info |= STATUS_TX_CSUM_LV; 1238 if (ip_proto == IPPROTO_UDP && ip_ver == ETH_P_IP) 1239 tx_csum_info |= STATUS_TX_CSUM_PROTO_UDP; 1240 } else { 1241 tx_csum_info = 0; 1242 } 1243 1244 status->tx_csum_info = tx_csum_info; 1245 } 1246 1247 return skb; 1248 } 1249 1250 static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev) 1251 { 1252 struct bcmgenet_priv *priv = netdev_priv(dev); 1253 struct bcmgenet_tx_ring *ring = NULL; 1254 struct netdev_queue *txq; 1255 unsigned long flags = 0; 1256 int nr_frags, index; 1257 u16 dma_desc_flags; 1258 int ret; 1259 int i; 1260 1261 index = skb_get_queue_mapping(skb); 1262 /* Mapping strategy: 1263 * queue_mapping = 0, unclassified, packet xmited through ring16 1264 * queue_mapping = 1, goes to ring 0. (highest priority queue 1265 * queue_mapping = 2, goes to ring 1. 1266 * queue_mapping = 3, goes to ring 2. 1267 * queue_mapping = 4, goes to ring 3. 1268 */ 1269 if (index == 0) 1270 index = DESC_INDEX; 1271 else 1272 index -= 1; 1273 1274 nr_frags = skb_shinfo(skb)->nr_frags; 1275 ring = &priv->tx_rings[index]; 1276 txq = netdev_get_tx_queue(dev, ring->queue); 1277 1278 spin_lock_irqsave(&ring->lock, flags); 1279 if (ring->free_bds <= nr_frags + 1) { 1280 netif_tx_stop_queue(txq); 1281 netdev_err(dev, "%s: tx ring %d full when queue %d awake\n", 1282 __func__, index, ring->queue); 1283 ret = NETDEV_TX_BUSY; 1284 goto out; 1285 } 1286 1287 if (skb_padto(skb, ETH_ZLEN)) { 1288 ret = NETDEV_TX_OK; 1289 goto out; 1290 } 1291 1292 /* set the SKB transmit checksum */ 1293 if (priv->desc_64b_en) { 1294 skb = bcmgenet_put_tx_csum(dev, skb); 1295 if (!skb) { 1296 ret = NETDEV_TX_OK; 1297 goto out; 1298 } 1299 } 1300 1301 dma_desc_flags = DMA_SOP; 1302 if (nr_frags == 0) 1303 dma_desc_flags |= DMA_EOP; 1304 1305 /* Transmit single SKB or head of fragment list */ 1306 ret = bcmgenet_xmit_single(dev, skb, dma_desc_flags, ring); 1307 if (ret) { 1308 ret = NETDEV_TX_OK; 1309 goto out; 1310 } 1311 1312 /* xmit fragment */ 1313 for (i = 0; i < nr_frags; i++) { 1314 ret = bcmgenet_xmit_frag(dev, 1315 &skb_shinfo(skb)->frags[i], 1316 (i == nr_frags - 1) ? DMA_EOP : 0, 1317 ring); 1318 if (ret) { 1319 ret = NETDEV_TX_OK; 1320 goto out; 1321 } 1322 } 1323 1324 skb_tx_timestamp(skb); 1325 1326 /* we kept a software copy of how much we should advance the TDMA 1327 * producer index, now write it down to the hardware 1328 */ 1329 bcmgenet_tdma_ring_writel(priv, ring->index, 1330 ring->prod_index, TDMA_PROD_INDEX); 1331 1332 if (ring->free_bds <= (MAX_SKB_FRAGS + 1)) 1333 netif_tx_stop_queue(txq); 1334 1335 out: 1336 spin_unlock_irqrestore(&ring->lock, flags); 1337 1338 return ret; 1339 } 1340 1341 1342 static int bcmgenet_rx_refill(struct bcmgenet_priv *priv, struct enet_cb *cb) 1343 { 1344 struct device *kdev = &priv->pdev->dev; 1345 struct sk_buff *skb; 1346 dma_addr_t mapping; 1347 int ret; 1348 1349 skb = netdev_alloc_skb(priv->dev, priv->rx_buf_len + SKB_ALIGNMENT); 1350 if (!skb) 1351 return -ENOMEM; 1352 1353 /* a caller did not release this control block */ 1354 WARN_ON(cb->skb != NULL); 1355 cb->skb = skb; 1356 mapping = dma_map_single(kdev, skb->data, 1357 priv->rx_buf_len, DMA_FROM_DEVICE); 1358 ret = dma_mapping_error(kdev, mapping); 1359 if (ret) { 1360 priv->mib.rx_dma_failed++; 1361 bcmgenet_free_cb(cb); 1362 netif_err(priv, rx_err, priv->dev, 1363 "%s DMA map failed\n", __func__); 1364 return ret; 1365 } 1366 1367 dma_unmap_addr_set(cb, dma_addr, mapping); 1368 /* assign packet, prepare descriptor, and advance pointer */ 1369 1370 dmadesc_set_addr(priv, priv->rx_bd_assign_ptr, mapping); 1371 1372 /* turn on the newly assigned BD for DMA to use */ 1373 priv->rx_bd_assign_index++; 1374 priv->rx_bd_assign_index &= (priv->num_rx_bds - 1); 1375 1376 priv->rx_bd_assign_ptr = priv->rx_bds + 1377 (priv->rx_bd_assign_index * DMA_DESC_SIZE); 1378 1379 return 0; 1380 } 1381 1382 /* bcmgenet_desc_rx - descriptor based rx process. 1383 * this could be called from bottom half, or from NAPI polling method. 1384 */ 1385 static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv, 1386 unsigned int budget) 1387 { 1388 struct net_device *dev = priv->dev; 1389 struct enet_cb *cb; 1390 struct sk_buff *skb; 1391 u32 dma_length_status; 1392 unsigned long dma_flag; 1393 int len, err; 1394 unsigned int rxpktprocessed = 0, rxpkttoprocess; 1395 unsigned int p_index; 1396 unsigned int chksum_ok = 0; 1397 1398 p_index = bcmgenet_rdma_ring_readl(priv, DESC_INDEX, RDMA_PROD_INDEX); 1399 p_index &= DMA_P_INDEX_MASK; 1400 1401 if (p_index < priv->rx_c_index) 1402 rxpkttoprocess = (DMA_C_INDEX_MASK + 1) - 1403 priv->rx_c_index + p_index; 1404 else 1405 rxpkttoprocess = p_index - priv->rx_c_index; 1406 1407 netif_dbg(priv, rx_status, dev, 1408 "RDMA: rxpkttoprocess=%d\n", rxpkttoprocess); 1409 1410 while ((rxpktprocessed < rxpkttoprocess) && 1411 (rxpktprocessed < budget)) { 1412 cb = &priv->rx_cbs[priv->rx_read_ptr]; 1413 skb = cb->skb; 1414 1415 /* We do not have a backing SKB, so we do not have a 1416 * corresponding DMA mapping for this incoming packet since 1417 * bcmgenet_rx_refill always either has both skb and mapping or 1418 * none. 1419 */ 1420 if (unlikely(!skb)) { 1421 dev->stats.rx_dropped++; 1422 dev->stats.rx_errors++; 1423 goto refill; 1424 } 1425 1426 /* Unmap the packet contents such that we can use the 1427 * RSV from the 64 bytes descriptor when enabled and save 1428 * a 32-bits register read 1429 */ 1430 dma_unmap_single(&dev->dev, dma_unmap_addr(cb, dma_addr), 1431 priv->rx_buf_len, DMA_FROM_DEVICE); 1432 1433 if (!priv->desc_64b_en) { 1434 dma_length_status = 1435 dmadesc_get_length_status(priv, 1436 priv->rx_bds + 1437 (priv->rx_read_ptr * 1438 DMA_DESC_SIZE)); 1439 } else { 1440 struct status_64 *status; 1441 1442 status = (struct status_64 *)skb->data; 1443 dma_length_status = status->length_status; 1444 } 1445 1446 /* DMA flags and length are still valid no matter how 1447 * we got the Receive Status Vector (64B RSB or register) 1448 */ 1449 dma_flag = dma_length_status & 0xffff; 1450 len = dma_length_status >> DMA_BUFLENGTH_SHIFT; 1451 1452 netif_dbg(priv, rx_status, dev, 1453 "%s:p_ind=%d c_ind=%d read_ptr=%d len_stat=0x%08x\n", 1454 __func__, p_index, priv->rx_c_index, 1455 priv->rx_read_ptr, dma_length_status); 1456 1457 if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) { 1458 netif_err(priv, rx_status, dev, 1459 "dropping fragmented packet!\n"); 1460 dev->stats.rx_dropped++; 1461 dev->stats.rx_errors++; 1462 dev_kfree_skb_any(cb->skb); 1463 cb->skb = NULL; 1464 goto refill; 1465 } 1466 /* report errors */ 1467 if (unlikely(dma_flag & (DMA_RX_CRC_ERROR | 1468 DMA_RX_OV | 1469 DMA_RX_NO | 1470 DMA_RX_LG | 1471 DMA_RX_RXER))) { 1472 netif_err(priv, rx_status, dev, "dma_flag=0x%x\n", 1473 (unsigned int)dma_flag); 1474 if (dma_flag & DMA_RX_CRC_ERROR) 1475 dev->stats.rx_crc_errors++; 1476 if (dma_flag & DMA_RX_OV) 1477 dev->stats.rx_over_errors++; 1478 if (dma_flag & DMA_RX_NO) 1479 dev->stats.rx_frame_errors++; 1480 if (dma_flag & DMA_RX_LG) 1481 dev->stats.rx_length_errors++; 1482 dev->stats.rx_dropped++; 1483 dev->stats.rx_errors++; 1484 1485 /* discard the packet and advance consumer index.*/ 1486 dev_kfree_skb_any(cb->skb); 1487 cb->skb = NULL; 1488 goto refill; 1489 } /* error packet */ 1490 1491 chksum_ok = (dma_flag & priv->dma_rx_chk_bit) && 1492 priv->desc_rxchk_en; 1493 1494 skb_put(skb, len); 1495 if (priv->desc_64b_en) { 1496 skb_pull(skb, 64); 1497 len -= 64; 1498 } 1499 1500 if (likely(chksum_ok)) 1501 skb->ip_summed = CHECKSUM_UNNECESSARY; 1502 1503 /* remove hardware 2bytes added for IP alignment */ 1504 skb_pull(skb, 2); 1505 len -= 2; 1506 1507 if (priv->crc_fwd_en) { 1508 skb_trim(skb, len - ETH_FCS_LEN); 1509 len -= ETH_FCS_LEN; 1510 } 1511 1512 /*Finish setting up the received SKB and send it to the kernel*/ 1513 skb->protocol = eth_type_trans(skb, priv->dev); 1514 dev->stats.rx_packets++; 1515 dev->stats.rx_bytes += len; 1516 if (dma_flag & DMA_RX_MULT) 1517 dev->stats.multicast++; 1518 1519 /* Notify kernel */ 1520 napi_gro_receive(&priv->napi, skb); 1521 cb->skb = NULL; 1522 netif_dbg(priv, rx_status, dev, "pushed up to kernel\n"); 1523 1524 /* refill RX path on the current control block */ 1525 refill: 1526 err = bcmgenet_rx_refill(priv, cb); 1527 if (err) { 1528 priv->mib.alloc_rx_buff_failed++; 1529 netif_err(priv, rx_err, dev, "Rx refill failed\n"); 1530 } 1531 1532 rxpktprocessed++; 1533 priv->rx_read_ptr++; 1534 priv->rx_read_ptr &= (priv->num_rx_bds - 1); 1535 } 1536 1537 return rxpktprocessed; 1538 } 1539 1540 /* Assign skb to RX DMA descriptor. */ 1541 static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv) 1542 { 1543 struct enet_cb *cb; 1544 int ret = 0; 1545 int i; 1546 1547 netif_dbg(priv, hw, priv->dev, "%s:\n", __func__); 1548 1549 /* loop here for each buffer needing assign */ 1550 for (i = 0; i < priv->num_rx_bds; i++) { 1551 cb = &priv->rx_cbs[priv->rx_bd_assign_index]; 1552 if (cb->skb) 1553 continue; 1554 1555 ret = bcmgenet_rx_refill(priv, cb); 1556 if (ret) 1557 break; 1558 } 1559 1560 return ret; 1561 } 1562 1563 static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv) 1564 { 1565 struct enet_cb *cb; 1566 int i; 1567 1568 for (i = 0; i < priv->num_rx_bds; i++) { 1569 cb = &priv->rx_cbs[i]; 1570 1571 if (dma_unmap_addr(cb, dma_addr)) { 1572 dma_unmap_single(&priv->dev->dev, 1573 dma_unmap_addr(cb, dma_addr), 1574 priv->rx_buf_len, DMA_FROM_DEVICE); 1575 dma_unmap_addr_set(cb, dma_addr, 0); 1576 } 1577 1578 if (cb->skb) 1579 bcmgenet_free_cb(cb); 1580 } 1581 } 1582 1583 static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable) 1584 { 1585 u32 reg; 1586 1587 reg = bcmgenet_umac_readl(priv, UMAC_CMD); 1588 if (enable) 1589 reg |= mask; 1590 else 1591 reg &= ~mask; 1592 bcmgenet_umac_writel(priv, reg, UMAC_CMD); 1593 1594 /* UniMAC stops on a packet boundary, wait for a full-size packet 1595 * to be processed 1596 */ 1597 if (enable == 0) 1598 usleep_range(1000, 2000); 1599 } 1600 1601 static int reset_umac(struct bcmgenet_priv *priv) 1602 { 1603 struct device *kdev = &priv->pdev->dev; 1604 unsigned int timeout = 0; 1605 u32 reg; 1606 1607 /* 7358a0/7552a0: bad default in RBUF_FLUSH_CTRL.umac_sw_rst */ 1608 bcmgenet_rbuf_ctrl_set(priv, 0); 1609 udelay(10); 1610 1611 /* disable MAC while updating its registers */ 1612 bcmgenet_umac_writel(priv, 0, UMAC_CMD); 1613 1614 /* issue soft reset, wait for it to complete */ 1615 bcmgenet_umac_writel(priv, CMD_SW_RESET, UMAC_CMD); 1616 while (timeout++ < 1000) { 1617 reg = bcmgenet_umac_readl(priv, UMAC_CMD); 1618 if (!(reg & CMD_SW_RESET)) 1619 return 0; 1620 1621 udelay(1); 1622 } 1623 1624 if (timeout == 1000) { 1625 dev_err(kdev, 1626 "timeout waiting for MAC to come out of reset\n"); 1627 return -ETIMEDOUT; 1628 } 1629 1630 return 0; 1631 } 1632 1633 static void bcmgenet_intr_disable(struct bcmgenet_priv *priv) 1634 { 1635 /* Mask all interrupts.*/ 1636 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET); 1637 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR); 1638 bcmgenet_intrl2_0_writel(priv, 0, INTRL2_CPU_MASK_CLEAR); 1639 bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET); 1640 bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR); 1641 bcmgenet_intrl2_1_writel(priv, 0, INTRL2_CPU_MASK_CLEAR); 1642 } 1643 1644 static int init_umac(struct bcmgenet_priv *priv) 1645 { 1646 struct device *kdev = &priv->pdev->dev; 1647 int ret; 1648 u32 reg, cpu_mask_clear; 1649 int index; 1650 1651 dev_dbg(&priv->pdev->dev, "bcmgenet: init_umac\n"); 1652 1653 ret = reset_umac(priv); 1654 if (ret) 1655 return ret; 1656 1657 bcmgenet_umac_writel(priv, 0, UMAC_CMD); 1658 /* clear tx/rx counter */ 1659 bcmgenet_umac_writel(priv, 1660 MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT, 1661 UMAC_MIB_CTRL); 1662 bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL); 1663 1664 bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN); 1665 1666 /* init rx registers, enable ip header optimization */ 1667 reg = bcmgenet_rbuf_readl(priv, RBUF_CTRL); 1668 reg |= RBUF_ALIGN_2B; 1669 bcmgenet_rbuf_writel(priv, reg, RBUF_CTRL); 1670 1671 if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv)) 1672 bcmgenet_rbuf_writel(priv, 1, RBUF_TBUF_SIZE_CTRL); 1673 1674 bcmgenet_intr_disable(priv); 1675 1676 cpu_mask_clear = UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_TXDMA_BDONE; 1677 1678 dev_dbg(kdev, "%s:Enabling RXDMA_BDONE interrupt\n", __func__); 1679 1680 /* Monitor cable plug/unplugged event for internal PHY */ 1681 if (phy_is_internal(priv->phydev)) { 1682 cpu_mask_clear |= (UMAC_IRQ_LINK_DOWN | UMAC_IRQ_LINK_UP); 1683 } else if (priv->ext_phy) { 1684 cpu_mask_clear |= (UMAC_IRQ_LINK_DOWN | UMAC_IRQ_LINK_UP); 1685 } else if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) { 1686 reg = bcmgenet_bp_mc_get(priv); 1687 reg |= BIT(priv->hw_params->bp_in_en_shift); 1688 1689 /* bp_mask: back pressure mask */ 1690 if (netif_is_multiqueue(priv->dev)) 1691 reg |= priv->hw_params->bp_in_mask; 1692 else 1693 reg &= ~priv->hw_params->bp_in_mask; 1694 bcmgenet_bp_mc_set(priv, reg); 1695 } 1696 1697 /* Enable MDIO interrupts on GENET v3+ */ 1698 if (priv->hw_params->flags & GENET_HAS_MDIO_INTR) 1699 cpu_mask_clear |= UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR; 1700 1701 bcmgenet_intrl2_0_writel(priv, cpu_mask_clear, INTRL2_CPU_MASK_CLEAR); 1702 1703 for (index = 0; index < priv->hw_params->tx_queues; index++) 1704 bcmgenet_intrl2_1_writel(priv, (1 << index), 1705 INTRL2_CPU_MASK_CLEAR); 1706 1707 /* Enable rx/tx engine.*/ 1708 dev_dbg(kdev, "done init umac\n"); 1709 1710 return 0; 1711 } 1712 1713 /* Initialize all house-keeping variables for a TX ring, along 1714 * with corresponding hardware registers 1715 */ 1716 static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, 1717 unsigned int index, unsigned int size, 1718 unsigned int write_ptr, unsigned int end_ptr) 1719 { 1720 struct bcmgenet_tx_ring *ring = &priv->tx_rings[index]; 1721 u32 words_per_bd = WORDS_PER_BD(priv); 1722 u32 flow_period_val = 0; 1723 unsigned int first_bd; 1724 1725 spin_lock_init(&ring->lock); 1726 ring->priv = priv; 1727 netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64); 1728 ring->index = index; 1729 if (index == DESC_INDEX) { 1730 ring->queue = 0; 1731 ring->int_enable = bcmgenet_tx_ring16_int_enable; 1732 ring->int_disable = bcmgenet_tx_ring16_int_disable; 1733 } else { 1734 ring->queue = index + 1; 1735 ring->int_enable = bcmgenet_tx_ring_int_enable; 1736 ring->int_disable = bcmgenet_tx_ring_int_disable; 1737 } 1738 ring->cbs = priv->tx_cbs + write_ptr; 1739 ring->size = size; 1740 ring->c_index = 0; 1741 ring->free_bds = size; 1742 ring->write_ptr = write_ptr; 1743 ring->cb_ptr = write_ptr; 1744 ring->end_ptr = end_ptr - 1; 1745 ring->prod_index = 0; 1746 1747 /* Set flow period for ring != 16 */ 1748 if (index != DESC_INDEX) 1749 flow_period_val = ENET_MAX_MTU_SIZE << 16; 1750 1751 bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX); 1752 bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX); 1753 bcmgenet_tdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH); 1754 /* Disable rate control for now */ 1755 bcmgenet_tdma_ring_writel(priv, index, flow_period_val, 1756 TDMA_FLOW_PERIOD); 1757 /* Unclassified traffic goes to ring 16 */ 1758 bcmgenet_tdma_ring_writel(priv, index, 1759 ((size << DMA_RING_SIZE_SHIFT) | 1760 RX_BUF_LENGTH), DMA_RING_BUF_SIZE); 1761 1762 first_bd = write_ptr; 1763 1764 /* Set start and end address, read and write pointers */ 1765 bcmgenet_tdma_ring_writel(priv, index, first_bd * words_per_bd, 1766 DMA_START_ADDR); 1767 bcmgenet_tdma_ring_writel(priv, index, first_bd * words_per_bd, 1768 TDMA_READ_PTR); 1769 bcmgenet_tdma_ring_writel(priv, index, first_bd, 1770 TDMA_WRITE_PTR); 1771 bcmgenet_tdma_ring_writel(priv, index, end_ptr * words_per_bd - 1, 1772 DMA_END_ADDR); 1773 1774 napi_enable(&ring->napi); 1775 } 1776 1777 static void bcmgenet_fini_tx_ring(struct bcmgenet_priv *priv, 1778 unsigned int index) 1779 { 1780 struct bcmgenet_tx_ring *ring = &priv->tx_rings[index]; 1781 1782 napi_disable(&ring->napi); 1783 netif_napi_del(&ring->napi); 1784 } 1785 1786 /* Initialize a RDMA ring */ 1787 static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, 1788 unsigned int index, unsigned int size) 1789 { 1790 u32 words_per_bd = WORDS_PER_BD(priv); 1791 int ret; 1792 1793 priv->num_rx_bds = TOTAL_DESC; 1794 priv->rx_bds = priv->base + priv->hw_params->rdma_offset; 1795 priv->rx_bd_assign_ptr = priv->rx_bds; 1796 priv->rx_bd_assign_index = 0; 1797 priv->rx_c_index = 0; 1798 priv->rx_read_ptr = 0; 1799 priv->rx_cbs = kcalloc(priv->num_rx_bds, sizeof(struct enet_cb), 1800 GFP_KERNEL); 1801 if (!priv->rx_cbs) 1802 return -ENOMEM; 1803 1804 ret = bcmgenet_alloc_rx_buffers(priv); 1805 if (ret) { 1806 kfree(priv->rx_cbs); 1807 return ret; 1808 } 1809 1810 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_WRITE_PTR); 1811 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX); 1812 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX); 1813 bcmgenet_rdma_ring_writel(priv, index, 1814 ((size << DMA_RING_SIZE_SHIFT) | 1815 RX_BUF_LENGTH), DMA_RING_BUF_SIZE); 1816 bcmgenet_rdma_ring_writel(priv, index, 0, DMA_START_ADDR); 1817 bcmgenet_rdma_ring_writel(priv, index, 1818 words_per_bd * size - 1, DMA_END_ADDR); 1819 bcmgenet_rdma_ring_writel(priv, index, 1820 (DMA_FC_THRESH_LO << 1821 DMA_XOFF_THRESHOLD_SHIFT) | 1822 DMA_FC_THRESH_HI, RDMA_XON_XOFF_THRESH); 1823 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_READ_PTR); 1824 1825 return ret; 1826 } 1827 1828 /* init multi xmit queues, only available for GENET2+ 1829 * the queue is partitioned as follows: 1830 * 1831 * queue 0 - 3 is priority based, each one has 32 descriptors, 1832 * with queue 0 being the highest priority queue. 1833 * 1834 * queue 16 is the default tx queue with GENET_DEFAULT_BD_CNT 1835 * descriptors: 256 - (number of tx queues * bds per queues) = 128 1836 * descriptors. 1837 * 1838 * The transmit control block pool is then partitioned as following: 1839 * - tx_cbs[0...127] are for queue 16 1840 * - tx_ring_cbs[0] points to tx_cbs[128..159] 1841 * - tx_ring_cbs[1] points to tx_cbs[160..191] 1842 * - tx_ring_cbs[2] points to tx_cbs[192..223] 1843 * - tx_ring_cbs[3] points to tx_cbs[224..255] 1844 */ 1845 static void bcmgenet_init_multiq(struct net_device *dev) 1846 { 1847 struct bcmgenet_priv *priv = netdev_priv(dev); 1848 unsigned int i, dma_enable; 1849 u32 reg, dma_ctrl, ring_cfg = 0; 1850 u32 dma_priority[3] = {0, 0, 0}; 1851 1852 if (!netif_is_multiqueue(dev)) { 1853 netdev_warn(dev, "called with non multi queue aware HW\n"); 1854 return; 1855 } 1856 1857 dma_ctrl = bcmgenet_tdma_readl(priv, DMA_CTRL); 1858 dma_enable = dma_ctrl & DMA_EN; 1859 dma_ctrl &= ~DMA_EN; 1860 bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL); 1861 1862 /* Enable strict priority arbiter mode */ 1863 bcmgenet_tdma_writel(priv, DMA_ARBITER_SP, DMA_ARB_CTRL); 1864 1865 for (i = 0; i < priv->hw_params->tx_queues; i++) { 1866 /* first 64 tx_cbs are reserved for default tx queue 1867 * (ring 16) 1868 */ 1869 bcmgenet_init_tx_ring(priv, i, priv->hw_params->bds_cnt, 1870 i * priv->hw_params->bds_cnt, 1871 (i + 1) * priv->hw_params->bds_cnt); 1872 1873 /* Configure ring as descriptor ring and setup priority */ 1874 ring_cfg |= 1 << i; 1875 dma_ctrl |= 1 << (i + DMA_RING_BUF_EN_SHIFT); 1876 1877 dma_priority[DMA_PRIO_REG_INDEX(i)] |= 1878 ((GENET_Q0_PRIORITY + i) << DMA_PRIO_REG_SHIFT(i)); 1879 } 1880 1881 /* Set ring 16 priority and program the hardware registers */ 1882 dma_priority[DMA_PRIO_REG_INDEX(DESC_INDEX)] |= 1883 ((GENET_Q0_PRIORITY + priv->hw_params->tx_queues) << 1884 DMA_PRIO_REG_SHIFT(DESC_INDEX)); 1885 bcmgenet_tdma_writel(priv, dma_priority[0], DMA_PRIORITY_0); 1886 bcmgenet_tdma_writel(priv, dma_priority[1], DMA_PRIORITY_1); 1887 bcmgenet_tdma_writel(priv, dma_priority[2], DMA_PRIORITY_2); 1888 1889 /* Enable rings */ 1890 reg = bcmgenet_tdma_readl(priv, DMA_RING_CFG); 1891 reg |= ring_cfg; 1892 bcmgenet_tdma_writel(priv, reg, DMA_RING_CFG); 1893 1894 /* Configure ring as descriptor ring and re-enable DMA if enabled */ 1895 reg = bcmgenet_tdma_readl(priv, DMA_CTRL); 1896 reg |= dma_ctrl; 1897 if (dma_enable) 1898 reg |= DMA_EN; 1899 bcmgenet_tdma_writel(priv, reg, DMA_CTRL); 1900 } 1901 1902 static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv) 1903 { 1904 int ret = 0; 1905 int timeout = 0; 1906 u32 reg; 1907 1908 /* Disable TDMA to stop add more frames in TX DMA */ 1909 reg = bcmgenet_tdma_readl(priv, DMA_CTRL); 1910 reg &= ~DMA_EN; 1911 bcmgenet_tdma_writel(priv, reg, DMA_CTRL); 1912 1913 /* Check TDMA status register to confirm TDMA is disabled */ 1914 while (timeout++ < DMA_TIMEOUT_VAL) { 1915 reg = bcmgenet_tdma_readl(priv, DMA_STATUS); 1916 if (reg & DMA_DISABLED) 1917 break; 1918 1919 udelay(1); 1920 } 1921 1922 if (timeout == DMA_TIMEOUT_VAL) { 1923 netdev_warn(priv->dev, "Timed out while disabling TX DMA\n"); 1924 ret = -ETIMEDOUT; 1925 } 1926 1927 /* Wait 10ms for packet drain in both tx and rx dma */ 1928 usleep_range(10000, 20000); 1929 1930 /* Disable RDMA */ 1931 reg = bcmgenet_rdma_readl(priv, DMA_CTRL); 1932 reg &= ~DMA_EN; 1933 bcmgenet_rdma_writel(priv, reg, DMA_CTRL); 1934 1935 timeout = 0; 1936 /* Check RDMA status register to confirm RDMA is disabled */ 1937 while (timeout++ < DMA_TIMEOUT_VAL) { 1938 reg = bcmgenet_rdma_readl(priv, DMA_STATUS); 1939 if (reg & DMA_DISABLED) 1940 break; 1941 1942 udelay(1); 1943 } 1944 1945 if (timeout == DMA_TIMEOUT_VAL) { 1946 netdev_warn(priv->dev, "Timed out while disabling RX DMA\n"); 1947 ret = -ETIMEDOUT; 1948 } 1949 1950 return ret; 1951 } 1952 1953 static void __bcmgenet_fini_dma(struct bcmgenet_priv *priv) 1954 { 1955 int i; 1956 1957 /* disable DMA */ 1958 bcmgenet_dma_teardown(priv); 1959 1960 for (i = 0; i < priv->num_tx_bds; i++) { 1961 if (priv->tx_cbs[i].skb != NULL) { 1962 dev_kfree_skb(priv->tx_cbs[i].skb); 1963 priv->tx_cbs[i].skb = NULL; 1964 } 1965 } 1966 1967 bcmgenet_free_rx_buffers(priv); 1968 kfree(priv->rx_cbs); 1969 kfree(priv->tx_cbs); 1970 } 1971 1972 static void bcmgenet_fini_dma(struct bcmgenet_priv *priv) 1973 { 1974 int i; 1975 1976 bcmgenet_fini_tx_ring(priv, DESC_INDEX); 1977 1978 for (i = 0; i < priv->hw_params->tx_queues; i++) 1979 bcmgenet_fini_tx_ring(priv, i); 1980 1981 __bcmgenet_fini_dma(priv); 1982 } 1983 1984 /* init_edma: Initialize DMA control register */ 1985 static int bcmgenet_init_dma(struct bcmgenet_priv *priv) 1986 { 1987 int ret; 1988 1989 netif_dbg(priv, hw, priv->dev, "bcmgenet: init_edma\n"); 1990 1991 /* by default, enable ring 16 (descriptor based) */ 1992 ret = bcmgenet_init_rx_ring(priv, DESC_INDEX, TOTAL_DESC); 1993 if (ret) { 1994 netdev_err(priv->dev, "failed to initialize RX ring\n"); 1995 return ret; 1996 } 1997 1998 /* init rDma */ 1999 bcmgenet_rdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE); 2000 2001 /* Init tDma */ 2002 bcmgenet_tdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE); 2003 2004 /* Initialize common TX ring structures */ 2005 priv->tx_bds = priv->base + priv->hw_params->tdma_offset; 2006 priv->num_tx_bds = TOTAL_DESC; 2007 priv->tx_cbs = kcalloc(priv->num_tx_bds, sizeof(struct enet_cb), 2008 GFP_KERNEL); 2009 if (!priv->tx_cbs) { 2010 __bcmgenet_fini_dma(priv); 2011 return -ENOMEM; 2012 } 2013 2014 /* initialize multi xmit queue */ 2015 bcmgenet_init_multiq(priv->dev); 2016 2017 /* initialize special ring 16 */ 2018 bcmgenet_init_tx_ring(priv, DESC_INDEX, GENET_DEFAULT_BD_CNT, 2019 priv->hw_params->tx_queues * 2020 priv->hw_params->bds_cnt, 2021 TOTAL_DESC); 2022 2023 return 0; 2024 } 2025 2026 /* NAPI polling method*/ 2027 static int bcmgenet_poll(struct napi_struct *napi, int budget) 2028 { 2029 struct bcmgenet_priv *priv = container_of(napi, 2030 struct bcmgenet_priv, napi); 2031 unsigned int work_done; 2032 2033 work_done = bcmgenet_desc_rx(priv, budget); 2034 2035 /* Advancing our consumer index*/ 2036 priv->rx_c_index += work_done; 2037 priv->rx_c_index &= DMA_C_INDEX_MASK; 2038 bcmgenet_rdma_ring_writel(priv, DESC_INDEX, 2039 priv->rx_c_index, RDMA_CONS_INDEX); 2040 if (work_done < budget) { 2041 napi_complete(napi); 2042 bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_BDONE, 2043 INTRL2_CPU_MASK_CLEAR); 2044 } 2045 2046 return work_done; 2047 } 2048 2049 /* Interrupt bottom half */ 2050 static void bcmgenet_irq_task(struct work_struct *work) 2051 { 2052 struct bcmgenet_priv *priv = container_of( 2053 work, struct bcmgenet_priv, bcmgenet_irq_work); 2054 2055 netif_dbg(priv, intr, priv->dev, "%s\n", __func__); 2056 2057 if (priv->irq0_stat & UMAC_IRQ_MPD_R) { 2058 priv->irq0_stat &= ~UMAC_IRQ_MPD_R; 2059 netif_dbg(priv, wol, priv->dev, 2060 "magic packet detected, waking up\n"); 2061 bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC); 2062 } 2063 2064 /* Link UP/DOWN event */ 2065 if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) && 2066 (priv->irq0_stat & (UMAC_IRQ_LINK_UP|UMAC_IRQ_LINK_DOWN))) { 2067 phy_mac_interrupt(priv->phydev, 2068 priv->irq0_stat & UMAC_IRQ_LINK_UP); 2069 priv->irq0_stat &= ~(UMAC_IRQ_LINK_UP|UMAC_IRQ_LINK_DOWN); 2070 } 2071 } 2072 2073 /* bcmgenet_isr1: interrupt handler for ring buffer. */ 2074 static irqreturn_t bcmgenet_isr1(int irq, void *dev_id) 2075 { 2076 struct bcmgenet_priv *priv = dev_id; 2077 struct bcmgenet_tx_ring *ring; 2078 unsigned int index; 2079 2080 /* Save irq status for bottom-half processing. */ 2081 priv->irq1_stat = 2082 bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) & 2083 ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS); 2084 /* clear interrupts */ 2085 bcmgenet_intrl2_1_writel(priv, priv->irq1_stat, INTRL2_CPU_CLEAR); 2086 2087 netif_dbg(priv, intr, priv->dev, 2088 "%s: IRQ=0x%x\n", __func__, priv->irq1_stat); 2089 2090 /* Check the MBDONE interrupts. 2091 * packet is done, reclaim descriptors 2092 */ 2093 for (index = 0; index < priv->hw_params->tx_queues; index++) { 2094 if (!(priv->irq1_stat & BIT(index))) 2095 continue; 2096 2097 ring = &priv->tx_rings[index]; 2098 2099 if (likely(napi_schedule_prep(&ring->napi))) { 2100 ring->int_disable(priv, ring); 2101 __napi_schedule(&ring->napi); 2102 } 2103 } 2104 2105 return IRQ_HANDLED; 2106 } 2107 2108 /* bcmgenet_isr0: Handle various interrupts. */ 2109 static irqreturn_t bcmgenet_isr0(int irq, void *dev_id) 2110 { 2111 struct bcmgenet_priv *priv = dev_id; 2112 2113 /* Save irq status for bottom-half processing. */ 2114 priv->irq0_stat = 2115 bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) & 2116 ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS); 2117 /* clear interrupts */ 2118 bcmgenet_intrl2_0_writel(priv, priv->irq0_stat, INTRL2_CPU_CLEAR); 2119 2120 netif_dbg(priv, intr, priv->dev, 2121 "IRQ=0x%x\n", priv->irq0_stat); 2122 2123 if (priv->irq0_stat & (UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_RXDMA_PDONE)) { 2124 /* We use NAPI(software interrupt throttling, if 2125 * Rx Descriptor throttling is not used. 2126 * Disable interrupt, will be enabled in the poll method. 2127 */ 2128 if (likely(napi_schedule_prep(&priv->napi))) { 2129 bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_BDONE, 2130 INTRL2_CPU_MASK_SET); 2131 __napi_schedule(&priv->napi); 2132 } 2133 } 2134 if (priv->irq0_stat & 2135 (UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE)) { 2136 struct bcmgenet_tx_ring *ring = &priv->tx_rings[DESC_INDEX]; 2137 2138 if (likely(napi_schedule_prep(&ring->napi))) { 2139 ring->int_disable(priv, ring); 2140 __napi_schedule(&ring->napi); 2141 } 2142 } 2143 if (priv->irq0_stat & (UMAC_IRQ_PHY_DET_R | 2144 UMAC_IRQ_PHY_DET_F | 2145 UMAC_IRQ_LINK_UP | 2146 UMAC_IRQ_LINK_DOWN | 2147 UMAC_IRQ_HFB_SM | 2148 UMAC_IRQ_HFB_MM | 2149 UMAC_IRQ_MPD_R)) { 2150 /* all other interested interrupts handled in bottom half */ 2151 schedule_work(&priv->bcmgenet_irq_work); 2152 } 2153 2154 if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) && 2155 priv->irq0_stat & (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR)) { 2156 priv->irq0_stat &= ~(UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR); 2157 wake_up(&priv->wq); 2158 } 2159 2160 return IRQ_HANDLED; 2161 } 2162 2163 static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id) 2164 { 2165 struct bcmgenet_priv *priv = dev_id; 2166 2167 pm_wakeup_event(&priv->pdev->dev, 0); 2168 2169 return IRQ_HANDLED; 2170 } 2171 2172 static void bcmgenet_umac_reset(struct bcmgenet_priv *priv) 2173 { 2174 u32 reg; 2175 2176 reg = bcmgenet_rbuf_ctrl_get(priv); 2177 reg |= BIT(1); 2178 bcmgenet_rbuf_ctrl_set(priv, reg); 2179 udelay(10); 2180 2181 reg &= ~BIT(1); 2182 bcmgenet_rbuf_ctrl_set(priv, reg); 2183 udelay(10); 2184 } 2185 2186 static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv, 2187 unsigned char *addr) 2188 { 2189 bcmgenet_umac_writel(priv, (addr[0] << 24) | (addr[1] << 16) | 2190 (addr[2] << 8) | addr[3], UMAC_MAC0); 2191 bcmgenet_umac_writel(priv, (addr[4] << 8) | addr[5], UMAC_MAC1); 2192 } 2193 2194 /* Returns a reusable dma control register value */ 2195 static u32 bcmgenet_dma_disable(struct bcmgenet_priv *priv) 2196 { 2197 u32 reg; 2198 u32 dma_ctrl; 2199 2200 /* disable DMA */ 2201 dma_ctrl = 1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT) | DMA_EN; 2202 reg = bcmgenet_tdma_readl(priv, DMA_CTRL); 2203 reg &= ~dma_ctrl; 2204 bcmgenet_tdma_writel(priv, reg, DMA_CTRL); 2205 2206 reg = bcmgenet_rdma_readl(priv, DMA_CTRL); 2207 reg &= ~dma_ctrl; 2208 bcmgenet_rdma_writel(priv, reg, DMA_CTRL); 2209 2210 bcmgenet_umac_writel(priv, 1, UMAC_TX_FLUSH); 2211 udelay(10); 2212 bcmgenet_umac_writel(priv, 0, UMAC_TX_FLUSH); 2213 2214 return dma_ctrl; 2215 } 2216 2217 static void bcmgenet_enable_dma(struct bcmgenet_priv *priv, u32 dma_ctrl) 2218 { 2219 u32 reg; 2220 2221 reg = bcmgenet_rdma_readl(priv, DMA_CTRL); 2222 reg |= dma_ctrl; 2223 bcmgenet_rdma_writel(priv, reg, DMA_CTRL); 2224 2225 reg = bcmgenet_tdma_readl(priv, DMA_CTRL); 2226 reg |= dma_ctrl; 2227 bcmgenet_tdma_writel(priv, reg, DMA_CTRL); 2228 } 2229 2230 static void bcmgenet_netif_start(struct net_device *dev) 2231 { 2232 struct bcmgenet_priv *priv = netdev_priv(dev); 2233 2234 /* Start the network engine */ 2235 napi_enable(&priv->napi); 2236 2237 umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, true); 2238 2239 if (phy_is_internal(priv->phydev)) 2240 bcmgenet_power_up(priv, GENET_POWER_PASSIVE); 2241 2242 netif_tx_start_all_queues(dev); 2243 2244 phy_start(priv->phydev); 2245 } 2246 2247 static int bcmgenet_open(struct net_device *dev) 2248 { 2249 struct bcmgenet_priv *priv = netdev_priv(dev); 2250 unsigned long dma_ctrl; 2251 u32 reg; 2252 int ret; 2253 2254 netif_dbg(priv, ifup, dev, "bcmgenet_open\n"); 2255 2256 /* Turn on the clock */ 2257 if (!IS_ERR(priv->clk)) 2258 clk_prepare_enable(priv->clk); 2259 2260 /* take MAC out of reset */ 2261 bcmgenet_umac_reset(priv); 2262 2263 ret = init_umac(priv); 2264 if (ret) 2265 goto err_clk_disable; 2266 2267 /* disable ethernet MAC while updating its registers */ 2268 umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, false); 2269 2270 /* Make sure we reflect the value of CRC_CMD_FWD */ 2271 reg = bcmgenet_umac_readl(priv, UMAC_CMD); 2272 priv->crc_fwd_en = !!(reg & CMD_CRC_FWD); 2273 2274 bcmgenet_set_hw_addr(priv, dev->dev_addr); 2275 2276 if (phy_is_internal(priv->phydev)) { 2277 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); 2278 reg |= EXT_ENERGY_DET_MASK; 2279 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT); 2280 } 2281 2282 /* Disable RX/TX DMA and flush TX queues */ 2283 dma_ctrl = bcmgenet_dma_disable(priv); 2284 2285 /* Reinitialize TDMA and RDMA and SW housekeeping */ 2286 ret = bcmgenet_init_dma(priv); 2287 if (ret) { 2288 netdev_err(dev, "failed to initialize DMA\n"); 2289 goto err_fini_dma; 2290 } 2291 2292 /* Always enable ring 16 - descriptor ring */ 2293 bcmgenet_enable_dma(priv, dma_ctrl); 2294 2295 ret = request_irq(priv->irq0, bcmgenet_isr0, IRQF_SHARED, 2296 dev->name, priv); 2297 if (ret < 0) { 2298 netdev_err(dev, "can't request IRQ %d\n", priv->irq0); 2299 goto err_fini_dma; 2300 } 2301 2302 ret = request_irq(priv->irq1, bcmgenet_isr1, IRQF_SHARED, 2303 dev->name, priv); 2304 if (ret < 0) { 2305 netdev_err(dev, "can't request IRQ %d\n", priv->irq1); 2306 goto err_irq0; 2307 } 2308 2309 /* Re-configure the port multiplexer towards the PHY device */ 2310 bcmgenet_mii_config(priv->dev, false); 2311 2312 phy_connect_direct(dev, priv->phydev, bcmgenet_mii_setup, 2313 priv->phy_interface); 2314 2315 bcmgenet_netif_start(dev); 2316 2317 return 0; 2318 2319 err_irq0: 2320 free_irq(priv->irq0, dev); 2321 err_fini_dma: 2322 bcmgenet_fini_dma(priv); 2323 err_clk_disable: 2324 if (!IS_ERR(priv->clk)) 2325 clk_disable_unprepare(priv->clk); 2326 return ret; 2327 } 2328 2329 static void bcmgenet_netif_stop(struct net_device *dev) 2330 { 2331 struct bcmgenet_priv *priv = netdev_priv(dev); 2332 2333 netif_tx_stop_all_queues(dev); 2334 napi_disable(&priv->napi); 2335 phy_stop(priv->phydev); 2336 2337 bcmgenet_intr_disable(priv); 2338 2339 /* Wait for pending work items to complete. Since interrupts are 2340 * disabled no new work will be scheduled. 2341 */ 2342 cancel_work_sync(&priv->bcmgenet_irq_work); 2343 2344 priv->old_link = -1; 2345 priv->old_speed = -1; 2346 priv->old_duplex = -1; 2347 priv->old_pause = -1; 2348 } 2349 2350 static int bcmgenet_close(struct net_device *dev) 2351 { 2352 struct bcmgenet_priv *priv = netdev_priv(dev); 2353 int ret; 2354 2355 netif_dbg(priv, ifdown, dev, "bcmgenet_close\n"); 2356 2357 bcmgenet_netif_stop(dev); 2358 2359 /* Really kill the PHY state machine and disconnect from it */ 2360 phy_disconnect(priv->phydev); 2361 2362 /* Disable MAC receive */ 2363 umac_enable_set(priv, CMD_RX_EN, false); 2364 2365 ret = bcmgenet_dma_teardown(priv); 2366 if (ret) 2367 return ret; 2368 2369 /* Disable MAC transmit. TX DMA disabled have to done before this */ 2370 umac_enable_set(priv, CMD_TX_EN, false); 2371 2372 /* tx reclaim */ 2373 bcmgenet_tx_reclaim_all(dev); 2374 bcmgenet_fini_dma(priv); 2375 2376 free_irq(priv->irq0, priv); 2377 free_irq(priv->irq1, priv); 2378 2379 if (phy_is_internal(priv->phydev)) 2380 bcmgenet_power_down(priv, GENET_POWER_PASSIVE); 2381 2382 if (!IS_ERR(priv->clk)) 2383 clk_disable_unprepare(priv->clk); 2384 2385 return 0; 2386 } 2387 2388 static void bcmgenet_timeout(struct net_device *dev) 2389 { 2390 struct bcmgenet_priv *priv = netdev_priv(dev); 2391 2392 netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n"); 2393 2394 dev->trans_start = jiffies; 2395 2396 dev->stats.tx_errors++; 2397 2398 netif_tx_wake_all_queues(dev); 2399 } 2400 2401 #define MAX_MC_COUNT 16 2402 2403 static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv, 2404 unsigned char *addr, 2405 int *i, 2406 int *mc) 2407 { 2408 u32 reg; 2409 2410 bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1], 2411 UMAC_MDF_ADDR + (*i * 4)); 2412 bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 | 2413 addr[4] << 8 | addr[5], 2414 UMAC_MDF_ADDR + ((*i + 1) * 4)); 2415 reg = bcmgenet_umac_readl(priv, UMAC_MDF_CTRL); 2416 reg |= (1 << (MAX_MC_COUNT - *mc)); 2417 bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL); 2418 *i += 2; 2419 (*mc)++; 2420 } 2421 2422 static void bcmgenet_set_rx_mode(struct net_device *dev) 2423 { 2424 struct bcmgenet_priv *priv = netdev_priv(dev); 2425 struct netdev_hw_addr *ha; 2426 int i, mc; 2427 u32 reg; 2428 2429 netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags); 2430 2431 /* Promiscuous mode */ 2432 reg = bcmgenet_umac_readl(priv, UMAC_CMD); 2433 if (dev->flags & IFF_PROMISC) { 2434 reg |= CMD_PROMISC; 2435 bcmgenet_umac_writel(priv, reg, UMAC_CMD); 2436 bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL); 2437 return; 2438 } else { 2439 reg &= ~CMD_PROMISC; 2440 bcmgenet_umac_writel(priv, reg, UMAC_CMD); 2441 } 2442 2443 /* UniMac doesn't support ALLMULTI */ 2444 if (dev->flags & IFF_ALLMULTI) { 2445 netdev_warn(dev, "ALLMULTI is not supported\n"); 2446 return; 2447 } 2448 2449 /* update MDF filter */ 2450 i = 0; 2451 mc = 0; 2452 /* Broadcast */ 2453 bcmgenet_set_mdf_addr(priv, dev->broadcast, &i, &mc); 2454 /* my own address.*/ 2455 bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i, &mc); 2456 /* Unicast list*/ 2457 if (netdev_uc_count(dev) > (MAX_MC_COUNT - mc)) 2458 return; 2459 2460 if (!netdev_uc_empty(dev)) 2461 netdev_for_each_uc_addr(ha, dev) 2462 bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc); 2463 /* Multicast */ 2464 if (netdev_mc_empty(dev) || netdev_mc_count(dev) >= (MAX_MC_COUNT - mc)) 2465 return; 2466 2467 netdev_for_each_mc_addr(ha, dev) 2468 bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc); 2469 } 2470 2471 /* Set the hardware MAC address. */ 2472 static int bcmgenet_set_mac_addr(struct net_device *dev, void *p) 2473 { 2474 struct sockaddr *addr = p; 2475 2476 /* Setting the MAC address at the hardware level is not possible 2477 * without disabling the UniMAC RX/TX enable bits. 2478 */ 2479 if (netif_running(dev)) 2480 return -EBUSY; 2481 2482 ether_addr_copy(dev->dev_addr, addr->sa_data); 2483 2484 return 0; 2485 } 2486 2487 static const struct net_device_ops bcmgenet_netdev_ops = { 2488 .ndo_open = bcmgenet_open, 2489 .ndo_stop = bcmgenet_close, 2490 .ndo_start_xmit = bcmgenet_xmit, 2491 .ndo_tx_timeout = bcmgenet_timeout, 2492 .ndo_set_rx_mode = bcmgenet_set_rx_mode, 2493 .ndo_set_mac_address = bcmgenet_set_mac_addr, 2494 .ndo_do_ioctl = bcmgenet_ioctl, 2495 .ndo_set_features = bcmgenet_set_features, 2496 }; 2497 2498 /* Array of GENET hardware parameters/characteristics */ 2499 static struct bcmgenet_hw_params bcmgenet_hw_params[] = { 2500 [GENET_V1] = { 2501 .tx_queues = 0, 2502 .rx_queues = 0, 2503 .bds_cnt = 0, 2504 .bp_in_en_shift = 16, 2505 .bp_in_mask = 0xffff, 2506 .hfb_filter_cnt = 16, 2507 .qtag_mask = 0x1F, 2508 .hfb_offset = 0x1000, 2509 .rdma_offset = 0x2000, 2510 .tdma_offset = 0x3000, 2511 .words_per_bd = 2, 2512 }, 2513 [GENET_V2] = { 2514 .tx_queues = 4, 2515 .rx_queues = 4, 2516 .bds_cnt = 32, 2517 .bp_in_en_shift = 16, 2518 .bp_in_mask = 0xffff, 2519 .hfb_filter_cnt = 16, 2520 .qtag_mask = 0x1F, 2521 .tbuf_offset = 0x0600, 2522 .hfb_offset = 0x1000, 2523 .hfb_reg_offset = 0x2000, 2524 .rdma_offset = 0x3000, 2525 .tdma_offset = 0x4000, 2526 .words_per_bd = 2, 2527 .flags = GENET_HAS_EXT, 2528 }, 2529 [GENET_V3] = { 2530 .tx_queues = 4, 2531 .rx_queues = 4, 2532 .bds_cnt = 32, 2533 .bp_in_en_shift = 17, 2534 .bp_in_mask = 0x1ffff, 2535 .hfb_filter_cnt = 48, 2536 .qtag_mask = 0x3F, 2537 .tbuf_offset = 0x0600, 2538 .hfb_offset = 0x8000, 2539 .hfb_reg_offset = 0xfc00, 2540 .rdma_offset = 0x10000, 2541 .tdma_offset = 0x11000, 2542 .words_per_bd = 2, 2543 .flags = GENET_HAS_EXT | GENET_HAS_MDIO_INTR, 2544 }, 2545 [GENET_V4] = { 2546 .tx_queues = 4, 2547 .rx_queues = 4, 2548 .bds_cnt = 32, 2549 .bp_in_en_shift = 17, 2550 .bp_in_mask = 0x1ffff, 2551 .hfb_filter_cnt = 48, 2552 .qtag_mask = 0x3F, 2553 .tbuf_offset = 0x0600, 2554 .hfb_offset = 0x8000, 2555 .hfb_reg_offset = 0xfc00, 2556 .rdma_offset = 0x2000, 2557 .tdma_offset = 0x4000, 2558 .words_per_bd = 3, 2559 .flags = GENET_HAS_40BITS | GENET_HAS_EXT | GENET_HAS_MDIO_INTR, 2560 }, 2561 }; 2562 2563 /* Infer hardware parameters from the detected GENET version */ 2564 static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv) 2565 { 2566 struct bcmgenet_hw_params *params; 2567 u32 reg; 2568 u8 major; 2569 u16 gphy_rev; 2570 2571 if (GENET_IS_V4(priv)) { 2572 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus; 2573 genet_dma_ring_regs = genet_dma_ring_regs_v4; 2574 priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS; 2575 priv->version = GENET_V4; 2576 } else if (GENET_IS_V3(priv)) { 2577 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus; 2578 genet_dma_ring_regs = genet_dma_ring_regs_v123; 2579 priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS; 2580 priv->version = GENET_V3; 2581 } else if (GENET_IS_V2(priv)) { 2582 bcmgenet_dma_regs = bcmgenet_dma_regs_v2; 2583 genet_dma_ring_regs = genet_dma_ring_regs_v123; 2584 priv->dma_rx_chk_bit = DMA_RX_CHK_V12; 2585 priv->version = GENET_V2; 2586 } else if (GENET_IS_V1(priv)) { 2587 bcmgenet_dma_regs = bcmgenet_dma_regs_v1; 2588 genet_dma_ring_regs = genet_dma_ring_regs_v123; 2589 priv->dma_rx_chk_bit = DMA_RX_CHK_V12; 2590 priv->version = GENET_V1; 2591 } 2592 2593 /* enum genet_version starts at 1 */ 2594 priv->hw_params = &bcmgenet_hw_params[priv->version]; 2595 params = priv->hw_params; 2596 2597 /* Read GENET HW version */ 2598 reg = bcmgenet_sys_readl(priv, SYS_REV_CTRL); 2599 major = (reg >> 24 & 0x0f); 2600 if (major == 5) 2601 major = 4; 2602 else if (major == 0) 2603 major = 1; 2604 if (major != priv->version) { 2605 dev_err(&priv->pdev->dev, 2606 "GENET version mismatch, got: %d, configured for: %d\n", 2607 major, priv->version); 2608 } 2609 2610 /* Print the GENET core version */ 2611 dev_info(&priv->pdev->dev, "GENET " GENET_VER_FMT, 2612 major, (reg >> 16) & 0x0f, reg & 0xffff); 2613 2614 /* Store the integrated PHY revision for the MDIO probing function 2615 * to pass this information to the PHY driver. The PHY driver expects 2616 * to find the PHY major revision in bits 15:8 while the GENET register 2617 * stores that information in bits 7:0, account for that. 2618 * 2619 * On newer chips, starting with PHY revision G0, a new scheme is 2620 * deployed similar to the Starfighter 2 switch with GPHY major 2621 * revision in bits 15:8 and patch level in bits 7:0. Major revision 0 2622 * is reserved as well as special value 0x01ff, we have a small 2623 * heuristic to check for the new GPHY revision and re-arrange things 2624 * so the GPHY driver is happy. 2625 */ 2626 gphy_rev = reg & 0xffff; 2627 2628 /* This is the good old scheme, just GPHY major, no minor nor patch */ 2629 if ((gphy_rev & 0xf0) != 0) 2630 priv->gphy_rev = gphy_rev << 8; 2631 2632 /* This is the new scheme, GPHY major rolls over with 0x10 = rev G0 */ 2633 else if ((gphy_rev & 0xff00) != 0) 2634 priv->gphy_rev = gphy_rev; 2635 2636 /* This is reserved so should require special treatment */ 2637 else if (gphy_rev == 0 || gphy_rev == 0x01ff) { 2638 pr_warn("Invalid GPHY revision detected: 0x%04x\n", gphy_rev); 2639 return; 2640 } 2641 2642 #ifdef CONFIG_PHYS_ADDR_T_64BIT 2643 if (!(params->flags & GENET_HAS_40BITS)) 2644 pr_warn("GENET does not support 40-bits PA\n"); 2645 #endif 2646 2647 pr_debug("Configuration for version: %d\n" 2648 "TXq: %1d, RXq: %1d, BDs: %1d\n" 2649 "BP << en: %2d, BP msk: 0x%05x\n" 2650 "HFB count: %2d, QTAQ msk: 0x%05x\n" 2651 "TBUF: 0x%04x, HFB: 0x%04x, HFBreg: 0x%04x\n" 2652 "RDMA: 0x%05x, TDMA: 0x%05x\n" 2653 "Words/BD: %d\n", 2654 priv->version, 2655 params->tx_queues, params->rx_queues, params->bds_cnt, 2656 params->bp_in_en_shift, params->bp_in_mask, 2657 params->hfb_filter_cnt, params->qtag_mask, 2658 params->tbuf_offset, params->hfb_offset, 2659 params->hfb_reg_offset, 2660 params->rdma_offset, params->tdma_offset, 2661 params->words_per_bd); 2662 } 2663 2664 static const struct of_device_id bcmgenet_match[] = { 2665 { .compatible = "brcm,genet-v1", .data = (void *)GENET_V1 }, 2666 { .compatible = "brcm,genet-v2", .data = (void *)GENET_V2 }, 2667 { .compatible = "brcm,genet-v3", .data = (void *)GENET_V3 }, 2668 { .compatible = "brcm,genet-v4", .data = (void *)GENET_V4 }, 2669 { }, 2670 }; 2671 2672 static int bcmgenet_probe(struct platform_device *pdev) 2673 { 2674 struct bcmgenet_platform_data *pd = pdev->dev.platform_data; 2675 struct device_node *dn = pdev->dev.of_node; 2676 const struct of_device_id *of_id = NULL; 2677 struct bcmgenet_priv *priv; 2678 struct net_device *dev; 2679 const void *macaddr; 2680 struct resource *r; 2681 int err = -EIO; 2682 2683 /* Up to GENET_MAX_MQ_CNT + 1 TX queues and a single RX queue */ 2684 dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1, 1); 2685 if (!dev) { 2686 dev_err(&pdev->dev, "can't allocate net device\n"); 2687 return -ENOMEM; 2688 } 2689 2690 if (dn) { 2691 of_id = of_match_node(bcmgenet_match, dn); 2692 if (!of_id) 2693 return -EINVAL; 2694 } 2695 2696 priv = netdev_priv(dev); 2697 priv->irq0 = platform_get_irq(pdev, 0); 2698 priv->irq1 = platform_get_irq(pdev, 1); 2699 priv->wol_irq = platform_get_irq(pdev, 2); 2700 if (!priv->irq0 || !priv->irq1) { 2701 dev_err(&pdev->dev, "can't find IRQs\n"); 2702 err = -EINVAL; 2703 goto err; 2704 } 2705 2706 if (dn) { 2707 macaddr = of_get_mac_address(dn); 2708 if (!macaddr) { 2709 dev_err(&pdev->dev, "can't find MAC address\n"); 2710 err = -EINVAL; 2711 goto err; 2712 } 2713 } else { 2714 macaddr = pd->mac_address; 2715 } 2716 2717 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2718 priv->base = devm_ioremap_resource(&pdev->dev, r); 2719 if (IS_ERR(priv->base)) { 2720 err = PTR_ERR(priv->base); 2721 goto err; 2722 } 2723 2724 SET_NETDEV_DEV(dev, &pdev->dev); 2725 dev_set_drvdata(&pdev->dev, dev); 2726 ether_addr_copy(dev->dev_addr, macaddr); 2727 dev->watchdog_timeo = 2 * HZ; 2728 dev->ethtool_ops = &bcmgenet_ethtool_ops; 2729 dev->netdev_ops = &bcmgenet_netdev_ops; 2730 netif_napi_add(dev, &priv->napi, bcmgenet_poll, 64); 2731 2732 priv->msg_enable = netif_msg_init(-1, GENET_MSG_DEFAULT); 2733 2734 /* Set hardware features */ 2735 dev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM | 2736 NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM; 2737 2738 /* Request the WOL interrupt and advertise suspend if available */ 2739 priv->wol_irq_disabled = true; 2740 err = devm_request_irq(&pdev->dev, priv->wol_irq, bcmgenet_wol_isr, 0, 2741 dev->name, priv); 2742 if (!err) 2743 device_set_wakeup_capable(&pdev->dev, 1); 2744 2745 /* Set the needed headroom to account for any possible 2746 * features enabling/disabling at runtime 2747 */ 2748 dev->needed_headroom += 64; 2749 2750 netdev_boot_setup_check(dev); 2751 2752 priv->dev = dev; 2753 priv->pdev = pdev; 2754 if (of_id) 2755 priv->version = (enum bcmgenet_version)of_id->data; 2756 else 2757 priv->version = pd->genet_version; 2758 2759 priv->clk = devm_clk_get(&priv->pdev->dev, "enet"); 2760 if (IS_ERR(priv->clk)) 2761 dev_warn(&priv->pdev->dev, "failed to get enet clock\n"); 2762 2763 if (!IS_ERR(priv->clk)) 2764 clk_prepare_enable(priv->clk); 2765 2766 bcmgenet_set_hw_params(priv); 2767 2768 /* Mii wait queue */ 2769 init_waitqueue_head(&priv->wq); 2770 /* Always use RX_BUF_LENGTH (2KB) buffer for all chips */ 2771 priv->rx_buf_len = RX_BUF_LENGTH; 2772 INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task); 2773 2774 priv->clk_wol = devm_clk_get(&priv->pdev->dev, "enet-wol"); 2775 if (IS_ERR(priv->clk_wol)) 2776 dev_warn(&priv->pdev->dev, "failed to get enet-wol clock\n"); 2777 2778 priv->clk_eee = devm_clk_get(&priv->pdev->dev, "enet-eee"); 2779 if (IS_ERR(priv->clk_eee)) { 2780 dev_warn(&priv->pdev->dev, "failed to get enet-eee clock\n"); 2781 priv->clk_eee = NULL; 2782 } 2783 2784 err = reset_umac(priv); 2785 if (err) 2786 goto err_clk_disable; 2787 2788 err = bcmgenet_mii_init(dev); 2789 if (err) 2790 goto err_clk_disable; 2791 2792 /* setup number of real queues + 1 (GENET_V1 has 0 hardware queues 2793 * just the ring 16 descriptor based TX 2794 */ 2795 netif_set_real_num_tx_queues(priv->dev, priv->hw_params->tx_queues + 1); 2796 netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1); 2797 2798 /* libphy will determine the link state */ 2799 netif_carrier_off(dev); 2800 2801 /* Turn off the main clock, WOL clock is handled separately */ 2802 if (!IS_ERR(priv->clk)) 2803 clk_disable_unprepare(priv->clk); 2804 2805 err = register_netdev(dev); 2806 if (err) 2807 goto err; 2808 2809 return err; 2810 2811 err_clk_disable: 2812 if (!IS_ERR(priv->clk)) 2813 clk_disable_unprepare(priv->clk); 2814 err: 2815 free_netdev(dev); 2816 return err; 2817 } 2818 2819 static int bcmgenet_remove(struct platform_device *pdev) 2820 { 2821 struct bcmgenet_priv *priv = dev_to_priv(&pdev->dev); 2822 2823 dev_set_drvdata(&pdev->dev, NULL); 2824 unregister_netdev(priv->dev); 2825 bcmgenet_mii_exit(priv->dev); 2826 free_netdev(priv->dev); 2827 2828 return 0; 2829 } 2830 2831 #ifdef CONFIG_PM_SLEEP 2832 static int bcmgenet_suspend(struct device *d) 2833 { 2834 struct net_device *dev = dev_get_drvdata(d); 2835 struct bcmgenet_priv *priv = netdev_priv(dev); 2836 int ret; 2837 2838 if (!netif_running(dev)) 2839 return 0; 2840 2841 bcmgenet_netif_stop(dev); 2842 2843 phy_suspend(priv->phydev); 2844 2845 netif_device_detach(dev); 2846 2847 /* Disable MAC receive */ 2848 umac_enable_set(priv, CMD_RX_EN, false); 2849 2850 ret = bcmgenet_dma_teardown(priv); 2851 if (ret) 2852 return ret; 2853 2854 /* Disable MAC transmit. TX DMA disabled have to done before this */ 2855 umac_enable_set(priv, CMD_TX_EN, false); 2856 2857 /* tx reclaim */ 2858 bcmgenet_tx_reclaim_all(dev); 2859 bcmgenet_fini_dma(priv); 2860 2861 /* Prepare the device for Wake-on-LAN and switch to the slow clock */ 2862 if (device_may_wakeup(d) && priv->wolopts) { 2863 bcmgenet_power_down(priv, GENET_POWER_WOL_MAGIC); 2864 clk_prepare_enable(priv->clk_wol); 2865 } 2866 2867 /* Turn off the clocks */ 2868 clk_disable_unprepare(priv->clk); 2869 2870 return 0; 2871 } 2872 2873 static int bcmgenet_resume(struct device *d) 2874 { 2875 struct net_device *dev = dev_get_drvdata(d); 2876 struct bcmgenet_priv *priv = netdev_priv(dev); 2877 unsigned long dma_ctrl; 2878 int ret; 2879 u32 reg; 2880 2881 if (!netif_running(dev)) 2882 return 0; 2883 2884 /* Turn on the clock */ 2885 ret = clk_prepare_enable(priv->clk); 2886 if (ret) 2887 return ret; 2888 2889 bcmgenet_umac_reset(priv); 2890 2891 ret = init_umac(priv); 2892 if (ret) 2893 goto out_clk_disable; 2894 2895 /* From WOL-enabled suspend, switch to regular clock */ 2896 if (priv->wolopts) 2897 clk_disable_unprepare(priv->clk_wol); 2898 2899 phy_init_hw(priv->phydev); 2900 /* Speed settings must be restored */ 2901 bcmgenet_mii_config(priv->dev, false); 2902 2903 /* disable ethernet MAC while updating its registers */ 2904 umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, false); 2905 2906 bcmgenet_set_hw_addr(priv, dev->dev_addr); 2907 2908 if (phy_is_internal(priv->phydev)) { 2909 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); 2910 reg |= EXT_ENERGY_DET_MASK; 2911 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT); 2912 } 2913 2914 if (priv->wolopts) 2915 bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC); 2916 2917 /* Disable RX/TX DMA and flush TX queues */ 2918 dma_ctrl = bcmgenet_dma_disable(priv); 2919 2920 /* Reinitialize TDMA and RDMA and SW housekeeping */ 2921 ret = bcmgenet_init_dma(priv); 2922 if (ret) { 2923 netdev_err(dev, "failed to initialize DMA\n"); 2924 goto out_clk_disable; 2925 } 2926 2927 /* Always enable ring 16 - descriptor ring */ 2928 bcmgenet_enable_dma(priv, dma_ctrl); 2929 2930 netif_device_attach(dev); 2931 2932 phy_resume(priv->phydev); 2933 2934 if (priv->eee.eee_enabled) 2935 bcmgenet_eee_enable_set(dev, true); 2936 2937 bcmgenet_netif_start(dev); 2938 2939 return 0; 2940 2941 out_clk_disable: 2942 clk_disable_unprepare(priv->clk); 2943 return ret; 2944 } 2945 #endif /* CONFIG_PM_SLEEP */ 2946 2947 static SIMPLE_DEV_PM_OPS(bcmgenet_pm_ops, bcmgenet_suspend, bcmgenet_resume); 2948 2949 static struct platform_driver bcmgenet_driver = { 2950 .probe = bcmgenet_probe, 2951 .remove = bcmgenet_remove, 2952 .driver = { 2953 .name = "bcmgenet", 2954 .of_match_table = bcmgenet_match, 2955 .pm = &bcmgenet_pm_ops, 2956 }, 2957 }; 2958 module_platform_driver(bcmgenet_driver); 2959 2960 MODULE_AUTHOR("Broadcom Corporation"); 2961 MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver"); 2962 MODULE_ALIAS("platform:bcmgenet"); 2963 MODULE_LICENSE("GPL"); 2964