1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Xilinx Axi Ethernet device driver 4 * 5 * Copyright (c) 2008 Nissin Systems Co., Ltd., Yoshio Kashiwagi 6 * Copyright (c) 2005-2008 DLA Systems, David H. Lynch Jr. <dhlii@dlasys.net> 7 * Copyright (c) 2008-2009 Secret Lab Technologies Ltd. 8 * Copyright (c) 2010 - 2011 Michal Simek <monstr@monstr.eu> 9 * Copyright (c) 2010 - 2011 PetaLogix 10 * Copyright (c) 2019 - 2022 Calian Advanced Technologies 11 * Copyright (c) 2010 - 2012 Xilinx, Inc. All rights reserved. 12 * 13 * This is a driver for the Xilinx Axi Ethernet which is used in the Virtex6 14 * and Spartan6. 15 * 16 * TODO: 17 * - Add Axi Fifo support. 18 * - Factor out Axi DMA code into separate driver. 19 * - Test and fix basic multicast filtering. 20 * - Add support for extended multicast filtering. 21 * - Test basic VLAN support. 22 * - Add support for extended VLAN support. 23 */ 24 25 #include <linux/clk.h> 26 #include <linux/delay.h> 27 #include <linux/etherdevice.h> 28 #include <linux/module.h> 29 #include <linux/netdevice.h> 30 #include <linux/of.h> 31 #include <linux/of_mdio.h> 32 #include <linux/of_net.h> 33 #include <linux/of_irq.h> 34 #include <linux/of_address.h> 35 #include <linux/platform_device.h> 36 #include <linux/skbuff.h> 37 #include <linux/math64.h> 38 #include <linux/phy.h> 39 #include <linux/mii.h> 40 #include <linux/ethtool.h> 41 #include <linux/dmaengine.h> 42 #include <linux/dma-mapping.h> 43 #include <linux/dma/xilinx_dma.h> 44 #include <linux/circ_buf.h> 45 #include <net/netdev_queues.h> 46 47 #include "xilinx_axienet.h" 48 49 /* Descriptors defines for Tx and Rx DMA */ 50 #define TX_BD_NUM_DEFAULT 128 51 #define RX_BD_NUM_DEFAULT 1024 52 #define TX_BD_NUM_MIN (MAX_SKB_FRAGS + 1) 53 #define TX_BD_NUM_MAX 4096 54 #define RX_BD_NUM_MAX 4096 55 #define DMA_NUM_APP_WORDS 5 56 #define LEN_APP 4 57 #define RX_BUF_NUM_DEFAULT 128 58 59 /* Must be shorter than length of ethtool_drvinfo.driver field to fit */ 60 #define DRIVER_NAME "xaxienet" 61 #define DRIVER_DESCRIPTION "Xilinx Axi Ethernet driver" 62 #define DRIVER_VERSION "1.00a" 63 64 #define AXIENET_REGS_N 40 65 66 static void axienet_rx_submit_desc(struct net_device *ndev); 67 68 /* Match table for of_platform binding */ 69 static const struct of_device_id axienet_of_match[] = { 70 { .compatible = "xlnx,axi-ethernet-1.00.a", }, 71 { .compatible = "xlnx,axi-ethernet-1.01.a", }, 72 { .compatible = "xlnx,axi-ethernet-2.01.a", }, 73 {}, 74 }; 75 76 MODULE_DEVICE_TABLE(of, axienet_of_match); 77 78 /* Option table for setting up Axi Ethernet hardware options */ 79 static struct axienet_option axienet_options[] = { 80 /* Turn on jumbo packet support for both Rx and Tx */ 81 { 82 .opt = XAE_OPTION_JUMBO, 83 .reg = XAE_TC_OFFSET, 84 .m_or = XAE_TC_JUM_MASK, 85 }, { 86 .opt = XAE_OPTION_JUMBO, 87 .reg = XAE_RCW1_OFFSET, 88 .m_or = XAE_RCW1_JUM_MASK, 89 }, { /* Turn on VLAN packet support for both Rx and Tx */ 90 .opt = XAE_OPTION_VLAN, 91 .reg = XAE_TC_OFFSET, 92 .m_or = XAE_TC_VLAN_MASK, 93 }, { 94 .opt = XAE_OPTION_VLAN, 95 .reg = XAE_RCW1_OFFSET, 96 .m_or = XAE_RCW1_VLAN_MASK, 97 }, { /* Turn on FCS stripping on receive packets */ 98 .opt = XAE_OPTION_FCS_STRIP, 99 .reg = XAE_RCW1_OFFSET, 100 .m_or = XAE_RCW1_FCS_MASK, 101 }, { /* Turn on FCS insertion on transmit packets */ 102 .opt = XAE_OPTION_FCS_INSERT, 103 .reg = XAE_TC_OFFSET, 104 .m_or = XAE_TC_FCS_MASK, 105 }, { /* Turn off length/type field checking on receive packets */ 106 .opt = XAE_OPTION_LENTYPE_ERR, 107 .reg = XAE_RCW1_OFFSET, 108 .m_or = XAE_RCW1_LT_DIS_MASK, 109 }, { /* Turn on Rx flow control */ 110 .opt = XAE_OPTION_FLOW_CONTROL, 111 .reg = XAE_FCC_OFFSET, 112 .m_or = XAE_FCC_FCRX_MASK, 113 }, { /* Turn on Tx flow control */ 114 .opt = XAE_OPTION_FLOW_CONTROL, 115 .reg = XAE_FCC_OFFSET, 116 .m_or = XAE_FCC_FCTX_MASK, 117 }, { /* Turn on promiscuous frame filtering */ 118 .opt = XAE_OPTION_PROMISC, 119 .reg = XAE_FMI_OFFSET, 120 .m_or = XAE_FMI_PM_MASK, 121 }, { /* Enable transmitter */ 122 .opt = XAE_OPTION_TXEN, 123 .reg = XAE_TC_OFFSET, 124 .m_or = XAE_TC_TX_MASK, 125 }, { /* Enable receiver */ 126 .opt = XAE_OPTION_RXEN, 127 .reg = XAE_RCW1_OFFSET, 128 .m_or = XAE_RCW1_RX_MASK, 129 }, 130 {} 131 }; 132 133 static struct skbuf_dma_descriptor *axienet_get_rx_desc(struct axienet_local *lp, int i) 134 { 135 return lp->rx_skb_ring[i & (RX_BUF_NUM_DEFAULT - 1)]; 136 } 137 138 static struct skbuf_dma_descriptor *axienet_get_tx_desc(struct axienet_local *lp, int i) 139 { 140 return lp->tx_skb_ring[i & (TX_BD_NUM_MAX - 1)]; 141 } 142 143 /** 144 * axienet_dma_in32 - Memory mapped Axi DMA register read 145 * @lp: Pointer to axienet local structure 146 * @reg: Address offset from the base address of the Axi DMA core 147 * 148 * Return: The contents of the Axi DMA register 149 * 150 * This function returns the contents of the corresponding Axi DMA register. 151 */ 152 static inline u32 axienet_dma_in32(struct axienet_local *lp, off_t reg) 153 { 154 return ioread32(lp->dma_regs + reg); 155 } 156 157 static void desc_set_phys_addr(struct axienet_local *lp, dma_addr_t addr, 158 struct axidma_bd *desc) 159 { 160 desc->phys = lower_32_bits(addr); 161 if (lp->features & XAE_FEATURE_DMA_64BIT) 162 desc->phys_msb = upper_32_bits(addr); 163 } 164 165 static dma_addr_t desc_get_phys_addr(struct axienet_local *lp, 166 struct axidma_bd *desc) 167 { 168 dma_addr_t ret = desc->phys; 169 170 if (lp->features & XAE_FEATURE_DMA_64BIT) 171 ret |= ((dma_addr_t)desc->phys_msb << 16) << 16; 172 173 return ret; 174 } 175 176 /** 177 * axienet_dma_bd_release - Release buffer descriptor rings 178 * @ndev: Pointer to the net_device structure 179 * 180 * This function is used to release the descriptors allocated in 181 * axienet_dma_bd_init. axienet_dma_bd_release is called when Axi Ethernet 182 * driver stop api is called. 183 */ 184 static void axienet_dma_bd_release(struct net_device *ndev) 185 { 186 int i; 187 struct axienet_local *lp = netdev_priv(ndev); 188 189 /* If we end up here, tx_bd_v must have been DMA allocated. */ 190 dma_free_coherent(lp->dev, 191 sizeof(*lp->tx_bd_v) * lp->tx_bd_num, 192 lp->tx_bd_v, 193 lp->tx_bd_p); 194 195 if (!lp->rx_bd_v) 196 return; 197 198 for (i = 0; i < lp->rx_bd_num; i++) { 199 dma_addr_t phys; 200 201 /* A NULL skb means this descriptor has not been initialised 202 * at all. 203 */ 204 if (!lp->rx_bd_v[i].skb) 205 break; 206 207 dev_kfree_skb(lp->rx_bd_v[i].skb); 208 209 /* For each descriptor, we programmed cntrl with the (non-zero) 210 * descriptor size, after it had been successfully allocated. 211 * So a non-zero value in there means we need to unmap it. 212 */ 213 if (lp->rx_bd_v[i].cntrl) { 214 phys = desc_get_phys_addr(lp, &lp->rx_bd_v[i]); 215 dma_unmap_single(lp->dev, phys, 216 lp->max_frm_size, DMA_FROM_DEVICE); 217 } 218 } 219 220 dma_free_coherent(lp->dev, 221 sizeof(*lp->rx_bd_v) * lp->rx_bd_num, 222 lp->rx_bd_v, 223 lp->rx_bd_p); 224 } 225 226 /** 227 * axienet_usec_to_timer - Calculate IRQ delay timer value 228 * @lp: Pointer to the axienet_local structure 229 * @coalesce_usec: Microseconds to convert into timer value 230 */ 231 static u32 axienet_usec_to_timer(struct axienet_local *lp, u32 coalesce_usec) 232 { 233 u32 result; 234 u64 clk_rate = 125000000; /* arbitrary guess if no clock rate set */ 235 236 if (lp->axi_clk) 237 clk_rate = clk_get_rate(lp->axi_clk); 238 239 /* 1 Timeout Interval = 125 * (clock period of SG clock) */ 240 result = DIV64_U64_ROUND_CLOSEST((u64)coalesce_usec * clk_rate, 241 (u64)125000000); 242 if (result > 255) 243 result = 255; 244 245 return result; 246 } 247 248 /** 249 * axienet_dma_start - Set up DMA registers and start DMA operation 250 * @lp: Pointer to the axienet_local structure 251 */ 252 static void axienet_dma_start(struct axienet_local *lp) 253 { 254 /* Start updating the Rx channel control register */ 255 lp->rx_dma_cr = (lp->coalesce_count_rx << XAXIDMA_COALESCE_SHIFT) | 256 XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_ERROR_MASK; 257 /* Only set interrupt delay timer if not generating an interrupt on 258 * the first RX packet. Otherwise leave at 0 to disable delay interrupt. 259 */ 260 if (lp->coalesce_count_rx > 1) 261 lp->rx_dma_cr |= (axienet_usec_to_timer(lp, lp->coalesce_usec_rx) 262 << XAXIDMA_DELAY_SHIFT) | 263 XAXIDMA_IRQ_DELAY_MASK; 264 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr); 265 266 /* Start updating the Tx channel control register */ 267 lp->tx_dma_cr = (lp->coalesce_count_tx << XAXIDMA_COALESCE_SHIFT) | 268 XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_ERROR_MASK; 269 /* Only set interrupt delay timer if not generating an interrupt on 270 * the first TX packet. Otherwise leave at 0 to disable delay interrupt. 271 */ 272 if (lp->coalesce_count_tx > 1) 273 lp->tx_dma_cr |= (axienet_usec_to_timer(lp, lp->coalesce_usec_tx) 274 << XAXIDMA_DELAY_SHIFT) | 275 XAXIDMA_IRQ_DELAY_MASK; 276 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr); 277 278 /* Populate the tail pointer and bring the Rx Axi DMA engine out of 279 * halted state. This will make the Rx side ready for reception. 280 */ 281 axienet_dma_out_addr(lp, XAXIDMA_RX_CDESC_OFFSET, lp->rx_bd_p); 282 lp->rx_dma_cr |= XAXIDMA_CR_RUNSTOP_MASK; 283 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr); 284 axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, lp->rx_bd_p + 285 (sizeof(*lp->rx_bd_v) * (lp->rx_bd_num - 1))); 286 287 /* Write to the RS (Run-stop) bit in the Tx channel control register. 288 * Tx channel is now ready to run. But only after we write to the 289 * tail pointer register that the Tx channel will start transmitting. 290 */ 291 axienet_dma_out_addr(lp, XAXIDMA_TX_CDESC_OFFSET, lp->tx_bd_p); 292 lp->tx_dma_cr |= XAXIDMA_CR_RUNSTOP_MASK; 293 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr); 294 } 295 296 /** 297 * axienet_dma_bd_init - Setup buffer descriptor rings for Axi DMA 298 * @ndev: Pointer to the net_device structure 299 * 300 * Return: 0, on success -ENOMEM, on failure 301 * 302 * This function is called to initialize the Rx and Tx DMA descriptor 303 * rings. This initializes the descriptors with required default values 304 * and is called when Axi Ethernet driver reset is called. 305 */ 306 static int axienet_dma_bd_init(struct net_device *ndev) 307 { 308 int i; 309 struct sk_buff *skb; 310 struct axienet_local *lp = netdev_priv(ndev); 311 312 /* Reset the indexes which are used for accessing the BDs */ 313 lp->tx_bd_ci = 0; 314 lp->tx_bd_tail = 0; 315 lp->rx_bd_ci = 0; 316 317 /* Allocate the Tx and Rx buffer descriptors. */ 318 lp->tx_bd_v = dma_alloc_coherent(lp->dev, 319 sizeof(*lp->tx_bd_v) * lp->tx_bd_num, 320 &lp->tx_bd_p, GFP_KERNEL); 321 if (!lp->tx_bd_v) 322 return -ENOMEM; 323 324 lp->rx_bd_v = dma_alloc_coherent(lp->dev, 325 sizeof(*lp->rx_bd_v) * lp->rx_bd_num, 326 &lp->rx_bd_p, GFP_KERNEL); 327 if (!lp->rx_bd_v) 328 goto out; 329 330 for (i = 0; i < lp->tx_bd_num; i++) { 331 dma_addr_t addr = lp->tx_bd_p + 332 sizeof(*lp->tx_bd_v) * 333 ((i + 1) % lp->tx_bd_num); 334 335 lp->tx_bd_v[i].next = lower_32_bits(addr); 336 if (lp->features & XAE_FEATURE_DMA_64BIT) 337 lp->tx_bd_v[i].next_msb = upper_32_bits(addr); 338 } 339 340 for (i = 0; i < lp->rx_bd_num; i++) { 341 dma_addr_t addr; 342 343 addr = lp->rx_bd_p + sizeof(*lp->rx_bd_v) * 344 ((i + 1) % lp->rx_bd_num); 345 lp->rx_bd_v[i].next = lower_32_bits(addr); 346 if (lp->features & XAE_FEATURE_DMA_64BIT) 347 lp->rx_bd_v[i].next_msb = upper_32_bits(addr); 348 349 skb = netdev_alloc_skb_ip_align(ndev, lp->max_frm_size); 350 if (!skb) 351 goto out; 352 353 lp->rx_bd_v[i].skb = skb; 354 addr = dma_map_single(lp->dev, skb->data, 355 lp->max_frm_size, DMA_FROM_DEVICE); 356 if (dma_mapping_error(lp->dev, addr)) { 357 netdev_err(ndev, "DMA mapping error\n"); 358 goto out; 359 } 360 desc_set_phys_addr(lp, addr, &lp->rx_bd_v[i]); 361 362 lp->rx_bd_v[i].cntrl = lp->max_frm_size; 363 } 364 365 axienet_dma_start(lp); 366 367 return 0; 368 out: 369 axienet_dma_bd_release(ndev); 370 return -ENOMEM; 371 } 372 373 /** 374 * axienet_set_mac_address - Write the MAC address 375 * @ndev: Pointer to the net_device structure 376 * @address: 6 byte Address to be written as MAC address 377 * 378 * This function is called to initialize the MAC address of the Axi Ethernet 379 * core. It writes to the UAW0 and UAW1 registers of the core. 380 */ 381 static void axienet_set_mac_address(struct net_device *ndev, 382 const void *address) 383 { 384 struct axienet_local *lp = netdev_priv(ndev); 385 386 if (address) 387 eth_hw_addr_set(ndev, address); 388 if (!is_valid_ether_addr(ndev->dev_addr)) 389 eth_hw_addr_random(ndev); 390 391 /* Set up unicast MAC address filter set its mac address */ 392 axienet_iow(lp, XAE_UAW0_OFFSET, 393 (ndev->dev_addr[0]) | 394 (ndev->dev_addr[1] << 8) | 395 (ndev->dev_addr[2] << 16) | 396 (ndev->dev_addr[3] << 24)); 397 axienet_iow(lp, XAE_UAW1_OFFSET, 398 (((axienet_ior(lp, XAE_UAW1_OFFSET)) & 399 ~XAE_UAW1_UNICASTADDR_MASK) | 400 (ndev->dev_addr[4] | 401 (ndev->dev_addr[5] << 8)))); 402 } 403 404 /** 405 * netdev_set_mac_address - Write the MAC address (from outside the driver) 406 * @ndev: Pointer to the net_device structure 407 * @p: 6 byte Address to be written as MAC address 408 * 409 * Return: 0 for all conditions. Presently, there is no failure case. 410 * 411 * This function is called to initialize the MAC address of the Axi Ethernet 412 * core. It calls the core specific axienet_set_mac_address. This is the 413 * function that goes into net_device_ops structure entry ndo_set_mac_address. 414 */ 415 static int netdev_set_mac_address(struct net_device *ndev, void *p) 416 { 417 struct sockaddr *addr = p; 418 419 axienet_set_mac_address(ndev, addr->sa_data); 420 return 0; 421 } 422 423 /** 424 * axienet_set_multicast_list - Prepare the multicast table 425 * @ndev: Pointer to the net_device structure 426 * 427 * This function is called to initialize the multicast table during 428 * initialization. The Axi Ethernet basic multicast support has a four-entry 429 * multicast table which is initialized here. Additionally this function 430 * goes into the net_device_ops structure entry ndo_set_multicast_list. This 431 * means whenever the multicast table entries need to be updated this 432 * function gets called. 433 */ 434 static void axienet_set_multicast_list(struct net_device *ndev) 435 { 436 int i = 0; 437 u32 reg, af0reg, af1reg; 438 struct axienet_local *lp = netdev_priv(ndev); 439 440 reg = axienet_ior(lp, XAE_FMI_OFFSET); 441 reg &= ~XAE_FMI_PM_MASK; 442 if (ndev->flags & IFF_PROMISC) 443 reg |= XAE_FMI_PM_MASK; 444 else 445 reg &= ~XAE_FMI_PM_MASK; 446 axienet_iow(lp, XAE_FMI_OFFSET, reg); 447 448 if (ndev->flags & IFF_ALLMULTI || 449 netdev_mc_count(ndev) > XAE_MULTICAST_CAM_TABLE_NUM) { 450 reg &= 0xFFFFFF00; 451 axienet_iow(lp, XAE_FMI_OFFSET, reg); 452 axienet_iow(lp, XAE_AF0_OFFSET, 1); /* Multicast bit */ 453 axienet_iow(lp, XAE_AF1_OFFSET, 0); 454 axienet_iow(lp, XAE_AM0_OFFSET, 1); /* ditto */ 455 axienet_iow(lp, XAE_AM1_OFFSET, 0); 456 axienet_iow(lp, XAE_FFE_OFFSET, 1); 457 i = 1; 458 } else if (!netdev_mc_empty(ndev)) { 459 struct netdev_hw_addr *ha; 460 461 netdev_for_each_mc_addr(ha, ndev) { 462 if (i >= XAE_MULTICAST_CAM_TABLE_NUM) 463 break; 464 465 af0reg = (ha->addr[0]); 466 af0reg |= (ha->addr[1] << 8); 467 af0reg |= (ha->addr[2] << 16); 468 af0reg |= (ha->addr[3] << 24); 469 470 af1reg = (ha->addr[4]); 471 af1reg |= (ha->addr[5] << 8); 472 473 reg &= 0xFFFFFF00; 474 reg |= i; 475 476 axienet_iow(lp, XAE_FMI_OFFSET, reg); 477 axienet_iow(lp, XAE_AF0_OFFSET, af0reg); 478 axienet_iow(lp, XAE_AF1_OFFSET, af1reg); 479 axienet_iow(lp, XAE_AM0_OFFSET, 0xffffffff); 480 axienet_iow(lp, XAE_AM1_OFFSET, 0x0000ffff); 481 axienet_iow(lp, XAE_FFE_OFFSET, 1); 482 i++; 483 } 484 } 485 486 for (; i < XAE_MULTICAST_CAM_TABLE_NUM; i++) { 487 reg &= 0xFFFFFF00; 488 reg |= i; 489 axienet_iow(lp, XAE_FMI_OFFSET, reg); 490 axienet_iow(lp, XAE_FFE_OFFSET, 0); 491 } 492 } 493 494 /** 495 * axienet_setoptions - Set an Axi Ethernet option 496 * @ndev: Pointer to the net_device structure 497 * @options: Option to be enabled/disabled 498 * 499 * The Axi Ethernet core has multiple features which can be selectively turned 500 * on or off. The typical options could be jumbo frame option, basic VLAN 501 * option, promiscuous mode option etc. This function is used to set or clear 502 * these options in the Axi Ethernet hardware. This is done through 503 * axienet_option structure . 504 */ 505 static void axienet_setoptions(struct net_device *ndev, u32 options) 506 { 507 int reg; 508 struct axienet_local *lp = netdev_priv(ndev); 509 struct axienet_option *tp = &axienet_options[0]; 510 511 while (tp->opt) { 512 reg = ((axienet_ior(lp, tp->reg)) & ~(tp->m_or)); 513 if (options & tp->opt) 514 reg |= tp->m_or; 515 axienet_iow(lp, tp->reg, reg); 516 tp++; 517 } 518 519 lp->options |= options; 520 } 521 522 static u64 axienet_stat(struct axienet_local *lp, enum temac_stat stat) 523 { 524 u32 counter; 525 526 if (lp->reset_in_progress) 527 return lp->hw_stat_base[stat]; 528 529 counter = axienet_ior(lp, XAE_STATS_OFFSET + stat * 8); 530 return lp->hw_stat_base[stat] + (counter - lp->hw_last_counter[stat]); 531 } 532 533 static void axienet_stats_update(struct axienet_local *lp, bool reset) 534 { 535 enum temac_stat stat; 536 537 write_seqcount_begin(&lp->hw_stats_seqcount); 538 lp->reset_in_progress = reset; 539 for (stat = 0; stat < STAT_COUNT; stat++) { 540 u32 counter = axienet_ior(lp, XAE_STATS_OFFSET + stat * 8); 541 542 lp->hw_stat_base[stat] += counter - lp->hw_last_counter[stat]; 543 lp->hw_last_counter[stat] = counter; 544 } 545 write_seqcount_end(&lp->hw_stats_seqcount); 546 } 547 548 static void axienet_refresh_stats(struct work_struct *work) 549 { 550 struct axienet_local *lp = container_of(work, struct axienet_local, 551 stats_work.work); 552 553 mutex_lock(&lp->stats_lock); 554 axienet_stats_update(lp, false); 555 mutex_unlock(&lp->stats_lock); 556 557 /* Just less than 2^32 bytes at 2.5 GBit/s */ 558 schedule_delayed_work(&lp->stats_work, 13 * HZ); 559 } 560 561 static int __axienet_device_reset(struct axienet_local *lp) 562 { 563 u32 value; 564 int ret; 565 566 /* Save statistics counters in case they will be reset */ 567 mutex_lock(&lp->stats_lock); 568 if (lp->features & XAE_FEATURE_STATS) 569 axienet_stats_update(lp, true); 570 571 /* Reset Axi DMA. This would reset Axi Ethernet core as well. The reset 572 * process of Axi DMA takes a while to complete as all pending 573 * commands/transfers will be flushed or completed during this 574 * reset process. 575 * Note that even though both TX and RX have their own reset register, 576 * they both reset the entire DMA core, so only one needs to be used. 577 */ 578 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, XAXIDMA_CR_RESET_MASK); 579 ret = read_poll_timeout(axienet_dma_in32, value, 580 !(value & XAXIDMA_CR_RESET_MASK), 581 DELAY_OF_ONE_MILLISEC, 50000, false, lp, 582 XAXIDMA_TX_CR_OFFSET); 583 if (ret) { 584 dev_err(lp->dev, "%s: DMA reset timeout!\n", __func__); 585 goto out; 586 } 587 588 /* Wait for PhyRstCmplt bit to be set, indicating the PHY reset has finished */ 589 ret = read_poll_timeout(axienet_ior, value, 590 value & XAE_INT_PHYRSTCMPLT_MASK, 591 DELAY_OF_ONE_MILLISEC, 50000, false, lp, 592 XAE_IS_OFFSET); 593 if (ret) { 594 dev_err(lp->dev, "%s: timeout waiting for PhyRstCmplt\n", __func__); 595 goto out; 596 } 597 598 /* Update statistics counters with new values */ 599 if (lp->features & XAE_FEATURE_STATS) { 600 enum temac_stat stat; 601 602 write_seqcount_begin(&lp->hw_stats_seqcount); 603 lp->reset_in_progress = false; 604 for (stat = 0; stat < STAT_COUNT; stat++) { 605 u32 counter = 606 axienet_ior(lp, XAE_STATS_OFFSET + stat * 8); 607 608 lp->hw_stat_base[stat] += 609 lp->hw_last_counter[stat] - counter; 610 lp->hw_last_counter[stat] = counter; 611 } 612 write_seqcount_end(&lp->hw_stats_seqcount); 613 } 614 615 out: 616 mutex_unlock(&lp->stats_lock); 617 return ret; 618 } 619 620 /** 621 * axienet_dma_stop - Stop DMA operation 622 * @lp: Pointer to the axienet_local structure 623 */ 624 static void axienet_dma_stop(struct axienet_local *lp) 625 { 626 int count; 627 u32 cr, sr; 628 629 cr = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET); 630 cr &= ~(XAXIDMA_CR_RUNSTOP_MASK | XAXIDMA_IRQ_ALL_MASK); 631 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr); 632 synchronize_irq(lp->rx_irq); 633 634 cr = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET); 635 cr &= ~(XAXIDMA_CR_RUNSTOP_MASK | XAXIDMA_IRQ_ALL_MASK); 636 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr); 637 synchronize_irq(lp->tx_irq); 638 639 /* Give DMAs a chance to halt gracefully */ 640 sr = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 641 for (count = 0; !(sr & XAXIDMA_SR_HALT_MASK) && count < 5; ++count) { 642 msleep(20); 643 sr = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 644 } 645 646 sr = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 647 for (count = 0; !(sr & XAXIDMA_SR_HALT_MASK) && count < 5; ++count) { 648 msleep(20); 649 sr = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 650 } 651 652 /* Do a reset to ensure DMA is really stopped */ 653 axienet_lock_mii(lp); 654 __axienet_device_reset(lp); 655 axienet_unlock_mii(lp); 656 } 657 658 /** 659 * axienet_device_reset - Reset and initialize the Axi Ethernet hardware. 660 * @ndev: Pointer to the net_device structure 661 * 662 * This function is called to reset and initialize the Axi Ethernet core. This 663 * is typically called during initialization. It does a reset of the Axi DMA 664 * Rx/Tx channels and initializes the Axi DMA BDs. Since Axi DMA reset lines 665 * are connected to Axi Ethernet reset lines, this in turn resets the Axi 666 * Ethernet core. No separate hardware reset is done for the Axi Ethernet 667 * core. 668 * Returns 0 on success or a negative error number otherwise. 669 */ 670 static int axienet_device_reset(struct net_device *ndev) 671 { 672 u32 axienet_status; 673 struct axienet_local *lp = netdev_priv(ndev); 674 int ret; 675 676 lp->max_frm_size = XAE_MAX_VLAN_FRAME_SIZE; 677 lp->options |= XAE_OPTION_VLAN; 678 lp->options &= (~XAE_OPTION_JUMBO); 679 680 if (ndev->mtu > XAE_MTU && ndev->mtu <= XAE_JUMBO_MTU) { 681 lp->max_frm_size = ndev->mtu + VLAN_ETH_HLEN + 682 XAE_TRL_SIZE; 683 684 if (lp->max_frm_size <= lp->rxmem) 685 lp->options |= XAE_OPTION_JUMBO; 686 } 687 688 if (!lp->use_dmaengine) { 689 ret = __axienet_device_reset(lp); 690 if (ret) 691 return ret; 692 693 ret = axienet_dma_bd_init(ndev); 694 if (ret) { 695 netdev_err(ndev, "%s: descriptor allocation failed\n", 696 __func__); 697 return ret; 698 } 699 } 700 701 axienet_status = axienet_ior(lp, XAE_RCW1_OFFSET); 702 axienet_status &= ~XAE_RCW1_RX_MASK; 703 axienet_iow(lp, XAE_RCW1_OFFSET, axienet_status); 704 705 axienet_status = axienet_ior(lp, XAE_IP_OFFSET); 706 if (axienet_status & XAE_INT_RXRJECT_MASK) 707 axienet_iow(lp, XAE_IS_OFFSET, XAE_INT_RXRJECT_MASK); 708 axienet_iow(lp, XAE_IE_OFFSET, lp->eth_irq > 0 ? 709 XAE_INT_RECV_ERROR_MASK : 0); 710 711 axienet_iow(lp, XAE_FCC_OFFSET, XAE_FCC_FCRX_MASK); 712 713 /* Sync default options with HW but leave receiver and 714 * transmitter disabled. 715 */ 716 axienet_setoptions(ndev, lp->options & 717 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 718 axienet_set_mac_address(ndev, NULL); 719 axienet_set_multicast_list(ndev); 720 axienet_setoptions(ndev, lp->options); 721 722 netif_trans_update(ndev); 723 724 return 0; 725 } 726 727 /** 728 * axienet_free_tx_chain - Clean up a series of linked TX descriptors. 729 * @lp: Pointer to the axienet_local structure 730 * @first_bd: Index of first descriptor to clean up 731 * @nr_bds: Max number of descriptors to clean up 732 * @force: Whether to clean descriptors even if not complete 733 * @sizep: Pointer to a u32 filled with the total sum of all bytes 734 * in all cleaned-up descriptors. Ignored if NULL. 735 * @budget: NAPI budget (use 0 when not called from NAPI poll) 736 * 737 * Would either be called after a successful transmit operation, or after 738 * there was an error when setting up the chain. 739 * Returns the number of descriptors handled. 740 */ 741 static int axienet_free_tx_chain(struct axienet_local *lp, u32 first_bd, 742 int nr_bds, bool force, u32 *sizep, int budget) 743 { 744 struct axidma_bd *cur_p; 745 unsigned int status; 746 dma_addr_t phys; 747 int i; 748 749 for (i = 0; i < nr_bds; i++) { 750 cur_p = &lp->tx_bd_v[(first_bd + i) % lp->tx_bd_num]; 751 status = cur_p->status; 752 753 /* If force is not specified, clean up only descriptors 754 * that have been completed by the MAC. 755 */ 756 if (!force && !(status & XAXIDMA_BD_STS_COMPLETE_MASK)) 757 break; 758 759 /* Ensure we see complete descriptor update */ 760 dma_rmb(); 761 phys = desc_get_phys_addr(lp, cur_p); 762 dma_unmap_single(lp->dev, phys, 763 (cur_p->cntrl & XAXIDMA_BD_CTRL_LENGTH_MASK), 764 DMA_TO_DEVICE); 765 766 if (cur_p->skb && (status & XAXIDMA_BD_STS_COMPLETE_MASK)) 767 napi_consume_skb(cur_p->skb, budget); 768 769 cur_p->app0 = 0; 770 cur_p->app1 = 0; 771 cur_p->app2 = 0; 772 cur_p->app4 = 0; 773 cur_p->skb = NULL; 774 /* ensure our transmit path and device don't prematurely see status cleared */ 775 wmb(); 776 cur_p->cntrl = 0; 777 cur_p->status = 0; 778 779 if (sizep) 780 *sizep += status & XAXIDMA_BD_STS_ACTUAL_LEN_MASK; 781 } 782 783 return i; 784 } 785 786 /** 787 * axienet_check_tx_bd_space - Checks if a BD/group of BDs are currently busy 788 * @lp: Pointer to the axienet_local structure 789 * @num_frag: The number of BDs to check for 790 * 791 * Return: 0, on success 792 * NETDEV_TX_BUSY, if any of the descriptors are not free 793 * 794 * This function is invoked before BDs are allocated and transmission starts. 795 * This function returns 0 if a BD or group of BDs can be allocated for 796 * transmission. If the BD or any of the BDs are not free the function 797 * returns a busy status. 798 */ 799 static inline int axienet_check_tx_bd_space(struct axienet_local *lp, 800 int num_frag) 801 { 802 struct axidma_bd *cur_p; 803 804 /* Ensure we see all descriptor updates from device or TX polling */ 805 rmb(); 806 cur_p = &lp->tx_bd_v[(READ_ONCE(lp->tx_bd_tail) + num_frag) % 807 lp->tx_bd_num]; 808 if (cur_p->cntrl) 809 return NETDEV_TX_BUSY; 810 return 0; 811 } 812 813 /** 814 * axienet_dma_tx_cb - DMA engine callback for TX channel. 815 * @data: Pointer to the axienet_local structure. 816 * @result: error reporting through dmaengine_result. 817 * This function is called by dmaengine driver for TX channel to notify 818 * that the transmit is done. 819 */ 820 static void axienet_dma_tx_cb(void *data, const struct dmaengine_result *result) 821 { 822 struct skbuf_dma_descriptor *skbuf_dma; 823 struct axienet_local *lp = data; 824 struct netdev_queue *txq; 825 int len; 826 827 skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_tail++); 828 len = skbuf_dma->skb->len; 829 txq = skb_get_tx_queue(lp->ndev, skbuf_dma->skb); 830 u64_stats_update_begin(&lp->tx_stat_sync); 831 u64_stats_add(&lp->tx_bytes, len); 832 u64_stats_add(&lp->tx_packets, 1); 833 u64_stats_update_end(&lp->tx_stat_sync); 834 dma_unmap_sg(lp->dev, skbuf_dma->sgl, skbuf_dma->sg_len, DMA_TO_DEVICE); 835 dev_consume_skb_any(skbuf_dma->skb); 836 netif_txq_completed_wake(txq, 1, len, 837 CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX), 838 2 * MAX_SKB_FRAGS); 839 } 840 841 /** 842 * axienet_start_xmit_dmaengine - Starts the transmission. 843 * @skb: sk_buff pointer that contains data to be Txed. 844 * @ndev: Pointer to net_device structure. 845 * 846 * Return: NETDEV_TX_OK on success or any non space errors. 847 * NETDEV_TX_BUSY when free element in TX skb ring buffer 848 * is not available. 849 * 850 * This function is invoked to initiate transmission. The 851 * function sets the skbs, register dma callback API and submit 852 * the dma transaction. 853 * Additionally if checksum offloading is supported, 854 * it populates AXI Stream Control fields with appropriate values. 855 */ 856 static netdev_tx_t 857 axienet_start_xmit_dmaengine(struct sk_buff *skb, struct net_device *ndev) 858 { 859 struct dma_async_tx_descriptor *dma_tx_desc = NULL; 860 struct axienet_local *lp = netdev_priv(ndev); 861 u32 app_metadata[DMA_NUM_APP_WORDS] = {0}; 862 struct skbuf_dma_descriptor *skbuf_dma; 863 struct dma_device *dma_dev; 864 struct netdev_queue *txq; 865 u32 csum_start_off; 866 u32 csum_index_off; 867 int sg_len; 868 int ret; 869 870 dma_dev = lp->tx_chan->device; 871 sg_len = skb_shinfo(skb)->nr_frags + 1; 872 if (CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX) <= sg_len) { 873 netif_stop_queue(ndev); 874 if (net_ratelimit()) 875 netdev_warn(ndev, "TX ring unexpectedly full\n"); 876 return NETDEV_TX_BUSY; 877 } 878 879 skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_head); 880 if (!skbuf_dma) 881 goto xmit_error_drop_skb; 882 883 lp->tx_ring_head++; 884 sg_init_table(skbuf_dma->sgl, sg_len); 885 ret = skb_to_sgvec(skb, skbuf_dma->sgl, 0, skb->len); 886 if (ret < 0) 887 goto xmit_error_drop_skb; 888 889 ret = dma_map_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE); 890 if (!ret) 891 goto xmit_error_drop_skb; 892 893 /* Fill up app fields for checksum */ 894 if (skb->ip_summed == CHECKSUM_PARTIAL) { 895 if (lp->features & XAE_FEATURE_FULL_TX_CSUM) { 896 /* Tx Full Checksum Offload Enabled */ 897 app_metadata[0] |= 2; 898 } else if (lp->features & XAE_FEATURE_PARTIAL_TX_CSUM) { 899 csum_start_off = skb_transport_offset(skb); 900 csum_index_off = csum_start_off + skb->csum_offset; 901 /* Tx Partial Checksum Offload Enabled */ 902 app_metadata[0] |= 1; 903 app_metadata[1] = (csum_start_off << 16) | csum_index_off; 904 } 905 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 906 app_metadata[0] |= 2; /* Tx Full Checksum Offload Enabled */ 907 } 908 909 dma_tx_desc = dma_dev->device_prep_slave_sg(lp->tx_chan, skbuf_dma->sgl, 910 sg_len, DMA_MEM_TO_DEV, 911 DMA_PREP_INTERRUPT, (void *)app_metadata); 912 if (!dma_tx_desc) 913 goto xmit_error_unmap_sg; 914 915 skbuf_dma->skb = skb; 916 skbuf_dma->sg_len = sg_len; 917 dma_tx_desc->callback_param = lp; 918 dma_tx_desc->callback_result = axienet_dma_tx_cb; 919 dmaengine_submit(dma_tx_desc); 920 dma_async_issue_pending(lp->tx_chan); 921 txq = skb_get_tx_queue(lp->ndev, skb); 922 netdev_tx_sent_queue(txq, skb->len); 923 netif_txq_maybe_stop(txq, CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX), 924 MAX_SKB_FRAGS + 1, 2 * MAX_SKB_FRAGS); 925 926 return NETDEV_TX_OK; 927 928 xmit_error_unmap_sg: 929 dma_unmap_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE); 930 xmit_error_drop_skb: 931 dev_kfree_skb_any(skb); 932 return NETDEV_TX_OK; 933 } 934 935 /** 936 * axienet_tx_poll - Invoked once a transmit is completed by the 937 * Axi DMA Tx channel. 938 * @napi: Pointer to NAPI structure. 939 * @budget: Max number of TX packets to process. 940 * 941 * Return: Number of TX packets processed. 942 * 943 * This function is invoked from the NAPI processing to notify the completion 944 * of transmit operation. It clears fields in the corresponding Tx BDs and 945 * unmaps the corresponding buffer so that CPU can regain ownership of the 946 * buffer. It finally invokes "netif_wake_queue" to restart transmission if 947 * required. 948 */ 949 static int axienet_tx_poll(struct napi_struct *napi, int budget) 950 { 951 struct axienet_local *lp = container_of(napi, struct axienet_local, napi_tx); 952 struct net_device *ndev = lp->ndev; 953 u32 size = 0; 954 int packets; 955 956 packets = axienet_free_tx_chain(lp, lp->tx_bd_ci, budget, false, &size, budget); 957 958 if (packets) { 959 lp->tx_bd_ci += packets; 960 if (lp->tx_bd_ci >= lp->tx_bd_num) 961 lp->tx_bd_ci %= lp->tx_bd_num; 962 963 u64_stats_update_begin(&lp->tx_stat_sync); 964 u64_stats_add(&lp->tx_packets, packets); 965 u64_stats_add(&lp->tx_bytes, size); 966 u64_stats_update_end(&lp->tx_stat_sync); 967 968 /* Matches barrier in axienet_start_xmit */ 969 smp_mb(); 970 971 if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) 972 netif_wake_queue(ndev); 973 } 974 975 if (packets < budget && napi_complete_done(napi, packets)) { 976 /* Re-enable TX completion interrupts. This should 977 * cause an immediate interrupt if any TX packets are 978 * already pending. 979 */ 980 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr); 981 } 982 return packets; 983 } 984 985 /** 986 * axienet_start_xmit - Starts the transmission. 987 * @skb: sk_buff pointer that contains data to be Txed. 988 * @ndev: Pointer to net_device structure. 989 * 990 * Return: NETDEV_TX_OK, on success 991 * NETDEV_TX_BUSY, if any of the descriptors are not free 992 * 993 * This function is invoked from upper layers to initiate transmission. The 994 * function uses the next available free BDs and populates their fields to 995 * start the transmission. Additionally if checksum offloading is supported, 996 * it populates AXI Stream Control fields with appropriate values. 997 */ 998 static netdev_tx_t 999 axienet_start_xmit(struct sk_buff *skb, struct net_device *ndev) 1000 { 1001 u32 ii; 1002 u32 num_frag; 1003 u32 csum_start_off; 1004 u32 csum_index_off; 1005 skb_frag_t *frag; 1006 dma_addr_t tail_p, phys; 1007 u32 orig_tail_ptr, new_tail_ptr; 1008 struct axienet_local *lp = netdev_priv(ndev); 1009 struct axidma_bd *cur_p; 1010 1011 orig_tail_ptr = lp->tx_bd_tail; 1012 new_tail_ptr = orig_tail_ptr; 1013 1014 num_frag = skb_shinfo(skb)->nr_frags; 1015 cur_p = &lp->tx_bd_v[orig_tail_ptr]; 1016 1017 if (axienet_check_tx_bd_space(lp, num_frag + 1)) { 1018 /* Should not happen as last start_xmit call should have 1019 * checked for sufficient space and queue should only be 1020 * woken when sufficient space is available. 1021 */ 1022 netif_stop_queue(ndev); 1023 if (net_ratelimit()) 1024 netdev_warn(ndev, "TX ring unexpectedly full\n"); 1025 return NETDEV_TX_BUSY; 1026 } 1027 1028 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1029 if (lp->features & XAE_FEATURE_FULL_TX_CSUM) { 1030 /* Tx Full Checksum Offload Enabled */ 1031 cur_p->app0 |= 2; 1032 } else if (lp->features & XAE_FEATURE_PARTIAL_TX_CSUM) { 1033 csum_start_off = skb_transport_offset(skb); 1034 csum_index_off = csum_start_off + skb->csum_offset; 1035 /* Tx Partial Checksum Offload Enabled */ 1036 cur_p->app0 |= 1; 1037 cur_p->app1 = (csum_start_off << 16) | csum_index_off; 1038 } 1039 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 1040 cur_p->app0 |= 2; /* Tx Full Checksum Offload Enabled */ 1041 } 1042 1043 phys = dma_map_single(lp->dev, skb->data, 1044 skb_headlen(skb), DMA_TO_DEVICE); 1045 if (unlikely(dma_mapping_error(lp->dev, phys))) { 1046 if (net_ratelimit()) 1047 netdev_err(ndev, "TX DMA mapping error\n"); 1048 ndev->stats.tx_dropped++; 1049 return NETDEV_TX_OK; 1050 } 1051 desc_set_phys_addr(lp, phys, cur_p); 1052 cur_p->cntrl = skb_headlen(skb) | XAXIDMA_BD_CTRL_TXSOF_MASK; 1053 1054 for (ii = 0; ii < num_frag; ii++) { 1055 if (++new_tail_ptr >= lp->tx_bd_num) 1056 new_tail_ptr = 0; 1057 cur_p = &lp->tx_bd_v[new_tail_ptr]; 1058 frag = &skb_shinfo(skb)->frags[ii]; 1059 phys = dma_map_single(lp->dev, 1060 skb_frag_address(frag), 1061 skb_frag_size(frag), 1062 DMA_TO_DEVICE); 1063 if (unlikely(dma_mapping_error(lp->dev, phys))) { 1064 if (net_ratelimit()) 1065 netdev_err(ndev, "TX DMA mapping error\n"); 1066 ndev->stats.tx_dropped++; 1067 axienet_free_tx_chain(lp, orig_tail_ptr, ii + 1, 1068 true, NULL, 0); 1069 return NETDEV_TX_OK; 1070 } 1071 desc_set_phys_addr(lp, phys, cur_p); 1072 cur_p->cntrl = skb_frag_size(frag); 1073 } 1074 1075 cur_p->cntrl |= XAXIDMA_BD_CTRL_TXEOF_MASK; 1076 cur_p->skb = skb; 1077 1078 tail_p = lp->tx_bd_p + sizeof(*lp->tx_bd_v) * new_tail_ptr; 1079 if (++new_tail_ptr >= lp->tx_bd_num) 1080 new_tail_ptr = 0; 1081 WRITE_ONCE(lp->tx_bd_tail, new_tail_ptr); 1082 1083 /* Start the transfer */ 1084 axienet_dma_out_addr(lp, XAXIDMA_TX_TDESC_OFFSET, tail_p); 1085 1086 /* Stop queue if next transmit may not have space */ 1087 if (axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) { 1088 netif_stop_queue(ndev); 1089 1090 /* Matches barrier in axienet_tx_poll */ 1091 smp_mb(); 1092 1093 /* Space might have just been freed - check again */ 1094 if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) 1095 netif_wake_queue(ndev); 1096 } 1097 1098 return NETDEV_TX_OK; 1099 } 1100 1101 /** 1102 * axienet_dma_rx_cb - DMA engine callback for RX channel. 1103 * @data: Pointer to the skbuf_dma_descriptor structure. 1104 * @result: error reporting through dmaengine_result. 1105 * This function is called by dmaengine driver for RX channel to notify 1106 * that the packet is received. 1107 */ 1108 static void axienet_dma_rx_cb(void *data, const struct dmaengine_result *result) 1109 { 1110 struct skbuf_dma_descriptor *skbuf_dma; 1111 size_t meta_len, meta_max_len, rx_len; 1112 struct axienet_local *lp = data; 1113 struct sk_buff *skb; 1114 u32 *app_metadata; 1115 1116 skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_tail++); 1117 skb = skbuf_dma->skb; 1118 app_metadata = dmaengine_desc_get_metadata_ptr(skbuf_dma->desc, &meta_len, 1119 &meta_max_len); 1120 dma_unmap_single(lp->dev, skbuf_dma->dma_address, lp->max_frm_size, 1121 DMA_FROM_DEVICE); 1122 /* TODO: Derive app word index programmatically */ 1123 rx_len = (app_metadata[LEN_APP] & 0xFFFF); 1124 skb_put(skb, rx_len); 1125 skb->protocol = eth_type_trans(skb, lp->ndev); 1126 skb->ip_summed = CHECKSUM_NONE; 1127 1128 __netif_rx(skb); 1129 u64_stats_update_begin(&lp->rx_stat_sync); 1130 u64_stats_add(&lp->rx_packets, 1); 1131 u64_stats_add(&lp->rx_bytes, rx_len); 1132 u64_stats_update_end(&lp->rx_stat_sync); 1133 axienet_rx_submit_desc(lp->ndev); 1134 dma_async_issue_pending(lp->rx_chan); 1135 } 1136 1137 /** 1138 * axienet_rx_poll - Triggered by RX ISR to complete the BD processing. 1139 * @napi: Pointer to NAPI structure. 1140 * @budget: Max number of RX packets to process. 1141 * 1142 * Return: Number of RX packets processed. 1143 */ 1144 static int axienet_rx_poll(struct napi_struct *napi, int budget) 1145 { 1146 u32 length; 1147 u32 csumstatus; 1148 u32 size = 0; 1149 int packets = 0; 1150 dma_addr_t tail_p = 0; 1151 struct axidma_bd *cur_p; 1152 struct sk_buff *skb, *new_skb; 1153 struct axienet_local *lp = container_of(napi, struct axienet_local, napi_rx); 1154 1155 cur_p = &lp->rx_bd_v[lp->rx_bd_ci]; 1156 1157 while (packets < budget && (cur_p->status & XAXIDMA_BD_STS_COMPLETE_MASK)) { 1158 dma_addr_t phys; 1159 1160 /* Ensure we see complete descriptor update */ 1161 dma_rmb(); 1162 1163 skb = cur_p->skb; 1164 cur_p->skb = NULL; 1165 1166 /* skb could be NULL if a previous pass already received the 1167 * packet for this slot in the ring, but failed to refill it 1168 * with a newly allocated buffer. In this case, don't try to 1169 * receive it again. 1170 */ 1171 if (likely(skb)) { 1172 length = cur_p->app4 & 0x0000FFFF; 1173 1174 phys = desc_get_phys_addr(lp, cur_p); 1175 dma_unmap_single(lp->dev, phys, lp->max_frm_size, 1176 DMA_FROM_DEVICE); 1177 1178 skb_put(skb, length); 1179 skb->protocol = eth_type_trans(skb, lp->ndev); 1180 /*skb_checksum_none_assert(skb);*/ 1181 skb->ip_summed = CHECKSUM_NONE; 1182 1183 /* if we're doing Rx csum offload, set it up */ 1184 if (lp->features & XAE_FEATURE_FULL_RX_CSUM) { 1185 csumstatus = (cur_p->app2 & 1186 XAE_FULL_CSUM_STATUS_MASK) >> 3; 1187 if (csumstatus == XAE_IP_TCP_CSUM_VALIDATED || 1188 csumstatus == XAE_IP_UDP_CSUM_VALIDATED) { 1189 skb->ip_summed = CHECKSUM_UNNECESSARY; 1190 } 1191 } else if ((lp->features & XAE_FEATURE_PARTIAL_RX_CSUM) != 0 && 1192 skb->protocol == htons(ETH_P_IP) && 1193 skb->len > 64) { 1194 skb->csum = be32_to_cpu(cur_p->app3 & 0xFFFF); 1195 skb->ip_summed = CHECKSUM_COMPLETE; 1196 } 1197 1198 napi_gro_receive(napi, skb); 1199 1200 size += length; 1201 packets++; 1202 } 1203 1204 new_skb = napi_alloc_skb(napi, lp->max_frm_size); 1205 if (!new_skb) 1206 break; 1207 1208 phys = dma_map_single(lp->dev, new_skb->data, 1209 lp->max_frm_size, 1210 DMA_FROM_DEVICE); 1211 if (unlikely(dma_mapping_error(lp->dev, phys))) { 1212 if (net_ratelimit()) 1213 netdev_err(lp->ndev, "RX DMA mapping error\n"); 1214 dev_kfree_skb(new_skb); 1215 break; 1216 } 1217 desc_set_phys_addr(lp, phys, cur_p); 1218 1219 cur_p->cntrl = lp->max_frm_size; 1220 cur_p->status = 0; 1221 cur_p->skb = new_skb; 1222 1223 /* Only update tail_p to mark this slot as usable after it has 1224 * been successfully refilled. 1225 */ 1226 tail_p = lp->rx_bd_p + sizeof(*lp->rx_bd_v) * lp->rx_bd_ci; 1227 1228 if (++lp->rx_bd_ci >= lp->rx_bd_num) 1229 lp->rx_bd_ci = 0; 1230 cur_p = &lp->rx_bd_v[lp->rx_bd_ci]; 1231 } 1232 1233 u64_stats_update_begin(&lp->rx_stat_sync); 1234 u64_stats_add(&lp->rx_packets, packets); 1235 u64_stats_add(&lp->rx_bytes, size); 1236 u64_stats_update_end(&lp->rx_stat_sync); 1237 1238 if (tail_p) 1239 axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, tail_p); 1240 1241 if (packets < budget && napi_complete_done(napi, packets)) { 1242 /* Re-enable RX completion interrupts. This should 1243 * cause an immediate interrupt if any RX packets are 1244 * already pending. 1245 */ 1246 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr); 1247 } 1248 return packets; 1249 } 1250 1251 /** 1252 * axienet_tx_irq - Tx Done Isr. 1253 * @irq: irq number 1254 * @_ndev: net_device pointer 1255 * 1256 * Return: IRQ_HANDLED if device generated a TX interrupt, IRQ_NONE otherwise. 1257 * 1258 * This is the Axi DMA Tx done Isr. It invokes NAPI polling to complete the 1259 * TX BD processing. 1260 */ 1261 static irqreturn_t axienet_tx_irq(int irq, void *_ndev) 1262 { 1263 unsigned int status; 1264 struct net_device *ndev = _ndev; 1265 struct axienet_local *lp = netdev_priv(ndev); 1266 1267 status = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 1268 1269 if (!(status & XAXIDMA_IRQ_ALL_MASK)) 1270 return IRQ_NONE; 1271 1272 axienet_dma_out32(lp, XAXIDMA_TX_SR_OFFSET, status); 1273 1274 if (unlikely(status & XAXIDMA_IRQ_ERROR_MASK)) { 1275 netdev_err(ndev, "DMA Tx error 0x%x\n", status); 1276 netdev_err(ndev, "Current BD is at: 0x%x%08x\n", 1277 (lp->tx_bd_v[lp->tx_bd_ci]).phys_msb, 1278 (lp->tx_bd_v[lp->tx_bd_ci]).phys); 1279 schedule_work(&lp->dma_err_task); 1280 } else { 1281 /* Disable further TX completion interrupts and schedule 1282 * NAPI to handle the completions. 1283 */ 1284 u32 cr = lp->tx_dma_cr; 1285 1286 cr &= ~(XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_DELAY_MASK); 1287 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr); 1288 1289 napi_schedule(&lp->napi_tx); 1290 } 1291 1292 return IRQ_HANDLED; 1293 } 1294 1295 /** 1296 * axienet_rx_irq - Rx Isr. 1297 * @irq: irq number 1298 * @_ndev: net_device pointer 1299 * 1300 * Return: IRQ_HANDLED if device generated a RX interrupt, IRQ_NONE otherwise. 1301 * 1302 * This is the Axi DMA Rx Isr. It invokes NAPI polling to complete the RX BD 1303 * processing. 1304 */ 1305 static irqreturn_t axienet_rx_irq(int irq, void *_ndev) 1306 { 1307 unsigned int status; 1308 struct net_device *ndev = _ndev; 1309 struct axienet_local *lp = netdev_priv(ndev); 1310 1311 status = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 1312 1313 if (!(status & XAXIDMA_IRQ_ALL_MASK)) 1314 return IRQ_NONE; 1315 1316 axienet_dma_out32(lp, XAXIDMA_RX_SR_OFFSET, status); 1317 1318 if (unlikely(status & XAXIDMA_IRQ_ERROR_MASK)) { 1319 netdev_err(ndev, "DMA Rx error 0x%x\n", status); 1320 netdev_err(ndev, "Current BD is at: 0x%x%08x\n", 1321 (lp->rx_bd_v[lp->rx_bd_ci]).phys_msb, 1322 (lp->rx_bd_v[lp->rx_bd_ci]).phys); 1323 schedule_work(&lp->dma_err_task); 1324 } else { 1325 /* Disable further RX completion interrupts and schedule 1326 * NAPI receive. 1327 */ 1328 u32 cr = lp->rx_dma_cr; 1329 1330 cr &= ~(XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_DELAY_MASK); 1331 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr); 1332 1333 napi_schedule(&lp->napi_rx); 1334 } 1335 1336 return IRQ_HANDLED; 1337 } 1338 1339 /** 1340 * axienet_eth_irq - Ethernet core Isr. 1341 * @irq: irq number 1342 * @_ndev: net_device pointer 1343 * 1344 * Return: IRQ_HANDLED if device generated a core interrupt, IRQ_NONE otherwise. 1345 * 1346 * Handle miscellaneous conditions indicated by Ethernet core IRQ. 1347 */ 1348 static irqreturn_t axienet_eth_irq(int irq, void *_ndev) 1349 { 1350 struct net_device *ndev = _ndev; 1351 struct axienet_local *lp = netdev_priv(ndev); 1352 unsigned int pending; 1353 1354 pending = axienet_ior(lp, XAE_IP_OFFSET); 1355 if (!pending) 1356 return IRQ_NONE; 1357 1358 if (pending & XAE_INT_RXFIFOOVR_MASK) 1359 ndev->stats.rx_missed_errors++; 1360 1361 if (pending & XAE_INT_RXRJECT_MASK) 1362 ndev->stats.rx_dropped++; 1363 1364 axienet_iow(lp, XAE_IS_OFFSET, pending); 1365 return IRQ_HANDLED; 1366 } 1367 1368 static void axienet_dma_err_handler(struct work_struct *work); 1369 1370 /** 1371 * axienet_rx_submit_desc - Submit the rx descriptors to dmaengine. 1372 * allocate skbuff, map the scatterlist and obtain a descriptor 1373 * and then add the callback information and submit descriptor. 1374 * 1375 * @ndev: net_device pointer 1376 * 1377 */ 1378 static void axienet_rx_submit_desc(struct net_device *ndev) 1379 { 1380 struct dma_async_tx_descriptor *dma_rx_desc = NULL; 1381 struct axienet_local *lp = netdev_priv(ndev); 1382 struct skbuf_dma_descriptor *skbuf_dma; 1383 struct sk_buff *skb; 1384 dma_addr_t addr; 1385 1386 skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_head); 1387 if (!skbuf_dma) 1388 return; 1389 1390 lp->rx_ring_head++; 1391 skb = netdev_alloc_skb(ndev, lp->max_frm_size); 1392 if (!skb) 1393 return; 1394 1395 sg_init_table(skbuf_dma->sgl, 1); 1396 addr = dma_map_single(lp->dev, skb->data, lp->max_frm_size, DMA_FROM_DEVICE); 1397 if (unlikely(dma_mapping_error(lp->dev, addr))) { 1398 if (net_ratelimit()) 1399 netdev_err(ndev, "DMA mapping error\n"); 1400 goto rx_submit_err_free_skb; 1401 } 1402 sg_dma_address(skbuf_dma->sgl) = addr; 1403 sg_dma_len(skbuf_dma->sgl) = lp->max_frm_size; 1404 dma_rx_desc = dmaengine_prep_slave_sg(lp->rx_chan, skbuf_dma->sgl, 1405 1, DMA_DEV_TO_MEM, 1406 DMA_PREP_INTERRUPT); 1407 if (!dma_rx_desc) 1408 goto rx_submit_err_unmap_skb; 1409 1410 skbuf_dma->skb = skb; 1411 skbuf_dma->dma_address = sg_dma_address(skbuf_dma->sgl); 1412 skbuf_dma->desc = dma_rx_desc; 1413 dma_rx_desc->callback_param = lp; 1414 dma_rx_desc->callback_result = axienet_dma_rx_cb; 1415 dmaengine_submit(dma_rx_desc); 1416 1417 return; 1418 1419 rx_submit_err_unmap_skb: 1420 dma_unmap_single(lp->dev, addr, lp->max_frm_size, DMA_FROM_DEVICE); 1421 rx_submit_err_free_skb: 1422 dev_kfree_skb(skb); 1423 } 1424 1425 /** 1426 * axienet_init_dmaengine - init the dmaengine code. 1427 * @ndev: Pointer to net_device structure 1428 * 1429 * Return: 0, on success. 1430 * non-zero error value on failure 1431 * 1432 * This is the dmaengine initialization code. 1433 */ 1434 static int axienet_init_dmaengine(struct net_device *ndev) 1435 { 1436 struct axienet_local *lp = netdev_priv(ndev); 1437 struct skbuf_dma_descriptor *skbuf_dma; 1438 int i, ret; 1439 1440 lp->tx_chan = dma_request_chan(lp->dev, "tx_chan0"); 1441 if (IS_ERR(lp->tx_chan)) { 1442 dev_err(lp->dev, "No Ethernet DMA (TX) channel found\n"); 1443 return PTR_ERR(lp->tx_chan); 1444 } 1445 1446 lp->rx_chan = dma_request_chan(lp->dev, "rx_chan0"); 1447 if (IS_ERR(lp->rx_chan)) { 1448 ret = PTR_ERR(lp->rx_chan); 1449 dev_err(lp->dev, "No Ethernet DMA (RX) channel found\n"); 1450 goto err_dma_release_tx; 1451 } 1452 1453 lp->tx_ring_tail = 0; 1454 lp->tx_ring_head = 0; 1455 lp->rx_ring_tail = 0; 1456 lp->rx_ring_head = 0; 1457 lp->tx_skb_ring = kcalloc(TX_BD_NUM_MAX, sizeof(*lp->tx_skb_ring), 1458 GFP_KERNEL); 1459 if (!lp->tx_skb_ring) { 1460 ret = -ENOMEM; 1461 goto err_dma_release_rx; 1462 } 1463 for (i = 0; i < TX_BD_NUM_MAX; i++) { 1464 skbuf_dma = kzalloc(sizeof(*skbuf_dma), GFP_KERNEL); 1465 if (!skbuf_dma) { 1466 ret = -ENOMEM; 1467 goto err_free_tx_skb_ring; 1468 } 1469 lp->tx_skb_ring[i] = skbuf_dma; 1470 } 1471 1472 lp->rx_skb_ring = kcalloc(RX_BUF_NUM_DEFAULT, sizeof(*lp->rx_skb_ring), 1473 GFP_KERNEL); 1474 if (!lp->rx_skb_ring) { 1475 ret = -ENOMEM; 1476 goto err_free_tx_skb_ring; 1477 } 1478 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) { 1479 skbuf_dma = kzalloc(sizeof(*skbuf_dma), GFP_KERNEL); 1480 if (!skbuf_dma) { 1481 ret = -ENOMEM; 1482 goto err_free_rx_skb_ring; 1483 } 1484 lp->rx_skb_ring[i] = skbuf_dma; 1485 } 1486 /* TODO: Instead of BD_NUM_DEFAULT use runtime support */ 1487 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) 1488 axienet_rx_submit_desc(ndev); 1489 dma_async_issue_pending(lp->rx_chan); 1490 1491 return 0; 1492 1493 err_free_rx_skb_ring: 1494 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) 1495 kfree(lp->rx_skb_ring[i]); 1496 kfree(lp->rx_skb_ring); 1497 err_free_tx_skb_ring: 1498 for (i = 0; i < TX_BD_NUM_MAX; i++) 1499 kfree(lp->tx_skb_ring[i]); 1500 kfree(lp->tx_skb_ring); 1501 err_dma_release_rx: 1502 dma_release_channel(lp->rx_chan); 1503 err_dma_release_tx: 1504 dma_release_channel(lp->tx_chan); 1505 return ret; 1506 } 1507 1508 /** 1509 * axienet_init_legacy_dma - init the dma legacy code. 1510 * @ndev: Pointer to net_device structure 1511 * 1512 * Return: 0, on success. 1513 * non-zero error value on failure 1514 * 1515 * This is the dma initialization code. It also allocates interrupt 1516 * service routines, enables the interrupt lines and ISR handling. 1517 * 1518 */ 1519 static int axienet_init_legacy_dma(struct net_device *ndev) 1520 { 1521 int ret; 1522 struct axienet_local *lp = netdev_priv(ndev); 1523 1524 /* Enable worker thread for Axi DMA error handling */ 1525 INIT_WORK(&lp->dma_err_task, axienet_dma_err_handler); 1526 1527 napi_enable(&lp->napi_rx); 1528 napi_enable(&lp->napi_tx); 1529 1530 /* Enable interrupts for Axi DMA Tx */ 1531 ret = request_irq(lp->tx_irq, axienet_tx_irq, IRQF_SHARED, 1532 ndev->name, ndev); 1533 if (ret) 1534 goto err_tx_irq; 1535 /* Enable interrupts for Axi DMA Rx */ 1536 ret = request_irq(lp->rx_irq, axienet_rx_irq, IRQF_SHARED, 1537 ndev->name, ndev); 1538 if (ret) 1539 goto err_rx_irq; 1540 /* Enable interrupts for Axi Ethernet core (if defined) */ 1541 if (lp->eth_irq > 0) { 1542 ret = request_irq(lp->eth_irq, axienet_eth_irq, IRQF_SHARED, 1543 ndev->name, ndev); 1544 if (ret) 1545 goto err_eth_irq; 1546 } 1547 1548 return 0; 1549 1550 err_eth_irq: 1551 free_irq(lp->rx_irq, ndev); 1552 err_rx_irq: 1553 free_irq(lp->tx_irq, ndev); 1554 err_tx_irq: 1555 napi_disable(&lp->napi_tx); 1556 napi_disable(&lp->napi_rx); 1557 cancel_work_sync(&lp->dma_err_task); 1558 dev_err(lp->dev, "request_irq() failed\n"); 1559 return ret; 1560 } 1561 1562 /** 1563 * axienet_open - Driver open routine. 1564 * @ndev: Pointer to net_device structure 1565 * 1566 * Return: 0, on success. 1567 * non-zero error value on failure 1568 * 1569 * This is the driver open routine. It calls phylink_start to start the 1570 * PHY device. 1571 * It also allocates interrupt service routines, enables the interrupt lines 1572 * and ISR handling. Axi Ethernet core is reset through Axi DMA core. Buffer 1573 * descriptors are initialized. 1574 */ 1575 static int axienet_open(struct net_device *ndev) 1576 { 1577 int ret; 1578 struct axienet_local *lp = netdev_priv(ndev); 1579 1580 /* When we do an Axi Ethernet reset, it resets the complete core 1581 * including the MDIO. MDIO must be disabled before resetting. 1582 * Hold MDIO bus lock to avoid MDIO accesses during the reset. 1583 */ 1584 axienet_lock_mii(lp); 1585 ret = axienet_device_reset(ndev); 1586 axienet_unlock_mii(lp); 1587 1588 ret = phylink_of_phy_connect(lp->phylink, lp->dev->of_node, 0); 1589 if (ret) { 1590 dev_err(lp->dev, "phylink_of_phy_connect() failed: %d\n", ret); 1591 return ret; 1592 } 1593 1594 phylink_start(lp->phylink); 1595 1596 /* Start the statistics refresh work */ 1597 schedule_delayed_work(&lp->stats_work, 0); 1598 1599 if (lp->use_dmaengine) { 1600 /* Enable interrupts for Axi Ethernet core (if defined) */ 1601 if (lp->eth_irq > 0) { 1602 ret = request_irq(lp->eth_irq, axienet_eth_irq, IRQF_SHARED, 1603 ndev->name, ndev); 1604 if (ret) 1605 goto err_phy; 1606 } 1607 1608 ret = axienet_init_dmaengine(ndev); 1609 if (ret < 0) 1610 goto err_free_eth_irq; 1611 } else { 1612 ret = axienet_init_legacy_dma(ndev); 1613 if (ret) 1614 goto err_phy; 1615 } 1616 1617 return 0; 1618 1619 err_free_eth_irq: 1620 if (lp->eth_irq > 0) 1621 free_irq(lp->eth_irq, ndev); 1622 err_phy: 1623 cancel_delayed_work_sync(&lp->stats_work); 1624 phylink_stop(lp->phylink); 1625 phylink_disconnect_phy(lp->phylink); 1626 return ret; 1627 } 1628 1629 /** 1630 * axienet_stop - Driver stop routine. 1631 * @ndev: Pointer to net_device structure 1632 * 1633 * Return: 0, on success. 1634 * 1635 * This is the driver stop routine. It calls phylink_disconnect to stop the PHY 1636 * device. It also removes the interrupt handlers and disables the interrupts. 1637 * The Axi DMA Tx/Rx BDs are released. 1638 */ 1639 static int axienet_stop(struct net_device *ndev) 1640 { 1641 struct axienet_local *lp = netdev_priv(ndev); 1642 int i; 1643 1644 if (!lp->use_dmaengine) { 1645 napi_disable(&lp->napi_tx); 1646 napi_disable(&lp->napi_rx); 1647 } 1648 1649 cancel_delayed_work_sync(&lp->stats_work); 1650 1651 phylink_stop(lp->phylink); 1652 phylink_disconnect_phy(lp->phylink); 1653 1654 axienet_setoptions(ndev, lp->options & 1655 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 1656 1657 if (!lp->use_dmaengine) { 1658 axienet_dma_stop(lp); 1659 cancel_work_sync(&lp->dma_err_task); 1660 free_irq(lp->tx_irq, ndev); 1661 free_irq(lp->rx_irq, ndev); 1662 axienet_dma_bd_release(ndev); 1663 } else { 1664 dmaengine_terminate_sync(lp->tx_chan); 1665 dmaengine_synchronize(lp->tx_chan); 1666 dmaengine_terminate_sync(lp->rx_chan); 1667 dmaengine_synchronize(lp->rx_chan); 1668 1669 for (i = 0; i < TX_BD_NUM_MAX; i++) 1670 kfree(lp->tx_skb_ring[i]); 1671 kfree(lp->tx_skb_ring); 1672 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) 1673 kfree(lp->rx_skb_ring[i]); 1674 kfree(lp->rx_skb_ring); 1675 1676 dma_release_channel(lp->rx_chan); 1677 dma_release_channel(lp->tx_chan); 1678 } 1679 1680 axienet_iow(lp, XAE_IE_OFFSET, 0); 1681 1682 if (lp->eth_irq > 0) 1683 free_irq(lp->eth_irq, ndev); 1684 return 0; 1685 } 1686 1687 /** 1688 * axienet_change_mtu - Driver change mtu routine. 1689 * @ndev: Pointer to net_device structure 1690 * @new_mtu: New mtu value to be applied 1691 * 1692 * Return: Always returns 0 (success). 1693 * 1694 * This is the change mtu driver routine. It checks if the Axi Ethernet 1695 * hardware supports jumbo frames before changing the mtu. This can be 1696 * called only when the device is not up. 1697 */ 1698 static int axienet_change_mtu(struct net_device *ndev, int new_mtu) 1699 { 1700 struct axienet_local *lp = netdev_priv(ndev); 1701 1702 if (netif_running(ndev)) 1703 return -EBUSY; 1704 1705 if ((new_mtu + VLAN_ETH_HLEN + 1706 XAE_TRL_SIZE) > lp->rxmem) 1707 return -EINVAL; 1708 1709 WRITE_ONCE(ndev->mtu, new_mtu); 1710 1711 return 0; 1712 } 1713 1714 #ifdef CONFIG_NET_POLL_CONTROLLER 1715 /** 1716 * axienet_poll_controller - Axi Ethernet poll mechanism. 1717 * @ndev: Pointer to net_device structure 1718 * 1719 * This implements Rx/Tx ISR poll mechanisms. The interrupts are disabled prior 1720 * to polling the ISRs and are enabled back after the polling is done. 1721 */ 1722 static void axienet_poll_controller(struct net_device *ndev) 1723 { 1724 struct axienet_local *lp = netdev_priv(ndev); 1725 1726 disable_irq(lp->tx_irq); 1727 disable_irq(lp->rx_irq); 1728 axienet_rx_irq(lp->tx_irq, ndev); 1729 axienet_tx_irq(lp->rx_irq, ndev); 1730 enable_irq(lp->tx_irq); 1731 enable_irq(lp->rx_irq); 1732 } 1733 #endif 1734 1735 static int axienet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 1736 { 1737 struct axienet_local *lp = netdev_priv(dev); 1738 1739 if (!netif_running(dev)) 1740 return -EINVAL; 1741 1742 return phylink_mii_ioctl(lp->phylink, rq, cmd); 1743 } 1744 1745 static void 1746 axienet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1747 { 1748 struct axienet_local *lp = netdev_priv(dev); 1749 unsigned int start; 1750 1751 netdev_stats_to_stats64(stats, &dev->stats); 1752 1753 do { 1754 start = u64_stats_fetch_begin(&lp->rx_stat_sync); 1755 stats->rx_packets = u64_stats_read(&lp->rx_packets); 1756 stats->rx_bytes = u64_stats_read(&lp->rx_bytes); 1757 } while (u64_stats_fetch_retry(&lp->rx_stat_sync, start)); 1758 1759 do { 1760 start = u64_stats_fetch_begin(&lp->tx_stat_sync); 1761 stats->tx_packets = u64_stats_read(&lp->tx_packets); 1762 stats->tx_bytes = u64_stats_read(&lp->tx_bytes); 1763 } while (u64_stats_fetch_retry(&lp->tx_stat_sync, start)); 1764 1765 if (!(lp->features & XAE_FEATURE_STATS)) 1766 return; 1767 1768 do { 1769 start = read_seqcount_begin(&lp->hw_stats_seqcount); 1770 stats->rx_length_errors = 1771 axienet_stat(lp, STAT_RX_LENGTH_ERRORS); 1772 stats->rx_crc_errors = axienet_stat(lp, STAT_RX_FCS_ERRORS); 1773 stats->rx_frame_errors = 1774 axienet_stat(lp, STAT_RX_ALIGNMENT_ERRORS); 1775 stats->rx_errors = axienet_stat(lp, STAT_UNDERSIZE_FRAMES) + 1776 axienet_stat(lp, STAT_FRAGMENT_FRAMES) + 1777 stats->rx_length_errors + 1778 stats->rx_crc_errors + 1779 stats->rx_frame_errors; 1780 stats->multicast = axienet_stat(lp, STAT_RX_MULTICAST_FRAMES); 1781 1782 stats->tx_aborted_errors = 1783 axienet_stat(lp, STAT_TX_EXCESS_COLLISIONS); 1784 stats->tx_fifo_errors = 1785 axienet_stat(lp, STAT_TX_UNDERRUN_ERRORS); 1786 stats->tx_window_errors = 1787 axienet_stat(lp, STAT_TX_LATE_COLLISIONS); 1788 stats->tx_errors = axienet_stat(lp, STAT_TX_EXCESS_DEFERRAL) + 1789 stats->tx_aborted_errors + 1790 stats->tx_fifo_errors + 1791 stats->tx_window_errors; 1792 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 1793 } 1794 1795 static const struct net_device_ops axienet_netdev_ops = { 1796 .ndo_open = axienet_open, 1797 .ndo_stop = axienet_stop, 1798 .ndo_start_xmit = axienet_start_xmit, 1799 .ndo_get_stats64 = axienet_get_stats64, 1800 .ndo_change_mtu = axienet_change_mtu, 1801 .ndo_set_mac_address = netdev_set_mac_address, 1802 .ndo_validate_addr = eth_validate_addr, 1803 .ndo_eth_ioctl = axienet_ioctl, 1804 .ndo_set_rx_mode = axienet_set_multicast_list, 1805 #ifdef CONFIG_NET_POLL_CONTROLLER 1806 .ndo_poll_controller = axienet_poll_controller, 1807 #endif 1808 }; 1809 1810 static const struct net_device_ops axienet_netdev_dmaengine_ops = { 1811 .ndo_open = axienet_open, 1812 .ndo_stop = axienet_stop, 1813 .ndo_start_xmit = axienet_start_xmit_dmaengine, 1814 .ndo_get_stats64 = axienet_get_stats64, 1815 .ndo_change_mtu = axienet_change_mtu, 1816 .ndo_set_mac_address = netdev_set_mac_address, 1817 .ndo_validate_addr = eth_validate_addr, 1818 .ndo_eth_ioctl = axienet_ioctl, 1819 .ndo_set_rx_mode = axienet_set_multicast_list, 1820 }; 1821 1822 /** 1823 * axienet_ethtools_get_drvinfo - Get various Axi Ethernet driver information. 1824 * @ndev: Pointer to net_device structure 1825 * @ed: Pointer to ethtool_drvinfo structure 1826 * 1827 * This implements ethtool command for getting the driver information. 1828 * Issue "ethtool -i ethX" under linux prompt to execute this function. 1829 */ 1830 static void axienet_ethtools_get_drvinfo(struct net_device *ndev, 1831 struct ethtool_drvinfo *ed) 1832 { 1833 strscpy(ed->driver, DRIVER_NAME, sizeof(ed->driver)); 1834 strscpy(ed->version, DRIVER_VERSION, sizeof(ed->version)); 1835 } 1836 1837 /** 1838 * axienet_ethtools_get_regs_len - Get the total regs length present in the 1839 * AxiEthernet core. 1840 * @ndev: Pointer to net_device structure 1841 * 1842 * This implements ethtool command for getting the total register length 1843 * information. 1844 * 1845 * Return: the total regs length 1846 */ 1847 static int axienet_ethtools_get_regs_len(struct net_device *ndev) 1848 { 1849 return sizeof(u32) * AXIENET_REGS_N; 1850 } 1851 1852 /** 1853 * axienet_ethtools_get_regs - Dump the contents of all registers present 1854 * in AxiEthernet core. 1855 * @ndev: Pointer to net_device structure 1856 * @regs: Pointer to ethtool_regs structure 1857 * @ret: Void pointer used to return the contents of the registers. 1858 * 1859 * This implements ethtool command for getting the Axi Ethernet register dump. 1860 * Issue "ethtool -d ethX" to execute this function. 1861 */ 1862 static void axienet_ethtools_get_regs(struct net_device *ndev, 1863 struct ethtool_regs *regs, void *ret) 1864 { 1865 u32 *data = (u32 *)ret; 1866 size_t len = sizeof(u32) * AXIENET_REGS_N; 1867 struct axienet_local *lp = netdev_priv(ndev); 1868 1869 regs->version = 0; 1870 regs->len = len; 1871 1872 memset(data, 0, len); 1873 data[0] = axienet_ior(lp, XAE_RAF_OFFSET); 1874 data[1] = axienet_ior(lp, XAE_TPF_OFFSET); 1875 data[2] = axienet_ior(lp, XAE_IFGP_OFFSET); 1876 data[3] = axienet_ior(lp, XAE_IS_OFFSET); 1877 data[4] = axienet_ior(lp, XAE_IP_OFFSET); 1878 data[5] = axienet_ior(lp, XAE_IE_OFFSET); 1879 data[6] = axienet_ior(lp, XAE_TTAG_OFFSET); 1880 data[7] = axienet_ior(lp, XAE_RTAG_OFFSET); 1881 data[8] = axienet_ior(lp, XAE_UAWL_OFFSET); 1882 data[9] = axienet_ior(lp, XAE_UAWU_OFFSET); 1883 data[10] = axienet_ior(lp, XAE_TPID0_OFFSET); 1884 data[11] = axienet_ior(lp, XAE_TPID1_OFFSET); 1885 data[12] = axienet_ior(lp, XAE_PPST_OFFSET); 1886 data[13] = axienet_ior(lp, XAE_RCW0_OFFSET); 1887 data[14] = axienet_ior(lp, XAE_RCW1_OFFSET); 1888 data[15] = axienet_ior(lp, XAE_TC_OFFSET); 1889 data[16] = axienet_ior(lp, XAE_FCC_OFFSET); 1890 data[17] = axienet_ior(lp, XAE_EMMC_OFFSET); 1891 data[18] = axienet_ior(lp, XAE_PHYC_OFFSET); 1892 data[19] = axienet_ior(lp, XAE_MDIO_MC_OFFSET); 1893 data[20] = axienet_ior(lp, XAE_MDIO_MCR_OFFSET); 1894 data[21] = axienet_ior(lp, XAE_MDIO_MWD_OFFSET); 1895 data[22] = axienet_ior(lp, XAE_MDIO_MRD_OFFSET); 1896 data[27] = axienet_ior(lp, XAE_UAW0_OFFSET); 1897 data[28] = axienet_ior(lp, XAE_UAW1_OFFSET); 1898 data[29] = axienet_ior(lp, XAE_FMI_OFFSET); 1899 data[30] = axienet_ior(lp, XAE_AF0_OFFSET); 1900 data[31] = axienet_ior(lp, XAE_AF1_OFFSET); 1901 if (!lp->use_dmaengine) { 1902 data[32] = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET); 1903 data[33] = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 1904 data[34] = axienet_dma_in32(lp, XAXIDMA_TX_CDESC_OFFSET); 1905 data[35] = axienet_dma_in32(lp, XAXIDMA_TX_TDESC_OFFSET); 1906 data[36] = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET); 1907 data[37] = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 1908 data[38] = axienet_dma_in32(lp, XAXIDMA_RX_CDESC_OFFSET); 1909 data[39] = axienet_dma_in32(lp, XAXIDMA_RX_TDESC_OFFSET); 1910 } 1911 } 1912 1913 static void 1914 axienet_ethtools_get_ringparam(struct net_device *ndev, 1915 struct ethtool_ringparam *ering, 1916 struct kernel_ethtool_ringparam *kernel_ering, 1917 struct netlink_ext_ack *extack) 1918 { 1919 struct axienet_local *lp = netdev_priv(ndev); 1920 1921 ering->rx_max_pending = RX_BD_NUM_MAX; 1922 ering->rx_mini_max_pending = 0; 1923 ering->rx_jumbo_max_pending = 0; 1924 ering->tx_max_pending = TX_BD_NUM_MAX; 1925 ering->rx_pending = lp->rx_bd_num; 1926 ering->rx_mini_pending = 0; 1927 ering->rx_jumbo_pending = 0; 1928 ering->tx_pending = lp->tx_bd_num; 1929 } 1930 1931 static int 1932 axienet_ethtools_set_ringparam(struct net_device *ndev, 1933 struct ethtool_ringparam *ering, 1934 struct kernel_ethtool_ringparam *kernel_ering, 1935 struct netlink_ext_ack *extack) 1936 { 1937 struct axienet_local *lp = netdev_priv(ndev); 1938 1939 if (ering->rx_pending > RX_BD_NUM_MAX || 1940 ering->rx_mini_pending || 1941 ering->rx_jumbo_pending || 1942 ering->tx_pending < TX_BD_NUM_MIN || 1943 ering->tx_pending > TX_BD_NUM_MAX) 1944 return -EINVAL; 1945 1946 if (netif_running(ndev)) 1947 return -EBUSY; 1948 1949 lp->rx_bd_num = ering->rx_pending; 1950 lp->tx_bd_num = ering->tx_pending; 1951 return 0; 1952 } 1953 1954 /** 1955 * axienet_ethtools_get_pauseparam - Get the pause parameter setting for 1956 * Tx and Rx paths. 1957 * @ndev: Pointer to net_device structure 1958 * @epauseparm: Pointer to ethtool_pauseparam structure. 1959 * 1960 * This implements ethtool command for getting axi ethernet pause frame 1961 * setting. Issue "ethtool -a ethX" to execute this function. 1962 */ 1963 static void 1964 axienet_ethtools_get_pauseparam(struct net_device *ndev, 1965 struct ethtool_pauseparam *epauseparm) 1966 { 1967 struct axienet_local *lp = netdev_priv(ndev); 1968 1969 phylink_ethtool_get_pauseparam(lp->phylink, epauseparm); 1970 } 1971 1972 /** 1973 * axienet_ethtools_set_pauseparam - Set device pause parameter(flow control) 1974 * settings. 1975 * @ndev: Pointer to net_device structure 1976 * @epauseparm:Pointer to ethtool_pauseparam structure 1977 * 1978 * This implements ethtool command for enabling flow control on Rx and Tx 1979 * paths. Issue "ethtool -A ethX tx on|off" under linux prompt to execute this 1980 * function. 1981 * 1982 * Return: 0 on success, -EFAULT if device is running 1983 */ 1984 static int 1985 axienet_ethtools_set_pauseparam(struct net_device *ndev, 1986 struct ethtool_pauseparam *epauseparm) 1987 { 1988 struct axienet_local *lp = netdev_priv(ndev); 1989 1990 return phylink_ethtool_set_pauseparam(lp->phylink, epauseparm); 1991 } 1992 1993 /** 1994 * axienet_ethtools_get_coalesce - Get DMA interrupt coalescing count. 1995 * @ndev: Pointer to net_device structure 1996 * @ecoalesce: Pointer to ethtool_coalesce structure 1997 * @kernel_coal: ethtool CQE mode setting structure 1998 * @extack: extack for reporting error messages 1999 * 2000 * This implements ethtool command for getting the DMA interrupt coalescing 2001 * count on Tx and Rx paths. Issue "ethtool -c ethX" under linux prompt to 2002 * execute this function. 2003 * 2004 * Return: 0 always 2005 */ 2006 static int 2007 axienet_ethtools_get_coalesce(struct net_device *ndev, 2008 struct ethtool_coalesce *ecoalesce, 2009 struct kernel_ethtool_coalesce *kernel_coal, 2010 struct netlink_ext_ack *extack) 2011 { 2012 struct axienet_local *lp = netdev_priv(ndev); 2013 2014 ecoalesce->rx_max_coalesced_frames = lp->coalesce_count_rx; 2015 ecoalesce->rx_coalesce_usecs = lp->coalesce_usec_rx; 2016 ecoalesce->tx_max_coalesced_frames = lp->coalesce_count_tx; 2017 ecoalesce->tx_coalesce_usecs = lp->coalesce_usec_tx; 2018 return 0; 2019 } 2020 2021 /** 2022 * axienet_ethtools_set_coalesce - Set DMA interrupt coalescing count. 2023 * @ndev: Pointer to net_device structure 2024 * @ecoalesce: Pointer to ethtool_coalesce structure 2025 * @kernel_coal: ethtool CQE mode setting structure 2026 * @extack: extack for reporting error messages 2027 * 2028 * This implements ethtool command for setting the DMA interrupt coalescing 2029 * count on Tx and Rx paths. Issue "ethtool -C ethX rx-frames 5" under linux 2030 * prompt to execute this function. 2031 * 2032 * Return: 0, on success, Non-zero error value on failure. 2033 */ 2034 static int 2035 axienet_ethtools_set_coalesce(struct net_device *ndev, 2036 struct ethtool_coalesce *ecoalesce, 2037 struct kernel_ethtool_coalesce *kernel_coal, 2038 struct netlink_ext_ack *extack) 2039 { 2040 struct axienet_local *lp = netdev_priv(ndev); 2041 2042 if (netif_running(ndev)) { 2043 NL_SET_ERR_MSG(extack, 2044 "Please stop netif before applying configuration"); 2045 return -EBUSY; 2046 } 2047 2048 if (ecoalesce->rx_max_coalesced_frames) 2049 lp->coalesce_count_rx = ecoalesce->rx_max_coalesced_frames; 2050 if (ecoalesce->rx_coalesce_usecs) 2051 lp->coalesce_usec_rx = ecoalesce->rx_coalesce_usecs; 2052 if (ecoalesce->tx_max_coalesced_frames) 2053 lp->coalesce_count_tx = ecoalesce->tx_max_coalesced_frames; 2054 if (ecoalesce->tx_coalesce_usecs) 2055 lp->coalesce_usec_tx = ecoalesce->tx_coalesce_usecs; 2056 2057 return 0; 2058 } 2059 2060 static int 2061 axienet_ethtools_get_link_ksettings(struct net_device *ndev, 2062 struct ethtool_link_ksettings *cmd) 2063 { 2064 struct axienet_local *lp = netdev_priv(ndev); 2065 2066 return phylink_ethtool_ksettings_get(lp->phylink, cmd); 2067 } 2068 2069 static int 2070 axienet_ethtools_set_link_ksettings(struct net_device *ndev, 2071 const struct ethtool_link_ksettings *cmd) 2072 { 2073 struct axienet_local *lp = netdev_priv(ndev); 2074 2075 return phylink_ethtool_ksettings_set(lp->phylink, cmd); 2076 } 2077 2078 static int axienet_ethtools_nway_reset(struct net_device *dev) 2079 { 2080 struct axienet_local *lp = netdev_priv(dev); 2081 2082 return phylink_ethtool_nway_reset(lp->phylink); 2083 } 2084 2085 static void axienet_ethtools_get_ethtool_stats(struct net_device *dev, 2086 struct ethtool_stats *stats, 2087 u64 *data) 2088 { 2089 struct axienet_local *lp = netdev_priv(dev); 2090 unsigned int start; 2091 2092 do { 2093 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2094 data[0] = axienet_stat(lp, STAT_RX_BYTES); 2095 data[1] = axienet_stat(lp, STAT_TX_BYTES); 2096 data[2] = axienet_stat(lp, STAT_RX_VLAN_FRAMES); 2097 data[3] = axienet_stat(lp, STAT_TX_VLAN_FRAMES); 2098 data[6] = axienet_stat(lp, STAT_TX_PFC_FRAMES); 2099 data[7] = axienet_stat(lp, STAT_RX_PFC_FRAMES); 2100 data[8] = axienet_stat(lp, STAT_USER_DEFINED0); 2101 data[9] = axienet_stat(lp, STAT_USER_DEFINED1); 2102 data[10] = axienet_stat(lp, STAT_USER_DEFINED2); 2103 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2104 } 2105 2106 static const char axienet_ethtool_stats_strings[][ETH_GSTRING_LEN] = { 2107 "Received bytes", 2108 "Transmitted bytes", 2109 "RX Good VLAN Tagged Frames", 2110 "TX Good VLAN Tagged Frames", 2111 "TX Good PFC Frames", 2112 "RX Good PFC Frames", 2113 "User Defined Counter 0", 2114 "User Defined Counter 1", 2115 "User Defined Counter 2", 2116 }; 2117 2118 static void axienet_ethtools_get_strings(struct net_device *dev, u32 stringset, u8 *data) 2119 { 2120 switch (stringset) { 2121 case ETH_SS_STATS: 2122 memcpy(data, axienet_ethtool_stats_strings, 2123 sizeof(axienet_ethtool_stats_strings)); 2124 break; 2125 } 2126 } 2127 2128 static int axienet_ethtools_get_sset_count(struct net_device *dev, int sset) 2129 { 2130 struct axienet_local *lp = netdev_priv(dev); 2131 2132 switch (sset) { 2133 case ETH_SS_STATS: 2134 if (lp->features & XAE_FEATURE_STATS) 2135 return ARRAY_SIZE(axienet_ethtool_stats_strings); 2136 fallthrough; 2137 default: 2138 return -EOPNOTSUPP; 2139 } 2140 } 2141 2142 static void 2143 axienet_ethtools_get_pause_stats(struct net_device *dev, 2144 struct ethtool_pause_stats *pause_stats) 2145 { 2146 struct axienet_local *lp = netdev_priv(dev); 2147 unsigned int start; 2148 2149 if (!(lp->features & XAE_FEATURE_STATS)) 2150 return; 2151 2152 do { 2153 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2154 pause_stats->tx_pause_frames = 2155 axienet_stat(lp, STAT_TX_PAUSE_FRAMES); 2156 pause_stats->rx_pause_frames = 2157 axienet_stat(lp, STAT_RX_PAUSE_FRAMES); 2158 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2159 } 2160 2161 static void 2162 axienet_ethtool_get_eth_mac_stats(struct net_device *dev, 2163 struct ethtool_eth_mac_stats *mac_stats) 2164 { 2165 struct axienet_local *lp = netdev_priv(dev); 2166 unsigned int start; 2167 2168 if (!(lp->features & XAE_FEATURE_STATS)) 2169 return; 2170 2171 do { 2172 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2173 mac_stats->FramesTransmittedOK = 2174 axienet_stat(lp, STAT_TX_GOOD_FRAMES); 2175 mac_stats->SingleCollisionFrames = 2176 axienet_stat(lp, STAT_TX_SINGLE_COLLISION_FRAMES); 2177 mac_stats->MultipleCollisionFrames = 2178 axienet_stat(lp, STAT_TX_MULTIPLE_COLLISION_FRAMES); 2179 mac_stats->FramesReceivedOK = 2180 axienet_stat(lp, STAT_RX_GOOD_FRAMES); 2181 mac_stats->FrameCheckSequenceErrors = 2182 axienet_stat(lp, STAT_RX_FCS_ERRORS); 2183 mac_stats->AlignmentErrors = 2184 axienet_stat(lp, STAT_RX_ALIGNMENT_ERRORS); 2185 mac_stats->FramesWithDeferredXmissions = 2186 axienet_stat(lp, STAT_TX_DEFERRED_FRAMES); 2187 mac_stats->LateCollisions = 2188 axienet_stat(lp, STAT_TX_LATE_COLLISIONS); 2189 mac_stats->FramesAbortedDueToXSColls = 2190 axienet_stat(lp, STAT_TX_EXCESS_COLLISIONS); 2191 mac_stats->MulticastFramesXmittedOK = 2192 axienet_stat(lp, STAT_TX_MULTICAST_FRAMES); 2193 mac_stats->BroadcastFramesXmittedOK = 2194 axienet_stat(lp, STAT_TX_BROADCAST_FRAMES); 2195 mac_stats->FramesWithExcessiveDeferral = 2196 axienet_stat(lp, STAT_TX_EXCESS_DEFERRAL); 2197 mac_stats->MulticastFramesReceivedOK = 2198 axienet_stat(lp, STAT_RX_MULTICAST_FRAMES); 2199 mac_stats->BroadcastFramesReceivedOK = 2200 axienet_stat(lp, STAT_RX_BROADCAST_FRAMES); 2201 mac_stats->InRangeLengthErrors = 2202 axienet_stat(lp, STAT_RX_LENGTH_ERRORS); 2203 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2204 } 2205 2206 static void 2207 axienet_ethtool_get_eth_ctrl_stats(struct net_device *dev, 2208 struct ethtool_eth_ctrl_stats *ctrl_stats) 2209 { 2210 struct axienet_local *lp = netdev_priv(dev); 2211 unsigned int start; 2212 2213 if (!(lp->features & XAE_FEATURE_STATS)) 2214 return; 2215 2216 do { 2217 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2218 ctrl_stats->MACControlFramesTransmitted = 2219 axienet_stat(lp, STAT_TX_CONTROL_FRAMES); 2220 ctrl_stats->MACControlFramesReceived = 2221 axienet_stat(lp, STAT_RX_CONTROL_FRAMES); 2222 ctrl_stats->UnsupportedOpcodesReceived = 2223 axienet_stat(lp, STAT_RX_CONTROL_OPCODE_ERRORS); 2224 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2225 } 2226 2227 static const struct ethtool_rmon_hist_range axienet_rmon_ranges[] = { 2228 { 64, 64 }, 2229 { 65, 127 }, 2230 { 128, 255 }, 2231 { 256, 511 }, 2232 { 512, 1023 }, 2233 { 1024, 1518 }, 2234 { 1519, 16384 }, 2235 { }, 2236 }; 2237 2238 static void 2239 axienet_ethtool_get_rmon_stats(struct net_device *dev, 2240 struct ethtool_rmon_stats *rmon_stats, 2241 const struct ethtool_rmon_hist_range **ranges) 2242 { 2243 struct axienet_local *lp = netdev_priv(dev); 2244 unsigned int start; 2245 2246 if (!(lp->features & XAE_FEATURE_STATS)) 2247 return; 2248 2249 do { 2250 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2251 rmon_stats->undersize_pkts = 2252 axienet_stat(lp, STAT_UNDERSIZE_FRAMES); 2253 rmon_stats->oversize_pkts = 2254 axienet_stat(lp, STAT_RX_OVERSIZE_FRAMES); 2255 rmon_stats->fragments = 2256 axienet_stat(lp, STAT_FRAGMENT_FRAMES); 2257 2258 rmon_stats->hist[0] = 2259 axienet_stat(lp, STAT_RX_64_BYTE_FRAMES); 2260 rmon_stats->hist[1] = 2261 axienet_stat(lp, STAT_RX_65_127_BYTE_FRAMES); 2262 rmon_stats->hist[2] = 2263 axienet_stat(lp, STAT_RX_128_255_BYTE_FRAMES); 2264 rmon_stats->hist[3] = 2265 axienet_stat(lp, STAT_RX_256_511_BYTE_FRAMES); 2266 rmon_stats->hist[4] = 2267 axienet_stat(lp, STAT_RX_512_1023_BYTE_FRAMES); 2268 rmon_stats->hist[5] = 2269 axienet_stat(lp, STAT_RX_1024_MAX_BYTE_FRAMES); 2270 rmon_stats->hist[6] = 2271 rmon_stats->oversize_pkts; 2272 2273 rmon_stats->hist_tx[0] = 2274 axienet_stat(lp, STAT_TX_64_BYTE_FRAMES); 2275 rmon_stats->hist_tx[1] = 2276 axienet_stat(lp, STAT_TX_65_127_BYTE_FRAMES); 2277 rmon_stats->hist_tx[2] = 2278 axienet_stat(lp, STAT_TX_128_255_BYTE_FRAMES); 2279 rmon_stats->hist_tx[3] = 2280 axienet_stat(lp, STAT_TX_256_511_BYTE_FRAMES); 2281 rmon_stats->hist_tx[4] = 2282 axienet_stat(lp, STAT_TX_512_1023_BYTE_FRAMES); 2283 rmon_stats->hist_tx[5] = 2284 axienet_stat(lp, STAT_TX_1024_MAX_BYTE_FRAMES); 2285 rmon_stats->hist_tx[6] = 2286 axienet_stat(lp, STAT_TX_OVERSIZE_FRAMES); 2287 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2288 2289 *ranges = axienet_rmon_ranges; 2290 } 2291 2292 static const struct ethtool_ops axienet_ethtool_ops = { 2293 .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | 2294 ETHTOOL_COALESCE_USECS, 2295 .get_drvinfo = axienet_ethtools_get_drvinfo, 2296 .get_regs_len = axienet_ethtools_get_regs_len, 2297 .get_regs = axienet_ethtools_get_regs, 2298 .get_link = ethtool_op_get_link, 2299 .get_ringparam = axienet_ethtools_get_ringparam, 2300 .set_ringparam = axienet_ethtools_set_ringparam, 2301 .get_pauseparam = axienet_ethtools_get_pauseparam, 2302 .set_pauseparam = axienet_ethtools_set_pauseparam, 2303 .get_coalesce = axienet_ethtools_get_coalesce, 2304 .set_coalesce = axienet_ethtools_set_coalesce, 2305 .get_link_ksettings = axienet_ethtools_get_link_ksettings, 2306 .set_link_ksettings = axienet_ethtools_set_link_ksettings, 2307 .nway_reset = axienet_ethtools_nway_reset, 2308 .get_ethtool_stats = axienet_ethtools_get_ethtool_stats, 2309 .get_strings = axienet_ethtools_get_strings, 2310 .get_sset_count = axienet_ethtools_get_sset_count, 2311 .get_pause_stats = axienet_ethtools_get_pause_stats, 2312 .get_eth_mac_stats = axienet_ethtool_get_eth_mac_stats, 2313 .get_eth_ctrl_stats = axienet_ethtool_get_eth_ctrl_stats, 2314 .get_rmon_stats = axienet_ethtool_get_rmon_stats, 2315 }; 2316 2317 static struct axienet_local *pcs_to_axienet_local(struct phylink_pcs *pcs) 2318 { 2319 return container_of(pcs, struct axienet_local, pcs); 2320 } 2321 2322 static void axienet_pcs_get_state(struct phylink_pcs *pcs, 2323 struct phylink_link_state *state) 2324 { 2325 struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy; 2326 2327 phylink_mii_c22_pcs_get_state(pcs_phy, state); 2328 } 2329 2330 static void axienet_pcs_an_restart(struct phylink_pcs *pcs) 2331 { 2332 struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy; 2333 2334 phylink_mii_c22_pcs_an_restart(pcs_phy); 2335 } 2336 2337 static int axienet_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode, 2338 phy_interface_t interface, 2339 const unsigned long *advertising, 2340 bool permit_pause_to_mac) 2341 { 2342 struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy; 2343 struct net_device *ndev = pcs_to_axienet_local(pcs)->ndev; 2344 struct axienet_local *lp = netdev_priv(ndev); 2345 int ret; 2346 2347 if (lp->switch_x_sgmii) { 2348 ret = mdiodev_write(pcs_phy, XLNX_MII_STD_SELECT_REG, 2349 interface == PHY_INTERFACE_MODE_SGMII ? 2350 XLNX_MII_STD_SELECT_SGMII : 0); 2351 if (ret < 0) { 2352 netdev_warn(ndev, 2353 "Failed to switch PHY interface: %d\n", 2354 ret); 2355 return ret; 2356 } 2357 } 2358 2359 ret = phylink_mii_c22_pcs_config(pcs_phy, interface, advertising, 2360 neg_mode); 2361 if (ret < 0) 2362 netdev_warn(ndev, "Failed to configure PCS: %d\n", ret); 2363 2364 return ret; 2365 } 2366 2367 static const struct phylink_pcs_ops axienet_pcs_ops = { 2368 .pcs_get_state = axienet_pcs_get_state, 2369 .pcs_config = axienet_pcs_config, 2370 .pcs_an_restart = axienet_pcs_an_restart, 2371 }; 2372 2373 static struct phylink_pcs *axienet_mac_select_pcs(struct phylink_config *config, 2374 phy_interface_t interface) 2375 { 2376 struct net_device *ndev = to_net_dev(config->dev); 2377 struct axienet_local *lp = netdev_priv(ndev); 2378 2379 if (interface == PHY_INTERFACE_MODE_1000BASEX || 2380 interface == PHY_INTERFACE_MODE_SGMII) 2381 return &lp->pcs; 2382 2383 return NULL; 2384 } 2385 2386 static void axienet_mac_config(struct phylink_config *config, unsigned int mode, 2387 const struct phylink_link_state *state) 2388 { 2389 /* nothing meaningful to do */ 2390 } 2391 2392 static void axienet_mac_link_down(struct phylink_config *config, 2393 unsigned int mode, 2394 phy_interface_t interface) 2395 { 2396 /* nothing meaningful to do */ 2397 } 2398 2399 static void axienet_mac_link_up(struct phylink_config *config, 2400 struct phy_device *phy, 2401 unsigned int mode, phy_interface_t interface, 2402 int speed, int duplex, 2403 bool tx_pause, bool rx_pause) 2404 { 2405 struct net_device *ndev = to_net_dev(config->dev); 2406 struct axienet_local *lp = netdev_priv(ndev); 2407 u32 emmc_reg, fcc_reg; 2408 2409 emmc_reg = axienet_ior(lp, XAE_EMMC_OFFSET); 2410 emmc_reg &= ~XAE_EMMC_LINKSPEED_MASK; 2411 2412 switch (speed) { 2413 case SPEED_1000: 2414 emmc_reg |= XAE_EMMC_LINKSPD_1000; 2415 break; 2416 case SPEED_100: 2417 emmc_reg |= XAE_EMMC_LINKSPD_100; 2418 break; 2419 case SPEED_10: 2420 emmc_reg |= XAE_EMMC_LINKSPD_10; 2421 break; 2422 default: 2423 dev_err(&ndev->dev, 2424 "Speed other than 10, 100 or 1Gbps is not supported\n"); 2425 break; 2426 } 2427 2428 axienet_iow(lp, XAE_EMMC_OFFSET, emmc_reg); 2429 2430 fcc_reg = axienet_ior(lp, XAE_FCC_OFFSET); 2431 if (tx_pause) 2432 fcc_reg |= XAE_FCC_FCTX_MASK; 2433 else 2434 fcc_reg &= ~XAE_FCC_FCTX_MASK; 2435 if (rx_pause) 2436 fcc_reg |= XAE_FCC_FCRX_MASK; 2437 else 2438 fcc_reg &= ~XAE_FCC_FCRX_MASK; 2439 axienet_iow(lp, XAE_FCC_OFFSET, fcc_reg); 2440 } 2441 2442 static const struct phylink_mac_ops axienet_phylink_ops = { 2443 .mac_select_pcs = axienet_mac_select_pcs, 2444 .mac_config = axienet_mac_config, 2445 .mac_link_down = axienet_mac_link_down, 2446 .mac_link_up = axienet_mac_link_up, 2447 }; 2448 2449 /** 2450 * axienet_dma_err_handler - Work queue task for Axi DMA Error 2451 * @work: pointer to work_struct 2452 * 2453 * Resets the Axi DMA and Axi Ethernet devices, and reconfigures the 2454 * Tx/Rx BDs. 2455 */ 2456 static void axienet_dma_err_handler(struct work_struct *work) 2457 { 2458 u32 i; 2459 u32 axienet_status; 2460 struct axidma_bd *cur_p; 2461 struct axienet_local *lp = container_of(work, struct axienet_local, 2462 dma_err_task); 2463 struct net_device *ndev = lp->ndev; 2464 2465 napi_disable(&lp->napi_tx); 2466 napi_disable(&lp->napi_rx); 2467 2468 axienet_setoptions(ndev, lp->options & 2469 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 2470 2471 axienet_dma_stop(lp); 2472 2473 for (i = 0; i < lp->tx_bd_num; i++) { 2474 cur_p = &lp->tx_bd_v[i]; 2475 if (cur_p->cntrl) { 2476 dma_addr_t addr = desc_get_phys_addr(lp, cur_p); 2477 2478 dma_unmap_single(lp->dev, addr, 2479 (cur_p->cntrl & 2480 XAXIDMA_BD_CTRL_LENGTH_MASK), 2481 DMA_TO_DEVICE); 2482 } 2483 if (cur_p->skb) 2484 dev_kfree_skb_irq(cur_p->skb); 2485 cur_p->phys = 0; 2486 cur_p->phys_msb = 0; 2487 cur_p->cntrl = 0; 2488 cur_p->status = 0; 2489 cur_p->app0 = 0; 2490 cur_p->app1 = 0; 2491 cur_p->app2 = 0; 2492 cur_p->app3 = 0; 2493 cur_p->app4 = 0; 2494 cur_p->skb = NULL; 2495 } 2496 2497 for (i = 0; i < lp->rx_bd_num; i++) { 2498 cur_p = &lp->rx_bd_v[i]; 2499 cur_p->status = 0; 2500 cur_p->app0 = 0; 2501 cur_p->app1 = 0; 2502 cur_p->app2 = 0; 2503 cur_p->app3 = 0; 2504 cur_p->app4 = 0; 2505 } 2506 2507 lp->tx_bd_ci = 0; 2508 lp->tx_bd_tail = 0; 2509 lp->rx_bd_ci = 0; 2510 2511 axienet_dma_start(lp); 2512 2513 axienet_status = axienet_ior(lp, XAE_RCW1_OFFSET); 2514 axienet_status &= ~XAE_RCW1_RX_MASK; 2515 axienet_iow(lp, XAE_RCW1_OFFSET, axienet_status); 2516 2517 axienet_status = axienet_ior(lp, XAE_IP_OFFSET); 2518 if (axienet_status & XAE_INT_RXRJECT_MASK) 2519 axienet_iow(lp, XAE_IS_OFFSET, XAE_INT_RXRJECT_MASK); 2520 axienet_iow(lp, XAE_IE_OFFSET, lp->eth_irq > 0 ? 2521 XAE_INT_RECV_ERROR_MASK : 0); 2522 axienet_iow(lp, XAE_FCC_OFFSET, XAE_FCC_FCRX_MASK); 2523 2524 /* Sync default options with HW but leave receiver and 2525 * transmitter disabled. 2526 */ 2527 axienet_setoptions(ndev, lp->options & 2528 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 2529 axienet_set_mac_address(ndev, NULL); 2530 axienet_set_multicast_list(ndev); 2531 napi_enable(&lp->napi_rx); 2532 napi_enable(&lp->napi_tx); 2533 axienet_setoptions(ndev, lp->options); 2534 } 2535 2536 /** 2537 * axienet_probe - Axi Ethernet probe function. 2538 * @pdev: Pointer to platform device structure. 2539 * 2540 * Return: 0, on success 2541 * Non-zero error value on failure. 2542 * 2543 * This is the probe routine for Axi Ethernet driver. This is called before 2544 * any other driver routines are invoked. It allocates and sets up the Ethernet 2545 * device. Parses through device tree and populates fields of 2546 * axienet_local. It registers the Ethernet device. 2547 */ 2548 static int axienet_probe(struct platform_device *pdev) 2549 { 2550 int ret; 2551 struct device_node *np; 2552 struct axienet_local *lp; 2553 struct net_device *ndev; 2554 struct resource *ethres; 2555 u8 mac_addr[ETH_ALEN]; 2556 int addr_width = 32; 2557 u32 value; 2558 2559 ndev = alloc_etherdev(sizeof(*lp)); 2560 if (!ndev) 2561 return -ENOMEM; 2562 2563 platform_set_drvdata(pdev, ndev); 2564 2565 SET_NETDEV_DEV(ndev, &pdev->dev); 2566 ndev->features = NETIF_F_SG; 2567 ndev->ethtool_ops = &axienet_ethtool_ops; 2568 2569 /* MTU range: 64 - 9000 */ 2570 ndev->min_mtu = 64; 2571 ndev->max_mtu = XAE_JUMBO_MTU; 2572 2573 lp = netdev_priv(ndev); 2574 lp->ndev = ndev; 2575 lp->dev = &pdev->dev; 2576 lp->options = XAE_OPTION_DEFAULTS; 2577 lp->rx_bd_num = RX_BD_NUM_DEFAULT; 2578 lp->tx_bd_num = TX_BD_NUM_DEFAULT; 2579 2580 u64_stats_init(&lp->rx_stat_sync); 2581 u64_stats_init(&lp->tx_stat_sync); 2582 2583 mutex_init(&lp->stats_lock); 2584 seqcount_mutex_init(&lp->hw_stats_seqcount, &lp->stats_lock); 2585 INIT_DEFERRABLE_WORK(&lp->stats_work, axienet_refresh_stats); 2586 2587 lp->axi_clk = devm_clk_get_optional(&pdev->dev, "s_axi_lite_clk"); 2588 if (!lp->axi_clk) { 2589 /* For backward compatibility, if named AXI clock is not present, 2590 * treat the first clock specified as the AXI clock. 2591 */ 2592 lp->axi_clk = devm_clk_get_optional(&pdev->dev, NULL); 2593 } 2594 if (IS_ERR(lp->axi_clk)) { 2595 ret = PTR_ERR(lp->axi_clk); 2596 goto free_netdev; 2597 } 2598 ret = clk_prepare_enable(lp->axi_clk); 2599 if (ret) { 2600 dev_err(&pdev->dev, "Unable to enable AXI clock: %d\n", ret); 2601 goto free_netdev; 2602 } 2603 2604 lp->misc_clks[0].id = "axis_clk"; 2605 lp->misc_clks[1].id = "ref_clk"; 2606 lp->misc_clks[2].id = "mgt_clk"; 2607 2608 ret = devm_clk_bulk_get_optional(&pdev->dev, XAE_NUM_MISC_CLOCKS, lp->misc_clks); 2609 if (ret) 2610 goto cleanup_clk; 2611 2612 ret = clk_bulk_prepare_enable(XAE_NUM_MISC_CLOCKS, lp->misc_clks); 2613 if (ret) 2614 goto cleanup_clk; 2615 2616 /* Map device registers */ 2617 lp->regs = devm_platform_get_and_ioremap_resource(pdev, 0, ðres); 2618 if (IS_ERR(lp->regs)) { 2619 ret = PTR_ERR(lp->regs); 2620 goto cleanup_clk; 2621 } 2622 lp->regs_start = ethres->start; 2623 2624 /* Setup checksum offload, but default to off if not specified */ 2625 lp->features = 0; 2626 2627 if (axienet_ior(lp, XAE_ABILITY_OFFSET) & XAE_ABILITY_STATS) 2628 lp->features |= XAE_FEATURE_STATS; 2629 2630 ret = of_property_read_u32(pdev->dev.of_node, "xlnx,txcsum", &value); 2631 if (!ret) { 2632 switch (value) { 2633 case 1: 2634 lp->csum_offload_on_tx_path = 2635 XAE_FEATURE_PARTIAL_TX_CSUM; 2636 lp->features |= XAE_FEATURE_PARTIAL_TX_CSUM; 2637 /* Can checksum TCP/UDP over IPv4. */ 2638 ndev->features |= NETIF_F_IP_CSUM; 2639 break; 2640 case 2: 2641 lp->csum_offload_on_tx_path = 2642 XAE_FEATURE_FULL_TX_CSUM; 2643 lp->features |= XAE_FEATURE_FULL_TX_CSUM; 2644 /* Can checksum TCP/UDP over IPv4. */ 2645 ndev->features |= NETIF_F_IP_CSUM; 2646 break; 2647 default: 2648 lp->csum_offload_on_tx_path = XAE_NO_CSUM_OFFLOAD; 2649 } 2650 } 2651 ret = of_property_read_u32(pdev->dev.of_node, "xlnx,rxcsum", &value); 2652 if (!ret) { 2653 switch (value) { 2654 case 1: 2655 lp->csum_offload_on_rx_path = 2656 XAE_FEATURE_PARTIAL_RX_CSUM; 2657 lp->features |= XAE_FEATURE_PARTIAL_RX_CSUM; 2658 break; 2659 case 2: 2660 lp->csum_offload_on_rx_path = 2661 XAE_FEATURE_FULL_RX_CSUM; 2662 lp->features |= XAE_FEATURE_FULL_RX_CSUM; 2663 break; 2664 default: 2665 lp->csum_offload_on_rx_path = XAE_NO_CSUM_OFFLOAD; 2666 } 2667 } 2668 /* For supporting jumbo frames, the Axi Ethernet hardware must have 2669 * a larger Rx/Tx Memory. Typically, the size must be large so that 2670 * we can enable jumbo option and start supporting jumbo frames. 2671 * Here we check for memory allocated for Rx/Tx in the hardware from 2672 * the device-tree and accordingly set flags. 2673 */ 2674 of_property_read_u32(pdev->dev.of_node, "xlnx,rxmem", &lp->rxmem); 2675 2676 lp->switch_x_sgmii = of_property_read_bool(pdev->dev.of_node, 2677 "xlnx,switch-x-sgmii"); 2678 2679 /* Start with the proprietary, and broken phy_type */ 2680 ret = of_property_read_u32(pdev->dev.of_node, "xlnx,phy-type", &value); 2681 if (!ret) { 2682 netdev_warn(ndev, "Please upgrade your device tree binary blob to use phy-mode"); 2683 switch (value) { 2684 case XAE_PHY_TYPE_MII: 2685 lp->phy_mode = PHY_INTERFACE_MODE_MII; 2686 break; 2687 case XAE_PHY_TYPE_GMII: 2688 lp->phy_mode = PHY_INTERFACE_MODE_GMII; 2689 break; 2690 case XAE_PHY_TYPE_RGMII_2_0: 2691 lp->phy_mode = PHY_INTERFACE_MODE_RGMII_ID; 2692 break; 2693 case XAE_PHY_TYPE_SGMII: 2694 lp->phy_mode = PHY_INTERFACE_MODE_SGMII; 2695 break; 2696 case XAE_PHY_TYPE_1000BASE_X: 2697 lp->phy_mode = PHY_INTERFACE_MODE_1000BASEX; 2698 break; 2699 default: 2700 ret = -EINVAL; 2701 goto cleanup_clk; 2702 } 2703 } else { 2704 ret = of_get_phy_mode(pdev->dev.of_node, &lp->phy_mode); 2705 if (ret) 2706 goto cleanup_clk; 2707 } 2708 if (lp->switch_x_sgmii && lp->phy_mode != PHY_INTERFACE_MODE_SGMII && 2709 lp->phy_mode != PHY_INTERFACE_MODE_1000BASEX) { 2710 dev_err(&pdev->dev, "xlnx,switch-x-sgmii only supported with SGMII or 1000BaseX\n"); 2711 ret = -EINVAL; 2712 goto cleanup_clk; 2713 } 2714 2715 if (!of_property_present(pdev->dev.of_node, "dmas")) { 2716 /* Find the DMA node, map the DMA registers, and decode the DMA IRQs */ 2717 np = of_parse_phandle(pdev->dev.of_node, "axistream-connected", 0); 2718 2719 if (np) { 2720 struct resource dmares; 2721 2722 ret = of_address_to_resource(np, 0, &dmares); 2723 if (ret) { 2724 dev_err(&pdev->dev, 2725 "unable to get DMA resource\n"); 2726 of_node_put(np); 2727 goto cleanup_clk; 2728 } 2729 lp->dma_regs = devm_ioremap_resource(&pdev->dev, 2730 &dmares); 2731 lp->rx_irq = irq_of_parse_and_map(np, 1); 2732 lp->tx_irq = irq_of_parse_and_map(np, 0); 2733 of_node_put(np); 2734 lp->eth_irq = platform_get_irq_optional(pdev, 0); 2735 } else { 2736 /* Check for these resources directly on the Ethernet node. */ 2737 lp->dma_regs = devm_platform_get_and_ioremap_resource(pdev, 1, NULL); 2738 lp->rx_irq = platform_get_irq(pdev, 1); 2739 lp->tx_irq = platform_get_irq(pdev, 0); 2740 lp->eth_irq = platform_get_irq_optional(pdev, 2); 2741 } 2742 if (IS_ERR(lp->dma_regs)) { 2743 dev_err(&pdev->dev, "could not map DMA regs\n"); 2744 ret = PTR_ERR(lp->dma_regs); 2745 goto cleanup_clk; 2746 } 2747 if (lp->rx_irq <= 0 || lp->tx_irq <= 0) { 2748 dev_err(&pdev->dev, "could not determine irqs\n"); 2749 ret = -ENOMEM; 2750 goto cleanup_clk; 2751 } 2752 2753 /* Reset core now that clocks are enabled, prior to accessing MDIO */ 2754 ret = __axienet_device_reset(lp); 2755 if (ret) 2756 goto cleanup_clk; 2757 2758 /* Autodetect the need for 64-bit DMA pointers. 2759 * When the IP is configured for a bus width bigger than 32 bits, 2760 * writing the MSB registers is mandatory, even if they are all 0. 2761 * We can detect this case by writing all 1's to one such register 2762 * and see if that sticks: when the IP is configured for 32 bits 2763 * only, those registers are RES0. 2764 * Those MSB registers were introduced in IP v7.1, which we check first. 2765 */ 2766 if ((axienet_ior(lp, XAE_ID_OFFSET) >> 24) >= 0x9) { 2767 void __iomem *desc = lp->dma_regs + XAXIDMA_TX_CDESC_OFFSET + 4; 2768 2769 iowrite32(0x0, desc); 2770 if (ioread32(desc) == 0) { /* sanity check */ 2771 iowrite32(0xffffffff, desc); 2772 if (ioread32(desc) > 0) { 2773 lp->features |= XAE_FEATURE_DMA_64BIT; 2774 addr_width = 64; 2775 dev_info(&pdev->dev, 2776 "autodetected 64-bit DMA range\n"); 2777 } 2778 iowrite32(0x0, desc); 2779 } 2780 } 2781 if (!IS_ENABLED(CONFIG_64BIT) && lp->features & XAE_FEATURE_DMA_64BIT) { 2782 dev_err(&pdev->dev, "64-bit addressable DMA is not compatible with 32-bit archecture\n"); 2783 ret = -EINVAL; 2784 goto cleanup_clk; 2785 } 2786 2787 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_width)); 2788 if (ret) { 2789 dev_err(&pdev->dev, "No suitable DMA available\n"); 2790 goto cleanup_clk; 2791 } 2792 netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll); 2793 netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll); 2794 } else { 2795 struct xilinx_vdma_config cfg; 2796 struct dma_chan *tx_chan; 2797 2798 lp->eth_irq = platform_get_irq_optional(pdev, 0); 2799 if (lp->eth_irq < 0 && lp->eth_irq != -ENXIO) { 2800 ret = lp->eth_irq; 2801 goto cleanup_clk; 2802 } 2803 tx_chan = dma_request_chan(lp->dev, "tx_chan0"); 2804 if (IS_ERR(tx_chan)) { 2805 ret = PTR_ERR(tx_chan); 2806 dev_err_probe(lp->dev, ret, "No Ethernet DMA (TX) channel found\n"); 2807 goto cleanup_clk; 2808 } 2809 2810 cfg.reset = 1; 2811 /* As name says VDMA but it has support for DMA channel reset */ 2812 ret = xilinx_vdma_channel_set_config(tx_chan, &cfg); 2813 if (ret < 0) { 2814 dev_err(&pdev->dev, "Reset channel failed\n"); 2815 dma_release_channel(tx_chan); 2816 goto cleanup_clk; 2817 } 2818 2819 dma_release_channel(tx_chan); 2820 lp->use_dmaengine = 1; 2821 } 2822 2823 if (lp->use_dmaengine) 2824 ndev->netdev_ops = &axienet_netdev_dmaengine_ops; 2825 else 2826 ndev->netdev_ops = &axienet_netdev_ops; 2827 /* Check for Ethernet core IRQ (optional) */ 2828 if (lp->eth_irq <= 0) 2829 dev_info(&pdev->dev, "Ethernet core IRQ not defined\n"); 2830 2831 /* Retrieve the MAC address */ 2832 ret = of_get_mac_address(pdev->dev.of_node, mac_addr); 2833 if (!ret) { 2834 axienet_set_mac_address(ndev, mac_addr); 2835 } else { 2836 dev_warn(&pdev->dev, "could not find MAC address property: %d\n", 2837 ret); 2838 axienet_set_mac_address(ndev, NULL); 2839 } 2840 2841 lp->coalesce_count_rx = XAXIDMA_DFT_RX_THRESHOLD; 2842 lp->coalesce_count_tx = XAXIDMA_DFT_TX_THRESHOLD; 2843 lp->coalesce_usec_rx = XAXIDMA_DFT_RX_USEC; 2844 lp->coalesce_usec_tx = XAXIDMA_DFT_TX_USEC; 2845 2846 ret = axienet_mdio_setup(lp); 2847 if (ret) 2848 dev_warn(&pdev->dev, 2849 "error registering MDIO bus: %d\n", ret); 2850 2851 if (lp->phy_mode == PHY_INTERFACE_MODE_SGMII || 2852 lp->phy_mode == PHY_INTERFACE_MODE_1000BASEX) { 2853 np = of_parse_phandle(pdev->dev.of_node, "pcs-handle", 0); 2854 if (!np) { 2855 /* Deprecated: Always use "pcs-handle" for pcs_phy. 2856 * Falling back to "phy-handle" here is only for 2857 * backward compatibility with old device trees. 2858 */ 2859 np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0); 2860 } 2861 if (!np) { 2862 dev_err(&pdev->dev, "pcs-handle (preferred) or phy-handle required for 1000BaseX/SGMII\n"); 2863 ret = -EINVAL; 2864 goto cleanup_mdio; 2865 } 2866 lp->pcs_phy = of_mdio_find_device(np); 2867 if (!lp->pcs_phy) { 2868 ret = -EPROBE_DEFER; 2869 of_node_put(np); 2870 goto cleanup_mdio; 2871 } 2872 of_node_put(np); 2873 lp->pcs.ops = &axienet_pcs_ops; 2874 lp->pcs.neg_mode = true; 2875 lp->pcs.poll = true; 2876 } 2877 2878 lp->phylink_config.dev = &ndev->dev; 2879 lp->phylink_config.type = PHYLINK_NETDEV; 2880 lp->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | 2881 MAC_10FD | MAC_100FD | MAC_1000FD; 2882 2883 __set_bit(lp->phy_mode, lp->phylink_config.supported_interfaces); 2884 if (lp->switch_x_sgmii) { 2885 __set_bit(PHY_INTERFACE_MODE_1000BASEX, 2886 lp->phylink_config.supported_interfaces); 2887 __set_bit(PHY_INTERFACE_MODE_SGMII, 2888 lp->phylink_config.supported_interfaces); 2889 } 2890 2891 lp->phylink = phylink_create(&lp->phylink_config, pdev->dev.fwnode, 2892 lp->phy_mode, 2893 &axienet_phylink_ops); 2894 if (IS_ERR(lp->phylink)) { 2895 ret = PTR_ERR(lp->phylink); 2896 dev_err(&pdev->dev, "phylink_create error (%i)\n", ret); 2897 goto cleanup_mdio; 2898 } 2899 2900 ret = register_netdev(lp->ndev); 2901 if (ret) { 2902 dev_err(lp->dev, "register_netdev() error (%i)\n", ret); 2903 goto cleanup_phylink; 2904 } 2905 2906 return 0; 2907 2908 cleanup_phylink: 2909 phylink_destroy(lp->phylink); 2910 2911 cleanup_mdio: 2912 if (lp->pcs_phy) 2913 put_device(&lp->pcs_phy->dev); 2914 if (lp->mii_bus) 2915 axienet_mdio_teardown(lp); 2916 cleanup_clk: 2917 clk_bulk_disable_unprepare(XAE_NUM_MISC_CLOCKS, lp->misc_clks); 2918 clk_disable_unprepare(lp->axi_clk); 2919 2920 free_netdev: 2921 free_netdev(ndev); 2922 2923 return ret; 2924 } 2925 2926 static void axienet_remove(struct platform_device *pdev) 2927 { 2928 struct net_device *ndev = platform_get_drvdata(pdev); 2929 struct axienet_local *lp = netdev_priv(ndev); 2930 2931 unregister_netdev(ndev); 2932 2933 if (lp->phylink) 2934 phylink_destroy(lp->phylink); 2935 2936 if (lp->pcs_phy) 2937 put_device(&lp->pcs_phy->dev); 2938 2939 axienet_mdio_teardown(lp); 2940 2941 clk_bulk_disable_unprepare(XAE_NUM_MISC_CLOCKS, lp->misc_clks); 2942 clk_disable_unprepare(lp->axi_clk); 2943 2944 free_netdev(ndev); 2945 } 2946 2947 static void axienet_shutdown(struct platform_device *pdev) 2948 { 2949 struct net_device *ndev = platform_get_drvdata(pdev); 2950 2951 rtnl_lock(); 2952 netif_device_detach(ndev); 2953 2954 if (netif_running(ndev)) 2955 dev_close(ndev); 2956 2957 rtnl_unlock(); 2958 } 2959 2960 static int axienet_suspend(struct device *dev) 2961 { 2962 struct net_device *ndev = dev_get_drvdata(dev); 2963 2964 if (!netif_running(ndev)) 2965 return 0; 2966 2967 netif_device_detach(ndev); 2968 2969 rtnl_lock(); 2970 axienet_stop(ndev); 2971 rtnl_unlock(); 2972 2973 return 0; 2974 } 2975 2976 static int axienet_resume(struct device *dev) 2977 { 2978 struct net_device *ndev = dev_get_drvdata(dev); 2979 2980 if (!netif_running(ndev)) 2981 return 0; 2982 2983 rtnl_lock(); 2984 axienet_open(ndev); 2985 rtnl_unlock(); 2986 2987 netif_device_attach(ndev); 2988 2989 return 0; 2990 } 2991 2992 static DEFINE_SIMPLE_DEV_PM_OPS(axienet_pm_ops, 2993 axienet_suspend, axienet_resume); 2994 2995 static struct platform_driver axienet_driver = { 2996 .probe = axienet_probe, 2997 .remove_new = axienet_remove, 2998 .shutdown = axienet_shutdown, 2999 .driver = { 3000 .name = "xilinx_axienet", 3001 .pm = &axienet_pm_ops, 3002 .of_match_table = axienet_of_match, 3003 }, 3004 }; 3005 3006 module_platform_driver(axienet_driver); 3007 3008 MODULE_DESCRIPTION("Xilinx Axi Ethernet driver"); 3009 MODULE_AUTHOR("Xilinx"); 3010 MODULE_LICENSE("GPL"); 3011