1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Xilinx Axi Ethernet device driver 4 * 5 * Copyright (c) 2008 Nissin Systems Co., Ltd., Yoshio Kashiwagi 6 * Copyright (c) 2005-2008 DLA Systems, David H. Lynch Jr. <dhlii@dlasys.net> 7 * Copyright (c) 2008-2009 Secret Lab Technologies Ltd. 8 * Copyright (c) 2010 - 2011 Michal Simek <monstr@monstr.eu> 9 * Copyright (c) 2010 - 2011 PetaLogix 10 * Copyright (c) 2019 - 2022 Calian Advanced Technologies 11 * Copyright (c) 2010 - 2012 Xilinx, Inc. All rights reserved. 12 * 13 * This is a driver for the Xilinx Axi Ethernet which is used in the Virtex6 14 * and Spartan6. 15 * 16 * TODO: 17 * - Add Axi Fifo support. 18 * - Factor out Axi DMA code into separate driver. 19 * - Test and fix basic multicast filtering. 20 * - Add support for extended multicast filtering. 21 * - Test basic VLAN support. 22 * - Add support for extended VLAN support. 23 */ 24 25 #include <linux/clk.h> 26 #include <linux/delay.h> 27 #include <linux/etherdevice.h> 28 #include <linux/module.h> 29 #include <linux/netdevice.h> 30 #include <linux/of.h> 31 #include <linux/of_mdio.h> 32 #include <linux/of_net.h> 33 #include <linux/of_irq.h> 34 #include <linux/of_address.h> 35 #include <linux/platform_device.h> 36 #include <linux/property.h> 37 #include <linux/skbuff.h> 38 #include <linux/math64.h> 39 #include <linux/phy.h> 40 #include <linux/mii.h> 41 #include <linux/ethtool.h> 42 #include <linux/dmaengine.h> 43 #include <linux/dma-mapping.h> 44 #include <linux/dma/xilinx_dma.h> 45 #include <linux/circ_buf.h> 46 #include <net/netdev_queues.h> 47 48 #include "xilinx_axienet.h" 49 50 /* Descriptors defines for Tx and Rx DMA */ 51 #define AXIENET_DRIVER_NAME "xilinx_axienet" 52 #define TX_BD_NUM_DEFAULT 128 53 #define RX_BD_NUM_DEFAULT 1024 54 #define TX_BD_NUM_MIN (MAX_SKB_FRAGS + 1) 55 #define TX_BD_NUM_MAX 4096 56 #define RX_BD_NUM_MAX 4096 57 #define DMA_NUM_APP_WORDS 5 58 #define LEN_APP 4 59 #define RX_BUF_NUM_DEFAULT 128 60 61 /* Must be shorter than length of ethtool_drvinfo.driver field to fit */ 62 #define DRIVER_NAME "xaxienet" 63 #define DRIVER_DESCRIPTION "Xilinx Axi Ethernet driver" 64 #define DRIVER_VERSION "1.00a" 65 66 #define AXIENET_REGS_N 40 67 68 static void axienet_rx_submit_desc(struct net_device *ndev); 69 70 /* Match table for of_platform binding */ 71 static const struct of_device_id axienet_of_match[] = { 72 { .compatible = "xlnx,axi-ethernet-1.00.a", }, 73 { .compatible = "xlnx,axi-ethernet-1.01.a", }, 74 { .compatible = "xlnx,axi-ethernet-2.01.a", }, 75 {}, 76 }; 77 78 MODULE_DEVICE_TABLE(of, axienet_of_match); 79 80 /* Option table for setting up Axi Ethernet hardware options */ 81 static struct axienet_option axienet_options[] = { 82 /* Turn on jumbo packet support for both Rx and Tx */ 83 { 84 .opt = XAE_OPTION_JUMBO, 85 .reg = XAE_TC_OFFSET, 86 .m_or = XAE_TC_JUM_MASK, 87 }, { 88 .opt = XAE_OPTION_JUMBO, 89 .reg = XAE_RCW1_OFFSET, 90 .m_or = XAE_RCW1_JUM_MASK, 91 }, { /* Turn on VLAN packet support for both Rx and Tx */ 92 .opt = XAE_OPTION_VLAN, 93 .reg = XAE_TC_OFFSET, 94 .m_or = XAE_TC_VLAN_MASK, 95 }, { 96 .opt = XAE_OPTION_VLAN, 97 .reg = XAE_RCW1_OFFSET, 98 .m_or = XAE_RCW1_VLAN_MASK, 99 }, { /* Turn on FCS stripping on receive packets */ 100 .opt = XAE_OPTION_FCS_STRIP, 101 .reg = XAE_RCW1_OFFSET, 102 .m_or = XAE_RCW1_FCS_MASK, 103 }, { /* Turn on FCS insertion on transmit packets */ 104 .opt = XAE_OPTION_FCS_INSERT, 105 .reg = XAE_TC_OFFSET, 106 .m_or = XAE_TC_FCS_MASK, 107 }, { /* Turn off length/type field checking on receive packets */ 108 .opt = XAE_OPTION_LENTYPE_ERR, 109 .reg = XAE_RCW1_OFFSET, 110 .m_or = XAE_RCW1_LT_DIS_MASK, 111 }, { /* Turn on Rx flow control */ 112 .opt = XAE_OPTION_FLOW_CONTROL, 113 .reg = XAE_FCC_OFFSET, 114 .m_or = XAE_FCC_FCRX_MASK, 115 }, { /* Turn on Tx flow control */ 116 .opt = XAE_OPTION_FLOW_CONTROL, 117 .reg = XAE_FCC_OFFSET, 118 .m_or = XAE_FCC_FCTX_MASK, 119 }, { /* Turn on promiscuous frame filtering */ 120 .opt = XAE_OPTION_PROMISC, 121 .reg = XAE_FMI_OFFSET, 122 .m_or = XAE_FMI_PM_MASK, 123 }, { /* Enable transmitter */ 124 .opt = XAE_OPTION_TXEN, 125 .reg = XAE_TC_OFFSET, 126 .m_or = XAE_TC_TX_MASK, 127 }, { /* Enable receiver */ 128 .opt = XAE_OPTION_RXEN, 129 .reg = XAE_RCW1_OFFSET, 130 .m_or = XAE_RCW1_RX_MASK, 131 }, 132 {} 133 }; 134 135 static struct skbuf_dma_descriptor *axienet_get_rx_desc(struct axienet_local *lp, int i) 136 { 137 return lp->rx_skb_ring[i & (RX_BUF_NUM_DEFAULT - 1)]; 138 } 139 140 static struct skbuf_dma_descriptor *axienet_get_tx_desc(struct axienet_local *lp, int i) 141 { 142 return lp->tx_skb_ring[i & (TX_BD_NUM_MAX - 1)]; 143 } 144 145 /** 146 * axienet_dma_in32 - Memory mapped Axi DMA register read 147 * @lp: Pointer to axienet local structure 148 * @reg: Address offset from the base address of the Axi DMA core 149 * 150 * Return: The contents of the Axi DMA register 151 * 152 * This function returns the contents of the corresponding Axi DMA register. 153 */ 154 static inline u32 axienet_dma_in32(struct axienet_local *lp, off_t reg) 155 { 156 return ioread32(lp->dma_regs + reg); 157 } 158 159 static void desc_set_phys_addr(struct axienet_local *lp, dma_addr_t addr, 160 struct axidma_bd *desc) 161 { 162 desc->phys = lower_32_bits(addr); 163 if (lp->features & XAE_FEATURE_DMA_64BIT) 164 desc->phys_msb = upper_32_bits(addr); 165 } 166 167 static dma_addr_t desc_get_phys_addr(struct axienet_local *lp, 168 struct axidma_bd *desc) 169 { 170 dma_addr_t ret = desc->phys; 171 172 if (lp->features & XAE_FEATURE_DMA_64BIT) 173 ret |= ((dma_addr_t)desc->phys_msb << 16) << 16; 174 175 return ret; 176 } 177 178 /** 179 * axienet_free_tx_bufs - Release the buffers still held by the TX ring 180 * @lp: Pointer to the axienet_local structure 181 * 182 * Unmap every descriptor whose mapping is still live, free any skb still 183 * attached, and clear the descriptor for reuse. axienet_free_tx_chain() 184 * clears cntrl when it reclaims a descriptor, so a non-zero value means 185 * the mapping is live. The DMA engine must be stopped. 186 */ 187 static void axienet_free_tx_bufs(struct axienet_local *lp) 188 { 189 struct axidma_bd *cur_p; 190 u32 i; 191 192 for (i = 0; i < lp->tx_bd_num; i++) { 193 cur_p = &lp->tx_bd_v[i]; 194 if (cur_p->cntrl) { 195 dma_addr_t addr = desc_get_phys_addr(lp, cur_p); 196 197 dma_unmap_single(lp->dev, addr, 198 (cur_p->cntrl & 199 XAXIDMA_BD_CTRL_LENGTH_MASK), 200 DMA_TO_DEVICE); 201 } 202 /* not reclaimed by axienet_free_tx_chain(), so a drop */ 203 if (cur_p->skb) 204 dev_kfree_skb_any(cur_p->skb); 205 cur_p->phys = 0; 206 cur_p->phys_msb = 0; 207 cur_p->cntrl = 0; 208 cur_p->status = 0; 209 cur_p->app0 = 0; 210 cur_p->app1 = 0; 211 cur_p->app2 = 0; 212 cur_p->app3 = 0; 213 cur_p->app4 = 0; 214 cur_p->skb = NULL; 215 } 216 } 217 218 /** 219 * axienet_dma_bd_release - Release buffer descriptor rings 220 * @ndev: Pointer to the net_device structure 221 * 222 * This function is used to release the descriptors allocated in 223 * axienet_dma_bd_init. axienet_dma_bd_release is called when Axi Ethernet 224 * driver stop api is called. 225 */ 226 static void axienet_dma_bd_release(struct net_device *ndev) 227 { 228 int i; 229 struct axienet_local *lp = netdev_priv(ndev); 230 231 /* tx_bd_v is NULL if axienet_dma_bd_init() did not get as far as 232 * allocating it, and is cleared below once the ring is freed; 233 * dma_free_coherent() accepts NULL. 234 */ 235 if (lp->tx_bd_v) 236 axienet_free_tx_bufs(lp); 237 238 dma_free_coherent(lp->dev, 239 sizeof(*lp->tx_bd_v) * lp->tx_bd_num, 240 lp->tx_bd_v, 241 lp->tx_bd_p); 242 lp->tx_bd_v = NULL; 243 244 if (!lp->rx_bd_v) 245 return; 246 247 for (i = 0; i < lp->rx_bd_num; i++) { 248 dma_addr_t phys; 249 250 /* A NULL skb means the descriptor holds no buffer: it was 251 * never filled, or axienet_rx_poll() took its skb and could 252 * not refill it, having already unmapped the old buffer. 253 * Such slots can sit anywhere in a ring that has been running. 254 */ 255 if (!lp->rx_bd_v[i].skb) 256 continue; 257 258 /* For each descriptor, we programmed cntrl with the (non-zero) 259 * descriptor size, after it had been successfully allocated. 260 * So a non-zero value in there means we need to unmap it. 261 */ 262 if (lp->rx_bd_v[i].cntrl) { 263 phys = desc_get_phys_addr(lp, &lp->rx_bd_v[i]); 264 dma_unmap_single(lp->dev, phys, 265 lp->max_frm_size, DMA_FROM_DEVICE); 266 } 267 268 dev_kfree_skb(lp->rx_bd_v[i].skb); 269 } 270 271 dma_free_coherent(lp->dev, 272 sizeof(*lp->rx_bd_v) * lp->rx_bd_num, 273 lp->rx_bd_v, 274 lp->rx_bd_p); 275 lp->rx_bd_v = NULL; 276 } 277 278 static u64 axienet_dma_rate(struct axienet_local *lp) 279 { 280 if (lp->axi_clk) 281 return clk_get_rate(lp->axi_clk); 282 return 125000000; /* arbitrary guess if no clock rate set */ 283 } 284 285 /** 286 * axienet_calc_cr() - Calculate control register value 287 * @lp: Device private data 288 * @count: Number of completions before an interrupt 289 * @usec: Microseconds after the last completion before an interrupt 290 * 291 * Calculate a control register value based on the coalescing settings. The 292 * run/stop bit is not set. 293 * 294 * Return: Control register value with coalescing settings configured. 295 */ 296 static u32 axienet_calc_cr(struct axienet_local *lp, u32 count, u32 usec) 297 { 298 u32 cr; 299 300 cr = FIELD_PREP(XAXIDMA_COALESCE_MASK, count) | XAXIDMA_IRQ_IOC_MASK | 301 XAXIDMA_IRQ_ERROR_MASK; 302 /* Only set interrupt delay timer if not generating an interrupt on 303 * the first packet. Otherwise leave at 0 to disable delay interrupt. 304 */ 305 if (count > 1) { 306 u64 clk_rate = axienet_dma_rate(lp); 307 u32 timer; 308 309 /* 1 Timeout Interval = 125 * (clock period of SG clock) */ 310 timer = DIV64_U64_ROUND_CLOSEST((u64)usec * clk_rate, 311 XAXIDMA_DELAY_SCALE); 312 313 timer = min(timer, FIELD_MAX(XAXIDMA_DELAY_MASK)); 314 cr |= FIELD_PREP(XAXIDMA_DELAY_MASK, timer) | 315 XAXIDMA_IRQ_DELAY_MASK; 316 } 317 318 return cr; 319 } 320 321 /** 322 * axienet_coalesce_params() - Extract coalesce parameters from the CR 323 * @lp: Device private data 324 * @cr: The control register to parse 325 * @count: Number of packets before an interrupt 326 * @usec: Idle time (in usec) before an interrupt 327 */ 328 static void axienet_coalesce_params(struct axienet_local *lp, u32 cr, 329 u32 *count, u32 *usec) 330 { 331 u64 clk_rate = axienet_dma_rate(lp); 332 u64 timer = FIELD_GET(XAXIDMA_DELAY_MASK, cr); 333 334 *count = FIELD_GET(XAXIDMA_COALESCE_MASK, cr); 335 *usec = DIV64_U64_ROUND_CLOSEST(timer * XAXIDMA_DELAY_SCALE, clk_rate); 336 } 337 338 /** 339 * axienet_dma_start - Set up DMA registers and start DMA operation 340 * @lp: Pointer to the axienet_local structure 341 */ 342 static void axienet_dma_start(struct axienet_local *lp) 343 { 344 spin_lock_irq(&lp->rx_cr_lock); 345 346 /* Start updating the Rx channel control register */ 347 lp->rx_dma_cr &= ~XAXIDMA_CR_RUNSTOP_MASK; 348 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr); 349 350 /* Populate the tail pointer and bring the Rx Axi DMA engine out of 351 * halted state. This will make the Rx side ready for reception. 352 */ 353 axienet_dma_out_addr(lp, XAXIDMA_RX_CDESC_OFFSET, lp->rx_bd_p); 354 lp->rx_dma_cr |= XAXIDMA_CR_RUNSTOP_MASK; 355 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr); 356 axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, lp->rx_bd_p + 357 (sizeof(*lp->rx_bd_v) * (lp->rx_bd_num - 1))); 358 lp->rx_dma_started = true; 359 360 spin_unlock_irq(&lp->rx_cr_lock); 361 spin_lock_irq(&lp->tx_cr_lock); 362 363 /* Start updating the Tx channel control register */ 364 lp->tx_dma_cr &= ~XAXIDMA_CR_RUNSTOP_MASK; 365 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr); 366 367 /* Write to the RS (Run-stop) bit in the Tx channel control register. 368 * Tx channel is now ready to run. But only after we write to the 369 * tail pointer register that the Tx channel will start transmitting. 370 */ 371 axienet_dma_out_addr(lp, XAXIDMA_TX_CDESC_OFFSET, lp->tx_bd_p); 372 lp->tx_dma_cr |= XAXIDMA_CR_RUNSTOP_MASK; 373 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr); 374 lp->tx_dma_started = true; 375 376 spin_unlock_irq(&lp->tx_cr_lock); 377 } 378 379 /** 380 * axienet_dma_bd_init - Setup buffer descriptor rings for Axi DMA 381 * @ndev: Pointer to the net_device structure 382 * 383 * Return: 0, on success -ENOMEM, on failure 384 * 385 * This function is called to initialize the Rx and Tx DMA descriptor 386 * rings. This initializes the descriptors with required default values 387 * and is called when Axi Ethernet driver reset is called. 388 */ 389 static int axienet_dma_bd_init(struct net_device *ndev) 390 { 391 int i; 392 struct sk_buff *skb; 393 struct axienet_local *lp = netdev_priv(ndev); 394 395 /* Reset the indexes which are used for accessing the BDs */ 396 lp->tx_bd_ci = 0; 397 lp->tx_bd_tail = 0; 398 lp->rx_bd_ci = 0; 399 400 /* Allocate the Tx and Rx buffer descriptors. */ 401 lp->tx_bd_v = dma_alloc_coherent(lp->dev, 402 sizeof(*lp->tx_bd_v) * lp->tx_bd_num, 403 &lp->tx_bd_p, GFP_KERNEL); 404 if (!lp->tx_bd_v) 405 return -ENOMEM; 406 407 lp->rx_bd_v = dma_alloc_coherent(lp->dev, 408 sizeof(*lp->rx_bd_v) * lp->rx_bd_num, 409 &lp->rx_bd_p, GFP_KERNEL); 410 if (!lp->rx_bd_v) 411 goto out; 412 413 for (i = 0; i < lp->tx_bd_num; i++) { 414 dma_addr_t addr = lp->tx_bd_p + 415 sizeof(*lp->tx_bd_v) * 416 ((i + 1) % lp->tx_bd_num); 417 418 lp->tx_bd_v[i].next = lower_32_bits(addr); 419 if (lp->features & XAE_FEATURE_DMA_64BIT) 420 lp->tx_bd_v[i].next_msb = upper_32_bits(addr); 421 } 422 423 for (i = 0; i < lp->rx_bd_num; i++) { 424 dma_addr_t addr; 425 426 addr = lp->rx_bd_p + sizeof(*lp->rx_bd_v) * 427 ((i + 1) % lp->rx_bd_num); 428 lp->rx_bd_v[i].next = lower_32_bits(addr); 429 if (lp->features & XAE_FEATURE_DMA_64BIT) 430 lp->rx_bd_v[i].next_msb = upper_32_bits(addr); 431 432 skb = netdev_alloc_skb_ip_align(ndev, lp->max_frm_size); 433 if (!skb) 434 goto out; 435 436 lp->rx_bd_v[i].skb = skb; 437 addr = dma_map_single(lp->dev, skb->data, 438 lp->max_frm_size, DMA_FROM_DEVICE); 439 if (dma_mapping_error(lp->dev, addr)) { 440 netdev_err(ndev, "DMA mapping error\n"); 441 goto out; 442 } 443 desc_set_phys_addr(lp, addr, &lp->rx_bd_v[i]); 444 445 lp->rx_bd_v[i].cntrl = lp->max_frm_size; 446 } 447 448 axienet_dma_start(lp); 449 450 return 0; 451 out: 452 axienet_dma_bd_release(ndev); 453 return -ENOMEM; 454 } 455 456 /** 457 * axienet_set_mac_address - Write the MAC address 458 * @ndev: Pointer to the net_device structure 459 * @address: 6 byte Address to be written as MAC address 460 * 461 * This function is called to initialize the MAC address of the Axi Ethernet 462 * core. It writes to the UAW0 and UAW1 registers of the core. 463 */ 464 static void axienet_set_mac_address(struct net_device *ndev, 465 const void *address) 466 { 467 struct axienet_local *lp = netdev_priv(ndev); 468 469 if (address) 470 eth_hw_addr_set(ndev, address); 471 if (!is_valid_ether_addr(ndev->dev_addr)) 472 eth_hw_addr_random(ndev); 473 474 /* Set up unicast MAC address filter set its mac address */ 475 axienet_iow(lp, XAE_UAW0_OFFSET, 476 (ndev->dev_addr[0]) | 477 (ndev->dev_addr[1] << 8) | 478 (ndev->dev_addr[2] << 16) | 479 (ndev->dev_addr[3] << 24)); 480 axienet_iow(lp, XAE_UAW1_OFFSET, 481 (((axienet_ior(lp, XAE_UAW1_OFFSET)) & 482 ~XAE_UAW1_UNICASTADDR_MASK) | 483 (ndev->dev_addr[4] | 484 (ndev->dev_addr[5] << 8)))); 485 } 486 487 /** 488 * netdev_set_mac_address - Write the MAC address (from outside the driver) 489 * @ndev: Pointer to the net_device structure 490 * @p: 6 byte Address to be written as MAC address 491 * 492 * Return: 0 for all conditions. Presently, there is no failure case. 493 * 494 * This function is called to initialize the MAC address of the Axi Ethernet 495 * core. It calls the core specific axienet_set_mac_address. This is the 496 * function that goes into net_device_ops structure entry ndo_set_mac_address. 497 */ 498 static int netdev_set_mac_address(struct net_device *ndev, void *p) 499 { 500 struct sockaddr *addr = p; 501 502 axienet_set_mac_address(ndev, addr->sa_data); 503 return 0; 504 } 505 506 /** 507 * axienet_set_multicast_list - Prepare the multicast table 508 * @ndev: Pointer to the net_device structure 509 * 510 * This function is called to initialize the multicast table during 511 * initialization. The Axi Ethernet basic multicast support has a four-entry 512 * multicast table which is initialized here. Additionally this function 513 * goes into the net_device_ops structure entry ndo_set_multicast_list. This 514 * means whenever the multicast table entries need to be updated this 515 * function gets called. 516 */ 517 static void axienet_set_multicast_list(struct net_device *ndev) 518 { 519 int i = 0; 520 u32 reg, af0reg, af1reg; 521 struct axienet_local *lp = netdev_priv(ndev); 522 523 reg = axienet_ior(lp, XAE_FMI_OFFSET); 524 reg &= ~XAE_FMI_PM_MASK; 525 if (ndev->flags & IFF_PROMISC) 526 reg |= XAE_FMI_PM_MASK; 527 else 528 reg &= ~XAE_FMI_PM_MASK; 529 axienet_iow(lp, XAE_FMI_OFFSET, reg); 530 531 if (ndev->flags & IFF_ALLMULTI || 532 netdev_mc_count(ndev) > XAE_MULTICAST_CAM_TABLE_NUM) { 533 reg &= 0xFFFFFF00; 534 axienet_iow(lp, XAE_FMI_OFFSET, reg); 535 axienet_iow(lp, XAE_AF0_OFFSET, 1); /* Multicast bit */ 536 axienet_iow(lp, XAE_AF1_OFFSET, 0); 537 axienet_iow(lp, XAE_AM0_OFFSET, 1); /* ditto */ 538 axienet_iow(lp, XAE_AM1_OFFSET, 0); 539 axienet_iow(lp, XAE_FFE_OFFSET, 1); 540 i = 1; 541 } else if (!netdev_mc_empty(ndev)) { 542 struct netdev_hw_addr *ha; 543 544 netdev_for_each_mc_addr(ha, ndev) { 545 if (i >= XAE_MULTICAST_CAM_TABLE_NUM) 546 break; 547 548 af0reg = (ha->addr[0]); 549 af0reg |= (ha->addr[1] << 8); 550 af0reg |= (ha->addr[2] << 16); 551 af0reg |= (ha->addr[3] << 24); 552 553 af1reg = (ha->addr[4]); 554 af1reg |= (ha->addr[5] << 8); 555 556 reg &= 0xFFFFFF00; 557 reg |= i; 558 559 axienet_iow(lp, XAE_FMI_OFFSET, reg); 560 axienet_iow(lp, XAE_AF0_OFFSET, af0reg); 561 axienet_iow(lp, XAE_AF1_OFFSET, af1reg); 562 axienet_iow(lp, XAE_AM0_OFFSET, 0xffffffff); 563 axienet_iow(lp, XAE_AM1_OFFSET, 0x0000ffff); 564 axienet_iow(lp, XAE_FFE_OFFSET, 1); 565 i++; 566 } 567 } 568 569 for (; i < XAE_MULTICAST_CAM_TABLE_NUM; i++) { 570 reg &= 0xFFFFFF00; 571 reg |= i; 572 axienet_iow(lp, XAE_FMI_OFFSET, reg); 573 axienet_iow(lp, XAE_FFE_OFFSET, 0); 574 } 575 } 576 577 /** 578 * axienet_setoptions - Set an Axi Ethernet option 579 * @ndev: Pointer to the net_device structure 580 * @options: Option to be enabled/disabled 581 * 582 * The Axi Ethernet core has multiple features which can be selectively turned 583 * on or off. The typical options could be jumbo frame option, basic VLAN 584 * option, promiscuous mode option etc. This function is used to set or clear 585 * these options in the Axi Ethernet hardware. This is done through 586 * axienet_option structure . 587 */ 588 static void axienet_setoptions(struct net_device *ndev, u32 options) 589 { 590 int reg; 591 struct axienet_local *lp = netdev_priv(ndev); 592 struct axienet_option *tp = &axienet_options[0]; 593 594 while (tp->opt) { 595 reg = ((axienet_ior(lp, tp->reg)) & ~(tp->m_or)); 596 if (options & tp->opt) 597 reg |= tp->m_or; 598 axienet_iow(lp, tp->reg, reg); 599 tp++; 600 } 601 602 lp->options |= options; 603 } 604 605 static u64 axienet_stat(struct axienet_local *lp, enum temac_stat stat) 606 { 607 u32 counter; 608 609 if (lp->reset_in_progress) 610 return lp->hw_stat_base[stat]; 611 612 counter = axienet_ior(lp, XAE_STATS_OFFSET + stat * 8); 613 return lp->hw_stat_base[stat] + (counter - lp->hw_last_counter[stat]); 614 } 615 616 static void axienet_stats_update(struct axienet_local *lp, bool reset) 617 { 618 enum temac_stat stat; 619 620 write_seqcount_begin(&lp->hw_stats_seqcount); 621 lp->reset_in_progress = reset; 622 for (stat = 0; stat < STAT_COUNT; stat++) { 623 u32 counter = axienet_ior(lp, XAE_STATS_OFFSET + stat * 8); 624 625 lp->hw_stat_base[stat] += counter - lp->hw_last_counter[stat]; 626 lp->hw_last_counter[stat] = counter; 627 } 628 write_seqcount_end(&lp->hw_stats_seqcount); 629 } 630 631 static void axienet_refresh_stats(struct work_struct *work) 632 { 633 struct axienet_local *lp = container_of(work, struct axienet_local, 634 stats_work.work); 635 636 mutex_lock(&lp->stats_lock); 637 axienet_stats_update(lp, false); 638 mutex_unlock(&lp->stats_lock); 639 640 /* Just less than 2^32 bytes at 2.5 GBit/s */ 641 schedule_delayed_work(&lp->stats_work, 13 * HZ); 642 } 643 644 static int __axienet_device_reset(struct axienet_local *lp) 645 { 646 u32 value; 647 int ret; 648 649 /* Save statistics counters in case they will be reset */ 650 mutex_lock(&lp->stats_lock); 651 if (lp->features & XAE_FEATURE_STATS) 652 axienet_stats_update(lp, true); 653 654 /* Reset Axi DMA. This would reset Axi Ethernet core as well. The reset 655 * process of Axi DMA takes a while to complete as all pending 656 * commands/transfers will be flushed or completed during this 657 * reset process. 658 * Note that even though both TX and RX have their own reset register, 659 * they both reset the entire DMA core, so only one needs to be used. 660 */ 661 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, XAXIDMA_CR_RESET_MASK); 662 ret = read_poll_timeout(axienet_dma_in32, value, 663 !(value & XAXIDMA_CR_RESET_MASK), 664 DELAY_OF_ONE_MILLISEC, 50000, false, lp, 665 XAXIDMA_TX_CR_OFFSET); 666 if (ret) { 667 dev_err(lp->dev, "%s: DMA reset timeout!\n", __func__); 668 goto out; 669 } 670 671 /* Wait for PhyRstCmplt bit to be set, indicating the PHY reset has finished */ 672 ret = read_poll_timeout(axienet_ior, value, 673 value & XAE_INT_PHYRSTCMPLT_MASK, 674 DELAY_OF_ONE_MILLISEC, 50000, false, lp, 675 XAE_IS_OFFSET); 676 if (ret) { 677 dev_err(lp->dev, "%s: timeout waiting for PhyRstCmplt\n", __func__); 678 goto out; 679 } 680 681 /* Update statistics counters with new values */ 682 if (lp->features & XAE_FEATURE_STATS) { 683 enum temac_stat stat; 684 685 write_seqcount_begin(&lp->hw_stats_seqcount); 686 lp->reset_in_progress = false; 687 for (stat = 0; stat < STAT_COUNT; stat++) { 688 u32 counter = 689 axienet_ior(lp, XAE_STATS_OFFSET + stat * 8); 690 691 lp->hw_stat_base[stat] += 692 lp->hw_last_counter[stat] - counter; 693 lp->hw_last_counter[stat] = counter; 694 } 695 write_seqcount_end(&lp->hw_stats_seqcount); 696 } 697 698 out: 699 mutex_unlock(&lp->stats_lock); 700 return ret; 701 } 702 703 /** 704 * axienet_dma_stop - Stop DMA operation 705 * @lp: Pointer to the axienet_local structure 706 */ 707 static void axienet_dma_stop(struct axienet_local *lp) 708 { 709 int count; 710 u32 cr, sr; 711 712 spin_lock_irq(&lp->rx_cr_lock); 713 714 cr = lp->rx_dma_cr & ~(XAXIDMA_CR_RUNSTOP_MASK | XAXIDMA_IRQ_ALL_MASK); 715 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr); 716 lp->rx_dma_started = false; 717 718 spin_unlock_irq(&lp->rx_cr_lock); 719 synchronize_irq(lp->rx_irq); 720 721 spin_lock_irq(&lp->tx_cr_lock); 722 723 cr = lp->tx_dma_cr & ~(XAXIDMA_CR_RUNSTOP_MASK | XAXIDMA_IRQ_ALL_MASK); 724 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr); 725 lp->tx_dma_started = false; 726 727 spin_unlock_irq(&lp->tx_cr_lock); 728 synchronize_irq(lp->tx_irq); 729 730 /* Give DMAs a chance to halt gracefully */ 731 sr = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 732 for (count = 0; !(sr & XAXIDMA_SR_HALT_MASK) && count < 5; ++count) { 733 msleep(20); 734 sr = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 735 } 736 737 sr = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 738 for (count = 0; !(sr & XAXIDMA_SR_HALT_MASK) && count < 5; ++count) { 739 msleep(20); 740 sr = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 741 } 742 743 /* Do a reset to ensure DMA is really stopped */ 744 axienet_lock_mii(lp); 745 __axienet_device_reset(lp); 746 axienet_unlock_mii(lp); 747 } 748 749 /** 750 * axienet_device_reset - Reset and initialize the Axi Ethernet hardware. 751 * @ndev: Pointer to the net_device structure 752 * 753 * This function is called to reset and initialize the Axi Ethernet core. This 754 * is typically called during initialization. It does a reset of the Axi DMA 755 * Rx/Tx channels and initializes the Axi DMA BDs. Since Axi DMA reset lines 756 * are connected to Axi Ethernet reset lines, this in turn resets the Axi 757 * Ethernet core. No separate hardware reset is done for the Axi Ethernet 758 * core. 759 * 760 * Return: 0 on success or a negative error number otherwise. 761 */ 762 static int axienet_device_reset(struct net_device *ndev) 763 { 764 u32 axienet_status; 765 struct axienet_local *lp = netdev_priv(ndev); 766 int ret; 767 768 lp->max_frm_size = XAE_MAX_VLAN_FRAME_SIZE; 769 lp->options |= XAE_OPTION_VLAN; 770 lp->options &= (~XAE_OPTION_JUMBO); 771 772 if (ndev->mtu > XAE_MTU && ndev->mtu <= XAE_JUMBO_MTU) { 773 lp->max_frm_size = ndev->mtu + VLAN_ETH_HLEN + 774 XAE_TRL_SIZE; 775 776 if (lp->max_frm_size <= lp->rxmem) 777 lp->options |= XAE_OPTION_JUMBO; 778 } 779 780 if (!lp->use_dmaengine) { 781 ret = __axienet_device_reset(lp); 782 if (ret) 783 return ret; 784 785 ret = axienet_dma_bd_init(ndev); 786 if (ret) { 787 netdev_err(ndev, "%s: descriptor allocation failed\n", 788 __func__); 789 return ret; 790 } 791 } 792 793 axienet_status = axienet_ior(lp, XAE_RCW1_OFFSET); 794 axienet_status &= ~XAE_RCW1_RX_MASK; 795 axienet_iow(lp, XAE_RCW1_OFFSET, axienet_status); 796 797 axienet_status = axienet_ior(lp, XAE_IP_OFFSET); 798 if (axienet_status & XAE_INT_RXRJECT_MASK) 799 axienet_iow(lp, XAE_IS_OFFSET, XAE_INT_RXRJECT_MASK); 800 axienet_iow(lp, XAE_IE_OFFSET, lp->eth_irq > 0 ? 801 XAE_INT_RECV_ERROR_MASK : 0); 802 803 axienet_iow(lp, XAE_FCC_OFFSET, XAE_FCC_FCRX_MASK); 804 805 /* Sync default options with HW but leave receiver and 806 * transmitter disabled. 807 */ 808 axienet_setoptions(ndev, lp->options & 809 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 810 axienet_set_mac_address(ndev, NULL); 811 axienet_set_multicast_list(ndev); 812 axienet_setoptions(ndev, lp->options); 813 814 netif_trans_update(ndev); 815 816 return 0; 817 } 818 819 /** 820 * axienet_free_tx_chain - Clean up a series of linked TX descriptors. 821 * @lp: Pointer to the axienet_local structure 822 * @first_bd: Index of first descriptor to clean up 823 * @nr_bds: Max number of descriptors to clean up 824 * @force: Whether to clean descriptors even if not complete 825 * @sizep: Pointer to a u32 accumulating the total byte count of 826 * completed packets (using skb->len). Ignored if NULL. 827 * @budget: NAPI budget (use 0 when not called from NAPI poll) 828 * 829 * Would either be called after a successful transmit operation, or after 830 * there was an error when setting up the chain. 831 * 832 * Return: The number of packets handled. 833 */ 834 static int axienet_free_tx_chain(struct axienet_local *lp, u32 first_bd, 835 int nr_bds, bool force, u32 *sizep, int budget) 836 { 837 struct axidma_bd *cur_p; 838 unsigned int status; 839 int i, packets = 0; 840 dma_addr_t phys; 841 842 for (i = 0; i < nr_bds; i++) { 843 cur_p = &lp->tx_bd_v[(first_bd + i) % lp->tx_bd_num]; 844 status = cur_p->status; 845 846 /* If force is not specified, clean up only descriptors 847 * that have been completed by the MAC. 848 */ 849 if (!force && !(status & XAXIDMA_BD_STS_COMPLETE_MASK)) 850 break; 851 852 /* Ensure we see complete descriptor update */ 853 dma_rmb(); 854 phys = desc_get_phys_addr(lp, cur_p); 855 dma_unmap_single(lp->dev, phys, 856 (cur_p->cntrl & XAXIDMA_BD_CTRL_LENGTH_MASK), 857 DMA_TO_DEVICE); 858 859 if (cur_p->skb && (status & XAXIDMA_BD_STS_COMPLETE_MASK)) { 860 if (sizep) 861 *sizep += cur_p->skb->len; 862 napi_consume_skb(cur_p->skb, budget); 863 packets++; 864 } 865 866 cur_p->app0 = 0; 867 cur_p->app1 = 0; 868 cur_p->app2 = 0; 869 cur_p->app4 = 0; 870 cur_p->skb = NULL; 871 /* ensure our transmit path and device don't prematurely see status cleared */ 872 wmb(); 873 cur_p->cntrl = 0; 874 cur_p->status = 0; 875 } 876 877 if (!force) { 878 lp->tx_bd_ci += i; 879 if (lp->tx_bd_ci >= lp->tx_bd_num) 880 lp->tx_bd_ci %= lp->tx_bd_num; 881 } 882 883 return packets; 884 } 885 886 /** 887 * axienet_check_tx_bd_space - Checks if a BD/group of BDs are currently busy 888 * @lp: Pointer to the axienet_local structure 889 * @num_frag: The number of BDs to check for 890 * 891 * Return: 0, on success 892 * NETDEV_TX_BUSY, if any of the descriptors are not free 893 * 894 * This function is invoked before BDs are allocated and transmission starts. 895 * This function returns 0 if a BD or group of BDs can be allocated for 896 * transmission. If the BD or any of the BDs are not free the function 897 * returns a busy status. 898 */ 899 static inline int axienet_check_tx_bd_space(struct axienet_local *lp, 900 int num_frag) 901 { 902 struct axidma_bd *cur_p; 903 904 /* Ensure we see all descriptor updates from device or TX polling */ 905 rmb(); 906 cur_p = &lp->tx_bd_v[(READ_ONCE(lp->tx_bd_tail) + num_frag) % 907 lp->tx_bd_num]; 908 if (cur_p->cntrl) 909 return NETDEV_TX_BUSY; 910 return 0; 911 } 912 913 /** 914 * axienet_dma_tx_cb - DMA engine callback for TX channel. 915 * @data: Pointer to the axienet_local structure. 916 * @result: error reporting through dmaengine_result. 917 * This function is called by dmaengine driver for TX channel to notify 918 * that the transmit is done. 919 */ 920 static void axienet_dma_tx_cb(void *data, const struct dmaengine_result *result) 921 { 922 struct skbuf_dma_descriptor *skbuf_dma; 923 struct axienet_local *lp = data; 924 struct netdev_queue *txq; 925 int len; 926 927 skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_tail++); 928 len = skbuf_dma->skb->len; 929 txq = skb_get_tx_queue(lp->ndev, skbuf_dma->skb); 930 u64_stats_update_begin(&lp->tx_stat_sync); 931 u64_stats_add(&lp->tx_bytes, len); 932 u64_stats_add(&lp->tx_packets, 1); 933 u64_stats_update_end(&lp->tx_stat_sync); 934 dma_unmap_sg(lp->dev, skbuf_dma->sgl, skbuf_dma->sg_len, DMA_TO_DEVICE); 935 dev_consume_skb_any(skbuf_dma->skb); 936 skbuf_dma->skb = NULL; 937 netif_txq_completed_wake(txq, 1, len, 938 CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX), 939 2); 940 } 941 942 /** 943 * axienet_start_xmit_dmaengine - Starts the transmission. 944 * @skb: sk_buff pointer that contains data to be Txed. 945 * @ndev: Pointer to net_device structure. 946 * 947 * Return: NETDEV_TX_OK on success or any non space errors. 948 * NETDEV_TX_BUSY when free element in TX skb ring buffer 949 * is not available. 950 * 951 * This function is invoked to initiate transmission. The 952 * function sets the skbs, register dma callback API and submit 953 * the dma transaction. 954 * Additionally if checksum offloading is supported, 955 * it populates AXI Stream Control fields with appropriate values. 956 */ 957 static netdev_tx_t 958 axienet_start_xmit_dmaengine(struct sk_buff *skb, struct net_device *ndev) 959 { 960 struct dma_async_tx_descriptor *dma_tx_desc = NULL; 961 struct axienet_local *lp = netdev_priv(ndev); 962 u32 app_metadata[DMA_NUM_APP_WORDS] = {0}; 963 struct skbuf_dma_descriptor *skbuf_dma; 964 struct dma_device *dma_dev; 965 struct netdev_queue *txq; 966 u32 csum_start_off; 967 u32 csum_index_off; 968 int sg_len; 969 int ret; 970 971 dma_dev = lp->tx_chan->device; 972 sg_len = skb_shinfo(skb)->nr_frags + 1; 973 if (CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX) <= 1) { 974 netif_stop_queue(ndev); 975 if (net_ratelimit()) 976 netdev_warn(ndev, "TX ring unexpectedly full\n"); 977 return NETDEV_TX_BUSY; 978 } 979 980 skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_head); 981 if (!skbuf_dma) 982 goto xmit_error_drop_skb; 983 984 sg_init_table(skbuf_dma->sgl, sg_len); 985 ret = skb_to_sgvec(skb, skbuf_dma->sgl, 0, skb->len); 986 if (ret < 0) 987 goto xmit_error_drop_skb; 988 989 ret = dma_map_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE); 990 if (!ret) 991 goto xmit_error_drop_skb; 992 993 /* Fill up app fields for checksum */ 994 if (skb->ip_summed == CHECKSUM_PARTIAL) { 995 if (lp->features & XAE_FEATURE_FULL_TX_CSUM) { 996 /* Tx Full Checksum Offload Enabled */ 997 app_metadata[0] |= 2; 998 } else if (lp->features & XAE_FEATURE_PARTIAL_TX_CSUM) { 999 csum_start_off = skb_transport_offset(skb); 1000 csum_index_off = csum_start_off + skb->csum_offset; 1001 /* Tx Partial Checksum Offload Enabled */ 1002 app_metadata[0] |= 1; 1003 app_metadata[1] = (csum_start_off << 16) | csum_index_off; 1004 } 1005 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 1006 app_metadata[0] |= 2; /* Tx Full Checksum Offload Enabled */ 1007 } 1008 1009 dma_tx_desc = dma_dev->device_prep_slave_sg(lp->tx_chan, skbuf_dma->sgl, 1010 sg_len, DMA_MEM_TO_DEV, 1011 DMA_PREP_INTERRUPT, (void *)app_metadata); 1012 if (!dma_tx_desc) 1013 goto xmit_error_unmap_sg; 1014 1015 skbuf_dma->skb = skb; 1016 skbuf_dma->sg_len = sg_len; 1017 dma_tx_desc->callback_param = lp; 1018 dma_tx_desc->callback_result = axienet_dma_tx_cb; 1019 /* Claim the ring slot only after the steps above that can fail. The 1020 * completion callback can't run before dmaengine_submit(). 1021 */ 1022 lp->tx_ring_head++; 1023 txq = skb_get_tx_queue(lp->ndev, skb); 1024 netdev_tx_sent_queue(txq, skb->len); 1025 netif_txq_maybe_stop(txq, CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX), 1026 1, 2); 1027 1028 dmaengine_submit(dma_tx_desc); 1029 dma_async_issue_pending(lp->tx_chan); 1030 return NETDEV_TX_OK; 1031 1032 xmit_error_unmap_sg: 1033 dma_unmap_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE); 1034 xmit_error_drop_skb: 1035 dev_kfree_skb_any(skb); 1036 return NETDEV_TX_OK; 1037 } 1038 1039 /** 1040 * axienet_tx_poll - Invoked once a transmit is completed by the 1041 * Axi DMA Tx channel. 1042 * @napi: Pointer to NAPI structure. 1043 * @budget: NAPI budget, or 0 when polled by netpoll. 1044 * 1045 * Return: Always 0. TX completions are not counted against the budget. 1046 * 1047 * This function is invoked from the NAPI processing to notify the completion 1048 * of transmit operation. It clears fields in the corresponding Tx BDs and 1049 * unmaps the corresponding buffer so that CPU can regain ownership of the 1050 * buffer. It finally invokes "netif_wake_queue" to restart transmission if 1051 * required. 1052 */ 1053 static int axienet_tx_poll(struct napi_struct *napi, int budget) 1054 { 1055 struct axienet_local *lp = container_of(napi, struct axienet_local, napi_tx); 1056 struct net_device *ndev = lp->ndev; 1057 u32 size = 0; 1058 int packets; 1059 1060 packets = axienet_free_tx_chain(lp, lp->tx_bd_ci, lp->tx_bd_num, false, 1061 &size, budget); 1062 1063 if (packets) { 1064 netdev_completed_queue(ndev, packets, size); 1065 u64_stats_update_begin(&lp->tx_stat_sync); 1066 u64_stats_add(&lp->tx_packets, packets); 1067 u64_stats_add(&lp->tx_bytes, size); 1068 u64_stats_update_end(&lp->tx_stat_sync); 1069 1070 /* Matches barrier in axienet_start_xmit */ 1071 smp_mb(); 1072 1073 if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) 1074 netif_wake_queue(ndev); 1075 } 1076 1077 /* The whole ring was reclaimed above, so there is nothing left to 1078 * poll for: complete with no work done, as TX completions do not 1079 * count against the budget. netpoll polls with a budget of 0 and 1080 * must not complete NAPI. 1081 */ 1082 if (budget && napi_complete_done(napi, 0)) { 1083 /* Re-enable TX completion interrupts. This should 1084 * cause an immediate interrupt if any TX packets are 1085 * already pending. 1086 */ 1087 spin_lock_irq(&lp->tx_cr_lock); 1088 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr); 1089 spin_unlock_irq(&lp->tx_cr_lock); 1090 } 1091 return 0; 1092 } 1093 1094 /** 1095 * axienet_start_xmit - Starts the transmission. 1096 * @skb: sk_buff pointer that contains data to be Txed. 1097 * @ndev: Pointer to net_device structure. 1098 * 1099 * Return: NETDEV_TX_OK, on success 1100 * NETDEV_TX_BUSY, if any of the descriptors are not free 1101 * 1102 * This function is invoked from upper layers to initiate transmission. The 1103 * function uses the next available free BDs and populates their fields to 1104 * start the transmission. Additionally if checksum offloading is supported, 1105 * it populates AXI Stream Control fields with appropriate values. 1106 */ 1107 static netdev_tx_t 1108 axienet_start_xmit(struct sk_buff *skb, struct net_device *ndev) 1109 { 1110 u32 ii; 1111 u32 num_frag; 1112 u32 csum_start_off; 1113 u32 csum_index_off; 1114 skb_frag_t *frag; 1115 dma_addr_t tail_p, phys; 1116 u32 orig_tail_ptr, new_tail_ptr; 1117 struct axienet_local *lp = netdev_priv(ndev); 1118 struct axidma_bd *cur_p; 1119 1120 orig_tail_ptr = lp->tx_bd_tail; 1121 new_tail_ptr = orig_tail_ptr; 1122 1123 num_frag = skb_shinfo(skb)->nr_frags; 1124 cur_p = &lp->tx_bd_v[orig_tail_ptr]; 1125 1126 if (axienet_check_tx_bd_space(lp, num_frag + 1)) { 1127 /* Should not happen as last start_xmit call should have 1128 * checked for sufficient space and queue should only be 1129 * woken when sufficient space is available. 1130 */ 1131 netif_stop_queue(ndev); 1132 if (net_ratelimit()) 1133 netdev_warn(ndev, "TX ring unexpectedly full\n"); 1134 return NETDEV_TX_BUSY; 1135 } 1136 1137 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1138 if (lp->features & XAE_FEATURE_FULL_TX_CSUM) { 1139 /* Tx Full Checksum Offload Enabled */ 1140 cur_p->app0 |= 2; 1141 } else if (lp->features & XAE_FEATURE_PARTIAL_TX_CSUM) { 1142 csum_start_off = skb_transport_offset(skb); 1143 csum_index_off = csum_start_off + skb->csum_offset; 1144 /* Tx Partial Checksum Offload Enabled */ 1145 cur_p->app0 |= 1; 1146 cur_p->app1 = (csum_start_off << 16) | csum_index_off; 1147 } 1148 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 1149 cur_p->app0 |= 2; /* Tx Full Checksum Offload Enabled */ 1150 } 1151 1152 phys = dma_map_single(lp->dev, skb->data, 1153 skb_headlen(skb), DMA_TO_DEVICE); 1154 if (unlikely(dma_mapping_error(lp->dev, phys))) { 1155 if (net_ratelimit()) 1156 netdev_err(ndev, "TX DMA mapping error\n"); 1157 ndev->stats.tx_dropped++; 1158 dev_kfree_skb_any(skb); 1159 return NETDEV_TX_OK; 1160 } 1161 desc_set_phys_addr(lp, phys, cur_p); 1162 cur_p->cntrl = skb_headlen(skb) | XAXIDMA_BD_CTRL_TXSOF_MASK; 1163 1164 for (ii = 0; ii < num_frag; ii++) { 1165 if (++new_tail_ptr >= lp->tx_bd_num) 1166 new_tail_ptr = 0; 1167 cur_p = &lp->tx_bd_v[new_tail_ptr]; 1168 frag = &skb_shinfo(skb)->frags[ii]; 1169 phys = dma_map_single(lp->dev, 1170 skb_frag_address(frag), 1171 skb_frag_size(frag), 1172 DMA_TO_DEVICE); 1173 if (unlikely(dma_mapping_error(lp->dev, phys))) { 1174 if (net_ratelimit()) 1175 netdev_err(ndev, "TX DMA mapping error\n"); 1176 ndev->stats.tx_dropped++; 1177 axienet_free_tx_chain(lp, orig_tail_ptr, ii + 1, 1178 true, NULL, 0); 1179 dev_kfree_skb_any(skb); 1180 return NETDEV_TX_OK; 1181 } 1182 desc_set_phys_addr(lp, phys, cur_p); 1183 cur_p->cntrl = skb_frag_size(frag); 1184 } 1185 1186 cur_p->cntrl |= XAXIDMA_BD_CTRL_TXEOF_MASK; 1187 cur_p->skb = skb; 1188 1189 tail_p = lp->tx_bd_p + sizeof(*lp->tx_bd_v) * new_tail_ptr; 1190 if (++new_tail_ptr >= lp->tx_bd_num) 1191 new_tail_ptr = 0; 1192 WRITE_ONCE(lp->tx_bd_tail, new_tail_ptr); 1193 netdev_sent_queue(ndev, skb->len); 1194 1195 /* Start the transfer */ 1196 axienet_dma_out_addr(lp, XAXIDMA_TX_TDESC_OFFSET, tail_p); 1197 1198 /* Stop queue if next transmit may not have space */ 1199 if (axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) { 1200 netif_stop_queue(ndev); 1201 1202 /* Matches barrier in axienet_tx_poll */ 1203 smp_mb(); 1204 1205 /* Space might have just been freed - check again */ 1206 if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) 1207 netif_wake_queue(ndev); 1208 } 1209 1210 return NETDEV_TX_OK; 1211 } 1212 1213 /** 1214 * axienet_dma_rx_cb - DMA engine callback for RX channel. 1215 * @data: Pointer to the skbuf_dma_descriptor structure. 1216 * @result: error reporting through dmaengine_result. 1217 * This function is called by dmaengine driver for RX channel to notify 1218 * that the packet is received. 1219 */ 1220 static void axienet_dma_rx_cb(void *data, const struct dmaengine_result *result) 1221 { 1222 struct skbuf_dma_descriptor *skbuf_dma; 1223 size_t meta_len, meta_max_len, rx_len; 1224 struct axienet_local *lp = data; 1225 struct sk_buff *skb; 1226 u32 *app_metadata; 1227 int i; 1228 1229 skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_tail++); 1230 skb = skbuf_dma->skb; 1231 app_metadata = dmaengine_desc_get_metadata_ptr(skbuf_dma->desc, &meta_len, 1232 &meta_max_len); 1233 dma_unmap_single(lp->dev, skbuf_dma->dma_address, lp->max_frm_size, 1234 DMA_FROM_DEVICE); 1235 skbuf_dma->skb = NULL; 1236 1237 if (IS_ERR(app_metadata)) { 1238 if (net_ratelimit()) 1239 netdev_err(lp->ndev, "Failed to get RX metadata pointer\n"); 1240 dev_kfree_skb_any(skb); 1241 lp->ndev->stats.rx_dropped++; 1242 goto rx_submit; 1243 } 1244 1245 /* TODO: Derive app word index programmatically */ 1246 rx_len = (app_metadata[LEN_APP] & 0xFFFF); 1247 skb_put(skb, rx_len); 1248 skb->protocol = eth_type_trans(skb, lp->ndev); 1249 skb->ip_summed = CHECKSUM_NONE; 1250 1251 __netif_rx(skb); 1252 u64_stats_update_begin(&lp->rx_stat_sync); 1253 u64_stats_add(&lp->rx_packets, 1); 1254 u64_stats_add(&lp->rx_bytes, rx_len); 1255 u64_stats_update_end(&lp->rx_stat_sync); 1256 1257 rx_submit: 1258 spin_lock(&lp->rx_submit_lock); 1259 if (lp->stopping) { 1260 spin_unlock(&lp->rx_submit_lock); 1261 return; 1262 } 1263 1264 for (i = 0; i < CIRC_SPACE(lp->rx_ring_head, lp->rx_ring_tail, 1265 RX_BUF_NUM_DEFAULT); i++) 1266 axienet_rx_submit_desc(lp->ndev); 1267 dma_async_issue_pending(lp->rx_chan); 1268 spin_unlock(&lp->rx_submit_lock); 1269 } 1270 1271 /** 1272 * axienet_rx_poll - Triggered by RX ISR to complete the BD processing. 1273 * @napi: Pointer to NAPI structure. 1274 * @budget: Max number of RX packets to process. 1275 * 1276 * Return: Number of RX packets processed. 1277 */ 1278 static int axienet_rx_poll(struct napi_struct *napi, int budget) 1279 { 1280 u32 length; 1281 u32 csumstatus; 1282 u32 size = 0; 1283 int packets = 0; 1284 dma_addr_t tail_p = 0; 1285 struct axidma_bd *cur_p; 1286 struct sk_buff *skb, *new_skb; 1287 struct axienet_local *lp = container_of(napi, struct axienet_local, napi_rx); 1288 1289 cur_p = &lp->rx_bd_v[lp->rx_bd_ci]; 1290 1291 while (packets < budget && (cur_p->status & XAXIDMA_BD_STS_COMPLETE_MASK)) { 1292 dma_addr_t phys; 1293 1294 /* Ensure we see complete descriptor update */ 1295 dma_rmb(); 1296 1297 skb = cur_p->skb; 1298 cur_p->skb = NULL; 1299 1300 /* skb could be NULL if a previous pass already received the 1301 * packet for this slot in the ring, but failed to refill it 1302 * with a newly allocated buffer. In this case, don't try to 1303 * receive it again. 1304 */ 1305 if (likely(skb)) { 1306 length = cur_p->app4 & 0x0000FFFF; 1307 1308 phys = desc_get_phys_addr(lp, cur_p); 1309 dma_unmap_single(lp->dev, phys, lp->max_frm_size, 1310 DMA_FROM_DEVICE); 1311 1312 skb_put(skb, length); 1313 skb->protocol = eth_type_trans(skb, lp->ndev); 1314 /*skb_checksum_none_assert(skb);*/ 1315 skb->ip_summed = CHECKSUM_NONE; 1316 1317 /* if we're doing Rx csum offload, set it up */ 1318 if (lp->features & XAE_FEATURE_FULL_RX_CSUM) { 1319 csumstatus = (cur_p->app2 & 1320 XAE_FULL_CSUM_STATUS_MASK) >> 3; 1321 if (csumstatus == XAE_IP_TCP_CSUM_VALIDATED || 1322 csumstatus == XAE_IP_UDP_CSUM_VALIDATED) { 1323 skb->ip_summed = CHECKSUM_UNNECESSARY; 1324 } 1325 } else if (lp->features & XAE_FEATURE_PARTIAL_RX_CSUM) { 1326 skb->csum = be32_to_cpu(cur_p->app3 & 0xFFFF); 1327 skb->ip_summed = CHECKSUM_COMPLETE; 1328 } 1329 1330 napi_gro_receive(napi, skb); 1331 1332 size += length; 1333 packets++; 1334 } 1335 1336 new_skb = napi_alloc_skb(napi, lp->max_frm_size); 1337 if (!new_skb) 1338 break; 1339 1340 phys = dma_map_single(lp->dev, new_skb->data, 1341 lp->max_frm_size, 1342 DMA_FROM_DEVICE); 1343 if (unlikely(dma_mapping_error(lp->dev, phys))) { 1344 if (net_ratelimit()) 1345 netdev_err(lp->ndev, "RX DMA mapping error\n"); 1346 dev_kfree_skb(new_skb); 1347 break; 1348 } 1349 desc_set_phys_addr(lp, phys, cur_p); 1350 1351 cur_p->cntrl = lp->max_frm_size; 1352 cur_p->status = 0; 1353 cur_p->skb = new_skb; 1354 1355 /* Only update tail_p to mark this slot as usable after it has 1356 * been successfully refilled. 1357 */ 1358 tail_p = lp->rx_bd_p + sizeof(*lp->rx_bd_v) * lp->rx_bd_ci; 1359 1360 if (++lp->rx_bd_ci >= lp->rx_bd_num) 1361 lp->rx_bd_ci = 0; 1362 cur_p = &lp->rx_bd_v[lp->rx_bd_ci]; 1363 } 1364 1365 u64_stats_update_begin(&lp->rx_stat_sync); 1366 u64_stats_add(&lp->rx_packets, packets); 1367 u64_stats_add(&lp->rx_bytes, size); 1368 u64_stats_update_end(&lp->rx_stat_sync); 1369 1370 if (tail_p) 1371 axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, tail_p); 1372 1373 if (packets < budget && napi_complete_done(napi, packets)) { 1374 if (READ_ONCE(lp->rx_dim_enabled)) { 1375 struct dim_sample sample = { 1376 .time = ktime_get(), 1377 /* Safe because we are the only writer */ 1378 .pkt_ctr = u64_stats_read(&lp->rx_packets), 1379 .byte_ctr = u64_stats_read(&lp->rx_bytes), 1380 .event_ctr = READ_ONCE(lp->rx_irqs), 1381 }; 1382 1383 net_dim(&lp->rx_dim, &sample); 1384 } 1385 1386 /* Re-enable RX completion interrupts. This should 1387 * cause an immediate interrupt if any RX packets are 1388 * already pending. 1389 */ 1390 spin_lock_irq(&lp->rx_cr_lock); 1391 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr); 1392 spin_unlock_irq(&lp->rx_cr_lock); 1393 } 1394 return packets; 1395 } 1396 1397 /** 1398 * axienet_tx_irq - Tx Done Isr. 1399 * @irq: irq number 1400 * @_ndev: net_device pointer 1401 * 1402 * Return: IRQ_HANDLED if device generated a TX interrupt, IRQ_NONE otherwise. 1403 * 1404 * This is the Axi DMA Tx done Isr. It invokes NAPI polling to complete the 1405 * TX BD processing. 1406 */ 1407 static irqreturn_t axienet_tx_irq(int irq, void *_ndev) 1408 { 1409 unsigned int status; 1410 struct net_device *ndev = _ndev; 1411 struct axienet_local *lp = netdev_priv(ndev); 1412 1413 status = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 1414 1415 if (!(status & XAXIDMA_IRQ_ALL_MASK)) 1416 return IRQ_NONE; 1417 1418 axienet_dma_out32(lp, XAXIDMA_TX_SR_OFFSET, status); 1419 1420 if (unlikely(status & XAXIDMA_IRQ_ERROR_MASK)) { 1421 netdev_err(ndev, "DMA Tx error 0x%x\n", status); 1422 netdev_err(ndev, "Current BD is at: 0x%x%08x\n", 1423 (lp->tx_bd_v[lp->tx_bd_ci]).phys_msb, 1424 (lp->tx_bd_v[lp->tx_bd_ci]).phys); 1425 schedule_work(&lp->dma_err_task); 1426 } else { 1427 /* Disable further TX completion interrupts and schedule 1428 * NAPI to handle the completions. 1429 */ 1430 if (napi_schedule_prep(&lp->napi_tx)) { 1431 u32 cr; 1432 1433 spin_lock(&lp->tx_cr_lock); 1434 cr = lp->tx_dma_cr; 1435 cr &= ~(XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_DELAY_MASK); 1436 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr); 1437 spin_unlock(&lp->tx_cr_lock); 1438 __napi_schedule(&lp->napi_tx); 1439 } 1440 } 1441 1442 return IRQ_HANDLED; 1443 } 1444 1445 /** 1446 * axienet_rx_irq - Rx Isr. 1447 * @irq: irq number 1448 * @_ndev: net_device pointer 1449 * 1450 * Return: IRQ_HANDLED if device generated a RX interrupt, IRQ_NONE otherwise. 1451 * 1452 * This is the Axi DMA Rx Isr. It invokes NAPI polling to complete the RX BD 1453 * processing. 1454 */ 1455 static irqreturn_t axienet_rx_irq(int irq, void *_ndev) 1456 { 1457 unsigned int status; 1458 struct net_device *ndev = _ndev; 1459 struct axienet_local *lp = netdev_priv(ndev); 1460 1461 status = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 1462 1463 if (!(status & XAXIDMA_IRQ_ALL_MASK)) 1464 return IRQ_NONE; 1465 1466 axienet_dma_out32(lp, XAXIDMA_RX_SR_OFFSET, status); 1467 1468 if (unlikely(status & XAXIDMA_IRQ_ERROR_MASK)) { 1469 netdev_err(ndev, "DMA Rx error 0x%x\n", status); 1470 netdev_err(ndev, "Current BD is at: 0x%x%08x\n", 1471 (lp->rx_bd_v[lp->rx_bd_ci]).phys_msb, 1472 (lp->rx_bd_v[lp->rx_bd_ci]).phys); 1473 schedule_work(&lp->dma_err_task); 1474 } else { 1475 /* Disable further RX completion interrupts and schedule 1476 * NAPI receive. 1477 */ 1478 WRITE_ONCE(lp->rx_irqs, READ_ONCE(lp->rx_irqs) + 1); 1479 if (napi_schedule_prep(&lp->napi_rx)) { 1480 u32 cr; 1481 1482 spin_lock(&lp->rx_cr_lock); 1483 cr = lp->rx_dma_cr; 1484 cr &= ~(XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_DELAY_MASK); 1485 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr); 1486 spin_unlock(&lp->rx_cr_lock); 1487 1488 __napi_schedule(&lp->napi_rx); 1489 } 1490 } 1491 1492 return IRQ_HANDLED; 1493 } 1494 1495 /** 1496 * axienet_eth_irq - Ethernet core Isr. 1497 * @irq: irq number 1498 * @_ndev: net_device pointer 1499 * 1500 * Return: IRQ_HANDLED if device generated a core interrupt, IRQ_NONE otherwise. 1501 * 1502 * Handle miscellaneous conditions indicated by Ethernet core IRQ. 1503 */ 1504 static irqreturn_t axienet_eth_irq(int irq, void *_ndev) 1505 { 1506 struct net_device *ndev = _ndev; 1507 struct axienet_local *lp = netdev_priv(ndev); 1508 unsigned int pending; 1509 1510 pending = axienet_ior(lp, XAE_IP_OFFSET); 1511 if (!pending) 1512 return IRQ_NONE; 1513 1514 if (pending & XAE_INT_RXFIFOOVR_MASK) 1515 ndev->stats.rx_missed_errors++; 1516 1517 if (pending & XAE_INT_RXRJECT_MASK) 1518 ndev->stats.rx_dropped++; 1519 1520 axienet_iow(lp, XAE_IS_OFFSET, pending); 1521 return IRQ_HANDLED; 1522 } 1523 1524 static void axienet_dma_err_handler(struct work_struct *work); 1525 1526 /** 1527 * axienet_rx_submit_desc - Submit the rx descriptors to dmaengine. 1528 * allocate skbuff, map the scatterlist and obtain a descriptor 1529 * and then add the callback information and submit descriptor. 1530 * 1531 * @ndev: net_device pointer 1532 * 1533 */ 1534 static void axienet_rx_submit_desc(struct net_device *ndev) 1535 { 1536 struct dma_async_tx_descriptor *dma_rx_desc = NULL; 1537 struct axienet_local *lp = netdev_priv(ndev); 1538 struct skbuf_dma_descriptor *skbuf_dma; 1539 struct sk_buff *skb; 1540 dma_addr_t addr; 1541 1542 skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_head); 1543 if (!skbuf_dma) 1544 return; 1545 1546 skb = netdev_alloc_skb(ndev, lp->max_frm_size); 1547 if (!skb) 1548 return; 1549 1550 sg_init_table(skbuf_dma->sgl, 1); 1551 addr = dma_map_single(lp->dev, skb->data, lp->max_frm_size, DMA_FROM_DEVICE); 1552 if (unlikely(dma_mapping_error(lp->dev, addr))) { 1553 if (net_ratelimit()) 1554 netdev_err(ndev, "DMA mapping error\n"); 1555 goto rx_submit_err_free_skb; 1556 } 1557 sg_dma_address(skbuf_dma->sgl) = addr; 1558 sg_dma_len(skbuf_dma->sgl) = lp->max_frm_size; 1559 dma_rx_desc = dmaengine_prep_slave_sg(lp->rx_chan, skbuf_dma->sgl, 1560 1, DMA_DEV_TO_MEM, 1561 DMA_PREP_INTERRUPT); 1562 if (!dma_rx_desc) 1563 goto rx_submit_err_unmap_skb; 1564 1565 skbuf_dma->skb = skb; 1566 skbuf_dma->dma_address = sg_dma_address(skbuf_dma->sgl); 1567 skbuf_dma->desc = dma_rx_desc; 1568 dma_rx_desc->callback_param = lp; 1569 dma_rx_desc->callback_result = axienet_dma_rx_cb; 1570 lp->rx_ring_head++; 1571 dmaengine_submit(dma_rx_desc); 1572 1573 return; 1574 1575 rx_submit_err_unmap_skb: 1576 dma_unmap_single(lp->dev, addr, lp->max_frm_size, DMA_FROM_DEVICE); 1577 rx_submit_err_free_skb: 1578 dev_kfree_skb(skb); 1579 } 1580 1581 /** 1582 * axienet_init_dmaengine - init the dmaengine code. 1583 * @ndev: Pointer to net_device structure 1584 * 1585 * Return: 0, on success. 1586 * non-zero error value on failure 1587 * 1588 * This is the dmaengine initialization code. 1589 */ 1590 static int axienet_init_dmaengine(struct net_device *ndev) 1591 { 1592 struct axienet_local *lp = netdev_priv(ndev); 1593 struct skbuf_dma_descriptor *skbuf_dma; 1594 int i, ret; 1595 1596 lp->tx_chan = dma_request_chan(lp->dev, "tx_chan0"); 1597 if (IS_ERR(lp->tx_chan)) { 1598 dev_err(lp->dev, "No Ethernet DMA (TX) channel found\n"); 1599 return PTR_ERR(lp->tx_chan); 1600 } 1601 1602 lp->rx_chan = dma_request_chan(lp->dev, "rx_chan0"); 1603 if (IS_ERR(lp->rx_chan)) { 1604 ret = PTR_ERR(lp->rx_chan); 1605 dev_err(lp->dev, "No Ethernet DMA (RX) channel found\n"); 1606 goto err_dma_release_tx; 1607 } 1608 1609 lp->tx_ring_tail = 0; 1610 lp->tx_ring_head = 0; 1611 lp->rx_ring_tail = 0; 1612 lp->rx_ring_head = 0; 1613 lp->stopping = false; 1614 lp->tx_skb_ring = kzalloc_objs(*lp->tx_skb_ring, TX_BD_NUM_MAX); 1615 if (!lp->tx_skb_ring) { 1616 ret = -ENOMEM; 1617 goto err_dma_release_rx; 1618 } 1619 for (i = 0; i < TX_BD_NUM_MAX; i++) { 1620 skbuf_dma = kzalloc_obj(*skbuf_dma); 1621 if (!skbuf_dma) { 1622 ret = -ENOMEM; 1623 goto err_free_tx_skb_ring; 1624 } 1625 lp->tx_skb_ring[i] = skbuf_dma; 1626 } 1627 1628 lp->rx_skb_ring = kzalloc_objs(*lp->rx_skb_ring, RX_BUF_NUM_DEFAULT); 1629 if (!lp->rx_skb_ring) { 1630 ret = -ENOMEM; 1631 goto err_free_tx_skb_ring; 1632 } 1633 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) { 1634 skbuf_dma = kzalloc_obj(*skbuf_dma); 1635 if (!skbuf_dma) { 1636 ret = -ENOMEM; 1637 goto err_free_rx_skb_ring; 1638 } 1639 lp->rx_skb_ring[i] = skbuf_dma; 1640 } 1641 /* TODO: Instead of BD_NUM_DEFAULT use runtime support */ 1642 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) 1643 axienet_rx_submit_desc(ndev); 1644 dma_async_issue_pending(lp->rx_chan); 1645 1646 return 0; 1647 1648 err_free_rx_skb_ring: 1649 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) 1650 kfree(lp->rx_skb_ring[i]); 1651 kfree(lp->rx_skb_ring); 1652 err_free_tx_skb_ring: 1653 for (i = 0; i < TX_BD_NUM_MAX; i++) 1654 kfree(lp->tx_skb_ring[i]); 1655 kfree(lp->tx_skb_ring); 1656 err_dma_release_rx: 1657 dma_release_channel(lp->rx_chan); 1658 err_dma_release_tx: 1659 dma_release_channel(lp->tx_chan); 1660 return ret; 1661 } 1662 1663 /** 1664 * axienet_init_legacy_dma - init the dma legacy code. 1665 * @ndev: Pointer to net_device structure 1666 * 1667 * Return: 0, on success. 1668 * non-zero error value on failure 1669 * 1670 * This is the dma initialization code. It also allocates interrupt 1671 * service routines, enables the interrupt lines and ISR handling. 1672 * 1673 */ 1674 static int axienet_init_legacy_dma(struct net_device *ndev) 1675 { 1676 int ret; 1677 struct axienet_local *lp = netdev_priv(ndev); 1678 1679 /* Enable worker thread for Axi DMA error handling */ 1680 lp->stopping = false; 1681 INIT_WORK(&lp->dma_err_task, axienet_dma_err_handler); 1682 1683 napi_enable(&lp->napi_rx); 1684 napi_enable(&lp->napi_tx); 1685 1686 /* Enable interrupts for Axi DMA Tx */ 1687 ret = request_irq(lp->tx_irq, axienet_tx_irq, IRQF_SHARED, 1688 ndev->name, ndev); 1689 if (ret) 1690 goto err_tx_irq; 1691 /* Enable interrupts for Axi DMA Rx */ 1692 ret = request_irq(lp->rx_irq, axienet_rx_irq, IRQF_SHARED, 1693 ndev->name, ndev); 1694 if (ret) 1695 goto err_rx_irq; 1696 /* Enable interrupts for Axi Ethernet core (if defined) */ 1697 if (lp->eth_irq > 0) { 1698 ret = request_irq(lp->eth_irq, axienet_eth_irq, IRQF_SHARED, 1699 ndev->name, ndev); 1700 if (ret) 1701 goto err_eth_irq; 1702 } 1703 1704 return 0; 1705 1706 err_eth_irq: 1707 free_irq(lp->rx_irq, ndev); 1708 err_rx_irq: 1709 free_irq(lp->tx_irq, ndev); 1710 err_tx_irq: 1711 napi_disable(&lp->napi_tx); 1712 napi_disable(&lp->napi_rx); 1713 cancel_work_sync(&lp->dma_err_task); 1714 dev_err(lp->dev, "request_irq() failed\n"); 1715 return ret; 1716 } 1717 1718 /** 1719 * axienet_open - Driver open routine. 1720 * @ndev: Pointer to net_device structure 1721 * 1722 * Return: 0, on success. 1723 * non-zero error value on failure 1724 * 1725 * This is the driver open routine. It calls phylink_start to start the 1726 * PHY device. 1727 * It also allocates interrupt service routines, enables the interrupt lines 1728 * and ISR handling. Axi Ethernet core is reset through Axi DMA core. Buffer 1729 * descriptors are initialized. 1730 */ 1731 static int axienet_open(struct net_device *ndev) 1732 { 1733 int ret; 1734 struct axienet_local *lp = netdev_priv(ndev); 1735 1736 /* When we do an Axi Ethernet reset, it resets the complete core 1737 * including the MDIO. MDIO must be disabled before resetting. 1738 * Hold MDIO bus lock to avoid MDIO accesses during the reset. 1739 */ 1740 axienet_lock_mii(lp); 1741 ret = axienet_device_reset(ndev); 1742 axienet_unlock_mii(lp); 1743 1744 ret = phylink_fwnode_phy_connect(lp->phylink, dev_fwnode(lp->dev), 0); 1745 if (ret) { 1746 dev_err(lp->dev, "phylink_of_phy_connect() failed: %d\n", ret); 1747 return ret; 1748 } 1749 1750 phylink_start(lp->phylink); 1751 1752 /* Start the statistics refresh work */ 1753 schedule_delayed_work(&lp->stats_work, 0); 1754 1755 if (lp->use_dmaengine) { 1756 /* Enable interrupts for Axi Ethernet core (if defined) */ 1757 if (lp->eth_irq > 0) { 1758 ret = request_irq(lp->eth_irq, axienet_eth_irq, IRQF_SHARED, 1759 ndev->name, ndev); 1760 if (ret) 1761 goto err_phy; 1762 } 1763 1764 ret = axienet_init_dmaengine(ndev); 1765 if (ret < 0) 1766 goto err_free_eth_irq; 1767 } else { 1768 ret = axienet_init_legacy_dma(ndev); 1769 if (ret) 1770 goto err_phy; 1771 } 1772 1773 return 0; 1774 1775 err_free_eth_irq: 1776 if (lp->eth_irq > 0) 1777 free_irq(lp->eth_irq, ndev); 1778 err_phy: 1779 cancel_work_sync(&lp->rx_dim.work); 1780 cancel_delayed_work_sync(&lp->stats_work); 1781 phylink_stop(lp->phylink); 1782 phylink_disconnect_phy(lp->phylink); 1783 return ret; 1784 } 1785 1786 /** 1787 * axienet_stop - Driver stop routine. 1788 * @ndev: Pointer to net_device structure 1789 * 1790 * Return: 0, on success. 1791 * 1792 * This is the driver stop routine. It calls phylink_disconnect to stop the PHY 1793 * device. It also removes the interrupt handlers and disables the interrupts. 1794 * The Axi DMA Tx/Rx BDs are released. 1795 */ 1796 static int axienet_stop(struct net_device *ndev) 1797 { 1798 struct axienet_local *lp = netdev_priv(ndev); 1799 int i; 1800 1801 if (!lp->use_dmaengine) { 1802 WRITE_ONCE(lp->stopping, true); 1803 flush_work(&lp->dma_err_task); 1804 1805 napi_disable(&lp->napi_tx); 1806 napi_disable(&lp->napi_rx); 1807 } 1808 1809 cancel_work_sync(&lp->rx_dim.work); 1810 cancel_delayed_work_sync(&lp->stats_work); 1811 1812 phylink_stop(lp->phylink); 1813 phylink_disconnect_phy(lp->phylink); 1814 1815 axienet_setoptions(ndev, lp->options & 1816 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 1817 1818 if (!lp->use_dmaengine) { 1819 axienet_dma_stop(lp); 1820 cancel_work_sync(&lp->dma_err_task); 1821 free_irq(lp->tx_irq, ndev); 1822 free_irq(lp->rx_irq, ndev); 1823 axienet_dma_bd_release(ndev); 1824 } else { 1825 struct skbuf_dma_descriptor *skbuf_dma; 1826 1827 spin_lock_bh(&lp->rx_submit_lock); 1828 lp->stopping = true; 1829 spin_unlock_bh(&lp->rx_submit_lock); 1830 1831 dmaengine_terminate_sync(lp->tx_chan); 1832 dmaengine_terminate_sync(lp->rx_chan); 1833 1834 dma_release_channel(lp->rx_chan); 1835 dma_release_channel(lp->tx_chan); 1836 1837 /* Unmap and free any buffer the terminate did not reclaim, so it 1838 * is not leaked; a non-NULL skb marks such a slot. 1839 */ 1840 for (i = 0; i < TX_BD_NUM_MAX; i++) { 1841 skbuf_dma = lp->tx_skb_ring[i]; 1842 if (skbuf_dma && skbuf_dma->skb) { 1843 dma_unmap_sg(lp->dev, skbuf_dma->sgl, 1844 skbuf_dma->sg_len, DMA_TO_DEVICE); 1845 dev_kfree_skb_any(skbuf_dma->skb); 1846 } 1847 kfree(skbuf_dma); 1848 } 1849 kfree(lp->tx_skb_ring); 1850 1851 for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) { 1852 skbuf_dma = lp->rx_skb_ring[i]; 1853 if (skbuf_dma && skbuf_dma->skb) { 1854 dma_unmap_single(lp->dev, skbuf_dma->dma_address, 1855 lp->max_frm_size, DMA_FROM_DEVICE); 1856 dev_kfree_skb_any(skbuf_dma->skb); 1857 } 1858 kfree(skbuf_dma); 1859 } 1860 kfree(lp->rx_skb_ring); 1861 } 1862 1863 netdev_reset_queue(ndev); 1864 axienet_iow(lp, XAE_IE_OFFSET, 0); 1865 1866 if (lp->eth_irq > 0) 1867 free_irq(lp->eth_irq, ndev); 1868 return 0; 1869 } 1870 1871 /** 1872 * axienet_change_mtu - Driver change mtu routine. 1873 * @ndev: Pointer to net_device structure 1874 * @new_mtu: New mtu value to be applied 1875 * 1876 * Return: Always returns 0 (success). 1877 * 1878 * This is the change mtu driver routine. It checks if the Axi Ethernet 1879 * hardware supports jumbo frames before changing the mtu. This can be 1880 * called only when the device is not up. 1881 */ 1882 static int axienet_change_mtu(struct net_device *ndev, int new_mtu) 1883 { 1884 struct axienet_local *lp = netdev_priv(ndev); 1885 1886 if (netif_running(ndev)) 1887 return -EBUSY; 1888 1889 if ((new_mtu + VLAN_ETH_HLEN + 1890 XAE_TRL_SIZE) > lp->rxmem) 1891 return -EINVAL; 1892 1893 WRITE_ONCE(ndev->mtu, new_mtu); 1894 1895 return 0; 1896 } 1897 1898 #ifdef CONFIG_NET_POLL_CONTROLLER 1899 /** 1900 * axienet_poll_controller - Axi Ethernet poll mechanism. 1901 * @ndev: Pointer to net_device structure 1902 * 1903 * This implements Rx/Tx ISR poll mechanisms. The interrupts are disabled prior 1904 * to polling the ISRs and are enabled back after the polling is done. 1905 */ 1906 static void axienet_poll_controller(struct net_device *ndev) 1907 { 1908 struct axienet_local *lp = netdev_priv(ndev); 1909 1910 disable_irq(lp->tx_irq); 1911 disable_irq(lp->rx_irq); 1912 axienet_rx_irq(lp->tx_irq, ndev); 1913 axienet_tx_irq(lp->rx_irq, ndev); 1914 enable_irq(lp->tx_irq); 1915 enable_irq(lp->rx_irq); 1916 } 1917 #endif 1918 1919 static int axienet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 1920 { 1921 struct axienet_local *lp = netdev_priv(dev); 1922 1923 if (!netif_running(dev)) 1924 return -EINVAL; 1925 1926 return phylink_mii_ioctl(lp->phylink, rq, cmd); 1927 } 1928 1929 static void 1930 axienet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1931 { 1932 struct axienet_local *lp = netdev_priv(dev); 1933 unsigned int start; 1934 1935 netdev_stats_to_stats64(stats, &dev->stats); 1936 1937 do { 1938 start = u64_stats_fetch_begin(&lp->rx_stat_sync); 1939 stats->rx_packets = u64_stats_read(&lp->rx_packets); 1940 stats->rx_bytes = u64_stats_read(&lp->rx_bytes); 1941 } while (u64_stats_fetch_retry(&lp->rx_stat_sync, start)); 1942 1943 do { 1944 start = u64_stats_fetch_begin(&lp->tx_stat_sync); 1945 stats->tx_packets = u64_stats_read(&lp->tx_packets); 1946 stats->tx_bytes = u64_stats_read(&lp->tx_bytes); 1947 } while (u64_stats_fetch_retry(&lp->tx_stat_sync, start)); 1948 1949 if (!(lp->features & XAE_FEATURE_STATS)) 1950 return; 1951 1952 do { 1953 start = read_seqcount_begin(&lp->hw_stats_seqcount); 1954 stats->rx_length_errors = 1955 axienet_stat(lp, STAT_RX_LENGTH_ERRORS); 1956 stats->rx_crc_errors = axienet_stat(lp, STAT_RX_FCS_ERRORS); 1957 stats->rx_frame_errors = 1958 axienet_stat(lp, STAT_RX_ALIGNMENT_ERRORS); 1959 stats->rx_errors = axienet_stat(lp, STAT_UNDERSIZE_FRAMES) + 1960 axienet_stat(lp, STAT_FRAGMENT_FRAMES) + 1961 stats->rx_length_errors + 1962 stats->rx_crc_errors + 1963 stats->rx_frame_errors; 1964 stats->multicast = axienet_stat(lp, STAT_RX_MULTICAST_FRAMES); 1965 1966 stats->tx_aborted_errors = 1967 axienet_stat(lp, STAT_TX_EXCESS_COLLISIONS); 1968 stats->tx_fifo_errors = 1969 axienet_stat(lp, STAT_TX_UNDERRUN_ERRORS); 1970 stats->tx_window_errors = 1971 axienet_stat(lp, STAT_TX_LATE_COLLISIONS); 1972 stats->tx_errors = axienet_stat(lp, STAT_TX_EXCESS_DEFERRAL) + 1973 stats->tx_aborted_errors + 1974 stats->tx_fifo_errors + 1975 stats->tx_window_errors; 1976 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 1977 } 1978 1979 static const struct net_device_ops axienet_netdev_ops = { 1980 .ndo_open = axienet_open, 1981 .ndo_stop = axienet_stop, 1982 .ndo_start_xmit = axienet_start_xmit, 1983 .ndo_get_stats64 = axienet_get_stats64, 1984 .ndo_change_mtu = axienet_change_mtu, 1985 .ndo_set_mac_address = netdev_set_mac_address, 1986 .ndo_validate_addr = eth_validate_addr, 1987 .ndo_eth_ioctl = axienet_ioctl, 1988 .ndo_set_rx_mode = axienet_set_multicast_list, 1989 #ifdef CONFIG_NET_POLL_CONTROLLER 1990 .ndo_poll_controller = axienet_poll_controller, 1991 #endif 1992 }; 1993 1994 static const struct net_device_ops axienet_netdev_dmaengine_ops = { 1995 .ndo_open = axienet_open, 1996 .ndo_stop = axienet_stop, 1997 .ndo_start_xmit = axienet_start_xmit_dmaengine, 1998 .ndo_get_stats64 = axienet_get_stats64, 1999 .ndo_change_mtu = axienet_change_mtu, 2000 .ndo_set_mac_address = netdev_set_mac_address, 2001 .ndo_validate_addr = eth_validate_addr, 2002 .ndo_eth_ioctl = axienet_ioctl, 2003 .ndo_set_rx_mode = axienet_set_multicast_list, 2004 }; 2005 2006 /** 2007 * axienet_ethtools_get_drvinfo - Get various Axi Ethernet driver information. 2008 * @ndev: Pointer to net_device structure 2009 * @ed: Pointer to ethtool_drvinfo structure 2010 * 2011 * This implements ethtool command for getting the driver information. 2012 * Issue "ethtool -i ethX" under linux prompt to execute this function. 2013 */ 2014 static void axienet_ethtools_get_drvinfo(struct net_device *ndev, 2015 struct ethtool_drvinfo *ed) 2016 { 2017 strscpy(ed->driver, DRIVER_NAME, sizeof(ed->driver)); 2018 strscpy(ed->version, DRIVER_VERSION, sizeof(ed->version)); 2019 } 2020 2021 /** 2022 * axienet_ethtools_get_regs_len - Get the total regs length present in the 2023 * AxiEthernet core. 2024 * @ndev: Pointer to net_device structure 2025 * 2026 * This implements ethtool command for getting the total register length 2027 * information. 2028 * 2029 * Return: the total regs length 2030 */ 2031 static int axienet_ethtools_get_regs_len(struct net_device *ndev) 2032 { 2033 return sizeof(u32) * AXIENET_REGS_N; 2034 } 2035 2036 /** 2037 * axienet_ethtools_get_regs - Dump the contents of all registers present 2038 * in AxiEthernet core. 2039 * @ndev: Pointer to net_device structure 2040 * @regs: Pointer to ethtool_regs structure 2041 * @ret: Void pointer used to return the contents of the registers. 2042 * 2043 * This implements ethtool command for getting the Axi Ethernet register dump. 2044 * Issue "ethtool -d ethX" to execute this function. 2045 */ 2046 static void axienet_ethtools_get_regs(struct net_device *ndev, 2047 struct ethtool_regs *regs, void *ret) 2048 { 2049 u32 *data = (u32 *)ret; 2050 size_t len = sizeof(u32) * AXIENET_REGS_N; 2051 struct axienet_local *lp = netdev_priv(ndev); 2052 2053 regs->version = 0; 2054 regs->len = len; 2055 2056 memset(data, 0, len); 2057 data[0] = axienet_ior(lp, XAE_RAF_OFFSET); 2058 data[1] = axienet_ior(lp, XAE_TPF_OFFSET); 2059 data[2] = axienet_ior(lp, XAE_IFGP_OFFSET); 2060 data[3] = axienet_ior(lp, XAE_IS_OFFSET); 2061 data[4] = axienet_ior(lp, XAE_IP_OFFSET); 2062 data[5] = axienet_ior(lp, XAE_IE_OFFSET); 2063 data[6] = axienet_ior(lp, XAE_TTAG_OFFSET); 2064 data[7] = axienet_ior(lp, XAE_RTAG_OFFSET); 2065 data[8] = axienet_ior(lp, XAE_UAWL_OFFSET); 2066 data[9] = axienet_ior(lp, XAE_UAWU_OFFSET); 2067 data[10] = axienet_ior(lp, XAE_TPID0_OFFSET); 2068 data[11] = axienet_ior(lp, XAE_TPID1_OFFSET); 2069 data[12] = axienet_ior(lp, XAE_PPST_OFFSET); 2070 data[13] = axienet_ior(lp, XAE_RCW0_OFFSET); 2071 data[14] = axienet_ior(lp, XAE_RCW1_OFFSET); 2072 data[15] = axienet_ior(lp, XAE_TC_OFFSET); 2073 data[16] = axienet_ior(lp, XAE_FCC_OFFSET); 2074 data[17] = axienet_ior(lp, XAE_EMMC_OFFSET); 2075 data[18] = axienet_ior(lp, XAE_PHYC_OFFSET); 2076 data[19] = axienet_ior(lp, XAE_MDIO_MC_OFFSET); 2077 data[20] = axienet_ior(lp, XAE_MDIO_MCR_OFFSET); 2078 data[21] = axienet_ior(lp, XAE_MDIO_MWD_OFFSET); 2079 data[22] = axienet_ior(lp, XAE_MDIO_MRD_OFFSET); 2080 data[27] = axienet_ior(lp, XAE_UAW0_OFFSET); 2081 data[28] = axienet_ior(lp, XAE_UAW1_OFFSET); 2082 data[29] = axienet_ior(lp, XAE_FMI_OFFSET); 2083 data[30] = axienet_ior(lp, XAE_AF0_OFFSET); 2084 data[31] = axienet_ior(lp, XAE_AF1_OFFSET); 2085 if (!lp->use_dmaengine) { 2086 data[32] = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET); 2087 data[33] = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET); 2088 data[34] = axienet_dma_in32(lp, XAXIDMA_TX_CDESC_OFFSET); 2089 data[35] = axienet_dma_in32(lp, XAXIDMA_TX_TDESC_OFFSET); 2090 data[36] = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET); 2091 data[37] = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET); 2092 data[38] = axienet_dma_in32(lp, XAXIDMA_RX_CDESC_OFFSET); 2093 data[39] = axienet_dma_in32(lp, XAXIDMA_RX_TDESC_OFFSET); 2094 } 2095 } 2096 2097 static void 2098 axienet_ethtools_get_ringparam(struct net_device *ndev, 2099 struct ethtool_ringparam *ering, 2100 struct kernel_ethtool_ringparam *kernel_ering, 2101 struct netlink_ext_ack *extack) 2102 { 2103 struct axienet_local *lp = netdev_priv(ndev); 2104 2105 ering->rx_max_pending = RX_BD_NUM_MAX; 2106 ering->rx_mini_max_pending = 0; 2107 ering->rx_jumbo_max_pending = 0; 2108 ering->tx_max_pending = TX_BD_NUM_MAX; 2109 ering->rx_pending = lp->rx_bd_num; 2110 ering->rx_mini_pending = 0; 2111 ering->rx_jumbo_pending = 0; 2112 ering->tx_pending = lp->tx_bd_num; 2113 } 2114 2115 static int 2116 axienet_ethtools_set_ringparam(struct net_device *ndev, 2117 struct ethtool_ringparam *ering, 2118 struct kernel_ethtool_ringparam *kernel_ering, 2119 struct netlink_ext_ack *extack) 2120 { 2121 struct axienet_local *lp = netdev_priv(ndev); 2122 2123 if (ering->rx_pending > RX_BD_NUM_MAX || 2124 ering->rx_mini_pending || 2125 ering->rx_jumbo_pending || 2126 ering->tx_pending < TX_BD_NUM_MIN || 2127 ering->tx_pending > TX_BD_NUM_MAX) 2128 return -EINVAL; 2129 2130 if (netif_running(ndev)) 2131 return -EBUSY; 2132 2133 lp->rx_bd_num = ering->rx_pending; 2134 lp->tx_bd_num = ering->tx_pending; 2135 return 0; 2136 } 2137 2138 /** 2139 * axienet_ethtools_get_pauseparam - Get the pause parameter setting for 2140 * Tx and Rx paths. 2141 * @ndev: Pointer to net_device structure 2142 * @epauseparm: Pointer to ethtool_pauseparam structure. 2143 * 2144 * This implements ethtool command for getting axi ethernet pause frame 2145 * setting. Issue "ethtool -a ethX" to execute this function. 2146 */ 2147 static void 2148 axienet_ethtools_get_pauseparam(struct net_device *ndev, 2149 struct ethtool_pauseparam *epauseparm) 2150 { 2151 struct axienet_local *lp = netdev_priv(ndev); 2152 2153 phylink_ethtool_get_pauseparam(lp->phylink, epauseparm); 2154 } 2155 2156 /** 2157 * axienet_ethtools_set_pauseparam - Set device pause parameter(flow control) 2158 * settings. 2159 * @ndev: Pointer to net_device structure 2160 * @epauseparm:Pointer to ethtool_pauseparam structure 2161 * 2162 * This implements ethtool command for enabling flow control on Rx and Tx 2163 * paths. Issue "ethtool -A ethX tx on|off" under linux prompt to execute this 2164 * function. 2165 * 2166 * Return: 0 on success, -EFAULT if device is running 2167 */ 2168 static int 2169 axienet_ethtools_set_pauseparam(struct net_device *ndev, 2170 struct ethtool_pauseparam *epauseparm) 2171 { 2172 struct axienet_local *lp = netdev_priv(ndev); 2173 2174 return phylink_ethtool_set_pauseparam(lp->phylink, epauseparm); 2175 } 2176 2177 /** 2178 * axienet_update_coalesce_rx() - Set RX CR 2179 * @lp: Device private data 2180 * @cr: Value to write to the RX CR 2181 * @mask: Bits to set from @cr 2182 */ 2183 static void axienet_update_coalesce_rx(struct axienet_local *lp, u32 cr, 2184 u32 mask) 2185 { 2186 spin_lock_irq(&lp->rx_cr_lock); 2187 lp->rx_dma_cr &= ~mask; 2188 lp->rx_dma_cr |= cr; 2189 /* If DMA isn't started, then the settings will be applied the next 2190 * time dma_start() is called. 2191 */ 2192 if (lp->rx_dma_started) { 2193 u32 reg = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET); 2194 2195 /* Don't enable IRQs if they are disabled by NAPI */ 2196 if (reg & XAXIDMA_IRQ_ALL_MASK) 2197 cr = lp->rx_dma_cr; 2198 else 2199 cr = lp->rx_dma_cr & ~XAXIDMA_IRQ_ALL_MASK; 2200 axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr); 2201 } 2202 spin_unlock_irq(&lp->rx_cr_lock); 2203 } 2204 2205 /** 2206 * axienet_dim_coalesce_count_rx() - RX coalesce count for DIM 2207 * @lp: Device private data 2208 * 2209 * Return: RX coalescing frame count value for DIM. 2210 */ 2211 static u32 axienet_dim_coalesce_count_rx(struct axienet_local *lp) 2212 { 2213 return min(1 << (lp->rx_dim.profile_ix << 1), 255); 2214 } 2215 2216 /** 2217 * axienet_rx_dim_work() - Adjust RX DIM settings 2218 * @work: The work struct 2219 */ 2220 static void axienet_rx_dim_work(struct work_struct *work) 2221 { 2222 struct axienet_local *lp = 2223 container_of(work, struct axienet_local, rx_dim.work); 2224 u32 cr = axienet_calc_cr(lp, axienet_dim_coalesce_count_rx(lp), 0); 2225 u32 mask = XAXIDMA_COALESCE_MASK | XAXIDMA_IRQ_IOC_MASK | 2226 XAXIDMA_IRQ_ERROR_MASK; 2227 2228 axienet_update_coalesce_rx(lp, cr, mask); 2229 lp->rx_dim.state = DIM_START_MEASURE; 2230 } 2231 2232 /** 2233 * axienet_update_coalesce_tx() - Set TX CR 2234 * @lp: Device private data 2235 * @cr: Value to write to the TX CR 2236 * @mask: Bits to set from @cr 2237 */ 2238 static void axienet_update_coalesce_tx(struct axienet_local *lp, u32 cr, 2239 u32 mask) 2240 { 2241 spin_lock_irq(&lp->tx_cr_lock); 2242 lp->tx_dma_cr &= ~mask; 2243 lp->tx_dma_cr |= cr; 2244 /* If DMA isn't started, then the settings will be applied the next 2245 * time dma_start() is called. 2246 */ 2247 if (lp->tx_dma_started) { 2248 u32 reg = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET); 2249 2250 /* Don't enable IRQs if they are disabled by NAPI */ 2251 if (reg & XAXIDMA_IRQ_ALL_MASK) 2252 cr = lp->tx_dma_cr; 2253 else 2254 cr = lp->tx_dma_cr & ~XAXIDMA_IRQ_ALL_MASK; 2255 axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr); 2256 } 2257 spin_unlock_irq(&lp->tx_cr_lock); 2258 } 2259 2260 /** 2261 * axienet_ethtools_get_coalesce - Get DMA interrupt coalescing count. 2262 * @ndev: Pointer to net_device structure 2263 * @ecoalesce: Pointer to ethtool_coalesce structure 2264 * @kernel_coal: ethtool CQE mode setting structure 2265 * @extack: extack for reporting error messages 2266 * 2267 * This implements ethtool command for getting the DMA interrupt coalescing 2268 * count on Tx and Rx paths. Issue "ethtool -c ethX" under linux prompt to 2269 * execute this function. 2270 * 2271 * Return: 0 always 2272 */ 2273 static int 2274 axienet_ethtools_get_coalesce(struct net_device *ndev, 2275 struct ethtool_coalesce *ecoalesce, 2276 struct kernel_ethtool_coalesce *kernel_coal, 2277 struct netlink_ext_ack *extack) 2278 { 2279 struct axienet_local *lp = netdev_priv(ndev); 2280 u32 cr; 2281 2282 ecoalesce->use_adaptive_rx_coalesce = lp->rx_dim_enabled; 2283 2284 spin_lock_irq(&lp->rx_cr_lock); 2285 cr = lp->rx_dma_cr; 2286 spin_unlock_irq(&lp->rx_cr_lock); 2287 axienet_coalesce_params(lp, cr, 2288 &ecoalesce->rx_max_coalesced_frames, 2289 &ecoalesce->rx_coalesce_usecs); 2290 2291 spin_lock_irq(&lp->tx_cr_lock); 2292 cr = lp->tx_dma_cr; 2293 spin_unlock_irq(&lp->tx_cr_lock); 2294 axienet_coalesce_params(lp, cr, 2295 &ecoalesce->tx_max_coalesced_frames, 2296 &ecoalesce->tx_coalesce_usecs); 2297 return 0; 2298 } 2299 2300 /** 2301 * axienet_ethtools_set_coalesce - Set DMA interrupt coalescing count. 2302 * @ndev: Pointer to net_device structure 2303 * @ecoalesce: Pointer to ethtool_coalesce structure 2304 * @kernel_coal: ethtool CQE mode setting structure 2305 * @extack: extack for reporting error messages 2306 * 2307 * This implements ethtool command for setting the DMA interrupt coalescing 2308 * count on Tx and Rx paths. Issue "ethtool -C ethX rx-frames 5" under linux 2309 * prompt to execute this function. 2310 * 2311 * Return: 0, on success, Non-zero error value on failure. 2312 */ 2313 static int 2314 axienet_ethtools_set_coalesce(struct net_device *ndev, 2315 struct ethtool_coalesce *ecoalesce, 2316 struct kernel_ethtool_coalesce *kernel_coal, 2317 struct netlink_ext_ack *extack) 2318 { 2319 struct axienet_local *lp = netdev_priv(ndev); 2320 bool new_dim = ecoalesce->use_adaptive_rx_coalesce; 2321 bool old_dim = lp->rx_dim_enabled; 2322 u32 cr, mask = ~XAXIDMA_CR_RUNSTOP_MASK; 2323 2324 if (ecoalesce->rx_max_coalesced_frames > 255 || 2325 ecoalesce->tx_max_coalesced_frames > 255) { 2326 NL_SET_ERR_MSG(extack, "frames must be less than 256"); 2327 return -EINVAL; 2328 } 2329 2330 if (!ecoalesce->rx_max_coalesced_frames || 2331 !ecoalesce->tx_max_coalesced_frames) { 2332 NL_SET_ERR_MSG(extack, "frames must be non-zero"); 2333 return -EINVAL; 2334 } 2335 2336 if (((ecoalesce->rx_max_coalesced_frames > 1 || new_dim) && 2337 !ecoalesce->rx_coalesce_usecs) || 2338 (ecoalesce->tx_max_coalesced_frames > 1 && 2339 !ecoalesce->tx_coalesce_usecs)) { 2340 NL_SET_ERR_MSG(extack, 2341 "usecs must be non-zero when frames is greater than one"); 2342 return -EINVAL; 2343 } 2344 2345 if (new_dim && !old_dim) { 2346 cr = axienet_calc_cr(lp, axienet_dim_coalesce_count_rx(lp), 2347 ecoalesce->rx_coalesce_usecs); 2348 } else if (!new_dim) { 2349 if (old_dim) { 2350 WRITE_ONCE(lp->rx_dim_enabled, false); 2351 napi_synchronize(&lp->napi_rx); 2352 flush_work(&lp->rx_dim.work); 2353 } 2354 2355 cr = axienet_calc_cr(lp, ecoalesce->rx_max_coalesced_frames, 2356 ecoalesce->rx_coalesce_usecs); 2357 } else { 2358 /* Dummy value for count just to calculate timer */ 2359 cr = axienet_calc_cr(lp, 2, ecoalesce->rx_coalesce_usecs); 2360 mask = XAXIDMA_DELAY_MASK | XAXIDMA_IRQ_DELAY_MASK; 2361 } 2362 2363 axienet_update_coalesce_rx(lp, cr, mask); 2364 if (new_dim && !old_dim) 2365 WRITE_ONCE(lp->rx_dim_enabled, true); 2366 2367 cr = axienet_calc_cr(lp, ecoalesce->tx_max_coalesced_frames, 2368 ecoalesce->tx_coalesce_usecs); 2369 axienet_update_coalesce_tx(lp, cr, ~XAXIDMA_CR_RUNSTOP_MASK); 2370 return 0; 2371 } 2372 2373 static int 2374 axienet_ethtools_get_link_ksettings(struct net_device *ndev, 2375 struct ethtool_link_ksettings *cmd) 2376 { 2377 struct axienet_local *lp = netdev_priv(ndev); 2378 2379 return phylink_ethtool_ksettings_get(lp->phylink, cmd); 2380 } 2381 2382 static int 2383 axienet_ethtools_set_link_ksettings(struct net_device *ndev, 2384 const struct ethtool_link_ksettings *cmd) 2385 { 2386 struct axienet_local *lp = netdev_priv(ndev); 2387 2388 return phylink_ethtool_ksettings_set(lp->phylink, cmd); 2389 } 2390 2391 static int axienet_ethtools_nway_reset(struct net_device *dev) 2392 { 2393 struct axienet_local *lp = netdev_priv(dev); 2394 2395 return phylink_ethtool_nway_reset(lp->phylink); 2396 } 2397 2398 static void axienet_ethtools_get_ethtool_stats(struct net_device *dev, 2399 struct ethtool_stats *stats, 2400 u64 *data) 2401 { 2402 struct axienet_local *lp = netdev_priv(dev); 2403 unsigned int start; 2404 2405 do { 2406 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2407 data[0] = axienet_stat(lp, STAT_RX_BYTES); 2408 data[1] = axienet_stat(lp, STAT_TX_BYTES); 2409 data[2] = axienet_stat(lp, STAT_RX_VLAN_FRAMES); 2410 data[3] = axienet_stat(lp, STAT_TX_VLAN_FRAMES); 2411 data[6] = axienet_stat(lp, STAT_TX_PFC_FRAMES); 2412 data[7] = axienet_stat(lp, STAT_RX_PFC_FRAMES); 2413 data[8] = axienet_stat(lp, STAT_USER_DEFINED0); 2414 data[9] = axienet_stat(lp, STAT_USER_DEFINED1); 2415 data[10] = axienet_stat(lp, STAT_USER_DEFINED2); 2416 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2417 } 2418 2419 static const char axienet_ethtool_stats_strings[][ETH_GSTRING_LEN] = { 2420 "Received bytes", 2421 "Transmitted bytes", 2422 "RX Good VLAN Tagged Frames", 2423 "TX Good VLAN Tagged Frames", 2424 "TX Good PFC Frames", 2425 "RX Good PFC Frames", 2426 "User Defined Counter 0", 2427 "User Defined Counter 1", 2428 "User Defined Counter 2", 2429 }; 2430 2431 static void axienet_ethtools_get_strings(struct net_device *dev, u32 stringset, u8 *data) 2432 { 2433 switch (stringset) { 2434 case ETH_SS_STATS: 2435 memcpy(data, axienet_ethtool_stats_strings, 2436 sizeof(axienet_ethtool_stats_strings)); 2437 break; 2438 } 2439 } 2440 2441 static int axienet_ethtools_get_sset_count(struct net_device *dev, int sset) 2442 { 2443 struct axienet_local *lp = netdev_priv(dev); 2444 2445 switch (sset) { 2446 case ETH_SS_STATS: 2447 if (lp->features & XAE_FEATURE_STATS) 2448 return ARRAY_SIZE(axienet_ethtool_stats_strings); 2449 fallthrough; 2450 default: 2451 return -EOPNOTSUPP; 2452 } 2453 } 2454 2455 static void 2456 axienet_ethtools_get_pause_stats(struct net_device *dev, 2457 struct ethtool_pause_stats *pause_stats) 2458 { 2459 struct axienet_local *lp = netdev_priv(dev); 2460 unsigned int start; 2461 2462 if (!(lp->features & XAE_FEATURE_STATS)) 2463 return; 2464 2465 do { 2466 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2467 pause_stats->tx_pause_frames = 2468 axienet_stat(lp, STAT_TX_PAUSE_FRAMES); 2469 pause_stats->rx_pause_frames = 2470 axienet_stat(lp, STAT_RX_PAUSE_FRAMES); 2471 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2472 } 2473 2474 static void 2475 axienet_ethtool_get_eth_mac_stats(struct net_device *dev, 2476 struct ethtool_eth_mac_stats *mac_stats) 2477 { 2478 struct axienet_local *lp = netdev_priv(dev); 2479 unsigned int start; 2480 2481 if (!(lp->features & XAE_FEATURE_STATS)) 2482 return; 2483 2484 do { 2485 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2486 mac_stats->FramesTransmittedOK = 2487 axienet_stat(lp, STAT_TX_GOOD_FRAMES); 2488 mac_stats->SingleCollisionFrames = 2489 axienet_stat(lp, STAT_TX_SINGLE_COLLISION_FRAMES); 2490 mac_stats->MultipleCollisionFrames = 2491 axienet_stat(lp, STAT_TX_MULTIPLE_COLLISION_FRAMES); 2492 mac_stats->FramesReceivedOK = 2493 axienet_stat(lp, STAT_RX_GOOD_FRAMES); 2494 mac_stats->FrameCheckSequenceErrors = 2495 axienet_stat(lp, STAT_RX_FCS_ERRORS); 2496 mac_stats->AlignmentErrors = 2497 axienet_stat(lp, STAT_RX_ALIGNMENT_ERRORS); 2498 mac_stats->FramesWithDeferredXmissions = 2499 axienet_stat(lp, STAT_TX_DEFERRED_FRAMES); 2500 mac_stats->LateCollisions = 2501 axienet_stat(lp, STAT_TX_LATE_COLLISIONS); 2502 mac_stats->FramesAbortedDueToXSColls = 2503 axienet_stat(lp, STAT_TX_EXCESS_COLLISIONS); 2504 mac_stats->MulticastFramesXmittedOK = 2505 axienet_stat(lp, STAT_TX_MULTICAST_FRAMES); 2506 mac_stats->BroadcastFramesXmittedOK = 2507 axienet_stat(lp, STAT_TX_BROADCAST_FRAMES); 2508 mac_stats->FramesWithExcessiveDeferral = 2509 axienet_stat(lp, STAT_TX_EXCESS_DEFERRAL); 2510 mac_stats->MulticastFramesReceivedOK = 2511 axienet_stat(lp, STAT_RX_MULTICAST_FRAMES); 2512 mac_stats->BroadcastFramesReceivedOK = 2513 axienet_stat(lp, STAT_RX_BROADCAST_FRAMES); 2514 mac_stats->InRangeLengthErrors = 2515 axienet_stat(lp, STAT_RX_LENGTH_ERRORS); 2516 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2517 } 2518 2519 static void 2520 axienet_ethtool_get_eth_ctrl_stats(struct net_device *dev, 2521 struct ethtool_eth_ctrl_stats *ctrl_stats) 2522 { 2523 struct axienet_local *lp = netdev_priv(dev); 2524 unsigned int start; 2525 2526 if (!(lp->features & XAE_FEATURE_STATS)) 2527 return; 2528 2529 do { 2530 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2531 ctrl_stats->MACControlFramesTransmitted = 2532 axienet_stat(lp, STAT_TX_CONTROL_FRAMES); 2533 ctrl_stats->MACControlFramesReceived = 2534 axienet_stat(lp, STAT_RX_CONTROL_FRAMES); 2535 ctrl_stats->UnsupportedOpcodesReceived = 2536 axienet_stat(lp, STAT_RX_CONTROL_OPCODE_ERRORS); 2537 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2538 } 2539 2540 static const struct ethtool_rmon_hist_range axienet_rmon_ranges[] = { 2541 { 64, 64 }, 2542 { 65, 127 }, 2543 { 128, 255 }, 2544 { 256, 511 }, 2545 { 512, 1023 }, 2546 { 1024, 1518 }, 2547 { 1519, 16384 }, 2548 { }, 2549 }; 2550 2551 static void 2552 axienet_ethtool_get_rmon_stats(struct net_device *dev, 2553 struct ethtool_rmon_stats *rmon_stats, 2554 const struct ethtool_rmon_hist_range **ranges) 2555 { 2556 struct axienet_local *lp = netdev_priv(dev); 2557 unsigned int start; 2558 2559 if (!(lp->features & XAE_FEATURE_STATS)) 2560 return; 2561 2562 do { 2563 start = read_seqcount_begin(&lp->hw_stats_seqcount); 2564 rmon_stats->undersize_pkts = 2565 axienet_stat(lp, STAT_UNDERSIZE_FRAMES); 2566 rmon_stats->oversize_pkts = 2567 axienet_stat(lp, STAT_RX_OVERSIZE_FRAMES); 2568 rmon_stats->fragments = 2569 axienet_stat(lp, STAT_FRAGMENT_FRAMES); 2570 2571 rmon_stats->hist[0] = 2572 axienet_stat(lp, STAT_RX_64_BYTE_FRAMES); 2573 rmon_stats->hist[1] = 2574 axienet_stat(lp, STAT_RX_65_127_BYTE_FRAMES); 2575 rmon_stats->hist[2] = 2576 axienet_stat(lp, STAT_RX_128_255_BYTE_FRAMES); 2577 rmon_stats->hist[3] = 2578 axienet_stat(lp, STAT_RX_256_511_BYTE_FRAMES); 2579 rmon_stats->hist[4] = 2580 axienet_stat(lp, STAT_RX_512_1023_BYTE_FRAMES); 2581 rmon_stats->hist[5] = 2582 axienet_stat(lp, STAT_RX_1024_MAX_BYTE_FRAMES); 2583 rmon_stats->hist[6] = 2584 rmon_stats->oversize_pkts; 2585 2586 rmon_stats->hist_tx[0] = 2587 axienet_stat(lp, STAT_TX_64_BYTE_FRAMES); 2588 rmon_stats->hist_tx[1] = 2589 axienet_stat(lp, STAT_TX_65_127_BYTE_FRAMES); 2590 rmon_stats->hist_tx[2] = 2591 axienet_stat(lp, STAT_TX_128_255_BYTE_FRAMES); 2592 rmon_stats->hist_tx[3] = 2593 axienet_stat(lp, STAT_TX_256_511_BYTE_FRAMES); 2594 rmon_stats->hist_tx[4] = 2595 axienet_stat(lp, STAT_TX_512_1023_BYTE_FRAMES); 2596 rmon_stats->hist_tx[5] = 2597 axienet_stat(lp, STAT_TX_1024_MAX_BYTE_FRAMES); 2598 rmon_stats->hist_tx[6] = 2599 axienet_stat(lp, STAT_TX_OVERSIZE_FRAMES); 2600 } while (read_seqcount_retry(&lp->hw_stats_seqcount, start)); 2601 2602 *ranges = axienet_rmon_ranges; 2603 } 2604 2605 static const struct ethtool_ops axienet_ethtool_ops = { 2606 .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | 2607 ETHTOOL_COALESCE_USECS | 2608 ETHTOOL_COALESCE_USE_ADAPTIVE_RX, 2609 .get_drvinfo = axienet_ethtools_get_drvinfo, 2610 .get_regs_len = axienet_ethtools_get_regs_len, 2611 .get_regs = axienet_ethtools_get_regs, 2612 .get_link = ethtool_op_get_link, 2613 .get_ringparam = axienet_ethtools_get_ringparam, 2614 .set_ringparam = axienet_ethtools_set_ringparam, 2615 .get_pauseparam = axienet_ethtools_get_pauseparam, 2616 .set_pauseparam = axienet_ethtools_set_pauseparam, 2617 .get_coalesce = axienet_ethtools_get_coalesce, 2618 .set_coalesce = axienet_ethtools_set_coalesce, 2619 .get_link_ksettings = axienet_ethtools_get_link_ksettings, 2620 .set_link_ksettings = axienet_ethtools_set_link_ksettings, 2621 .nway_reset = axienet_ethtools_nway_reset, 2622 .get_ethtool_stats = axienet_ethtools_get_ethtool_stats, 2623 .get_strings = axienet_ethtools_get_strings, 2624 .get_sset_count = axienet_ethtools_get_sset_count, 2625 .get_pause_stats = axienet_ethtools_get_pause_stats, 2626 .get_eth_mac_stats = axienet_ethtool_get_eth_mac_stats, 2627 .get_eth_ctrl_stats = axienet_ethtool_get_eth_ctrl_stats, 2628 .get_rmon_stats = axienet_ethtool_get_rmon_stats, 2629 }; 2630 2631 static const struct ethtool_ops axienet_ethtool_dmaengine_ops = { 2632 .get_drvinfo = axienet_ethtools_get_drvinfo, 2633 .get_regs_len = axienet_ethtools_get_regs_len, 2634 .get_regs = axienet_ethtools_get_regs, 2635 .get_link = ethtool_op_get_link, 2636 .get_pauseparam = axienet_ethtools_get_pauseparam, 2637 .set_pauseparam = axienet_ethtools_set_pauseparam, 2638 .get_link_ksettings = axienet_ethtools_get_link_ksettings, 2639 .set_link_ksettings = axienet_ethtools_set_link_ksettings, 2640 .nway_reset = axienet_ethtools_nway_reset, 2641 .get_ethtool_stats = axienet_ethtools_get_ethtool_stats, 2642 .get_strings = axienet_ethtools_get_strings, 2643 .get_sset_count = axienet_ethtools_get_sset_count, 2644 .get_pause_stats = axienet_ethtools_get_pause_stats, 2645 .get_eth_mac_stats = axienet_ethtool_get_eth_mac_stats, 2646 .get_eth_ctrl_stats = axienet_ethtool_get_eth_ctrl_stats, 2647 .get_rmon_stats = axienet_ethtool_get_rmon_stats, 2648 }; 2649 2650 static struct axienet_local *pcs_to_axienet_local(struct phylink_pcs *pcs) 2651 { 2652 return container_of(pcs, struct axienet_local, pcs); 2653 } 2654 2655 static void axienet_pcs_get_state(struct phylink_pcs *pcs, 2656 unsigned int neg_mode, 2657 struct phylink_link_state *state) 2658 { 2659 struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy; 2660 2661 phylink_mii_c22_pcs_get_state(pcs_phy, neg_mode, state); 2662 } 2663 2664 static void axienet_pcs_an_restart(struct phylink_pcs *pcs) 2665 { 2666 struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy; 2667 2668 phylink_mii_c22_pcs_an_restart(pcs_phy); 2669 } 2670 2671 static int axienet_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode, 2672 phy_interface_t interface, 2673 const unsigned long *advertising, 2674 bool permit_pause_to_mac) 2675 { 2676 struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy; 2677 struct net_device *ndev = pcs_to_axienet_local(pcs)->ndev; 2678 struct axienet_local *lp = netdev_priv(ndev); 2679 int ret; 2680 2681 if (lp->switch_x_sgmii) { 2682 ret = mdiodev_write(pcs_phy, XLNX_MII_STD_SELECT_REG, 2683 interface == PHY_INTERFACE_MODE_SGMII ? 2684 XLNX_MII_STD_SELECT_SGMII : 0); 2685 if (ret < 0) { 2686 netdev_warn(ndev, 2687 "Failed to switch PHY interface: %d\n", 2688 ret); 2689 return ret; 2690 } 2691 } 2692 2693 ret = phylink_mii_c22_pcs_config(pcs_phy, interface, advertising, 2694 neg_mode); 2695 if (ret < 0) 2696 netdev_warn(ndev, "Failed to configure PCS: %d\n", ret); 2697 2698 return ret; 2699 } 2700 2701 static const struct phylink_pcs_ops axienet_pcs_ops = { 2702 .pcs_get_state = axienet_pcs_get_state, 2703 .pcs_config = axienet_pcs_config, 2704 .pcs_an_restart = axienet_pcs_an_restart, 2705 }; 2706 2707 static struct phylink_pcs *axienet_mac_select_pcs(struct phylink_config *config, 2708 phy_interface_t interface) 2709 { 2710 struct net_device *ndev = to_net_dev(config->dev); 2711 struct axienet_local *lp = netdev_priv(ndev); 2712 2713 if (interface == PHY_INTERFACE_MODE_1000BASEX || 2714 interface == PHY_INTERFACE_MODE_SGMII) 2715 return &lp->pcs; 2716 2717 return NULL; 2718 } 2719 2720 static void axienet_mac_config(struct phylink_config *config, unsigned int mode, 2721 const struct phylink_link_state *state) 2722 { 2723 /* nothing meaningful to do */ 2724 } 2725 2726 static void axienet_mac_link_down(struct phylink_config *config, 2727 unsigned int mode, 2728 phy_interface_t interface) 2729 { 2730 /* nothing meaningful to do */ 2731 } 2732 2733 static void axienet_mac_link_up(struct phylink_config *config, 2734 struct phy_device *phy, 2735 unsigned int mode, phy_interface_t interface, 2736 int speed, int duplex, 2737 bool tx_pause, bool rx_pause) 2738 { 2739 struct net_device *ndev = to_net_dev(config->dev); 2740 struct axienet_local *lp = netdev_priv(ndev); 2741 u32 emmc_reg, fcc_reg; 2742 2743 emmc_reg = axienet_ior(lp, XAE_EMMC_OFFSET); 2744 emmc_reg &= ~XAE_EMMC_LINKSPEED_MASK; 2745 2746 switch (speed) { 2747 case SPEED_1000: 2748 emmc_reg |= XAE_EMMC_LINKSPD_1000; 2749 break; 2750 case SPEED_100: 2751 emmc_reg |= XAE_EMMC_LINKSPD_100; 2752 break; 2753 case SPEED_10: 2754 emmc_reg |= XAE_EMMC_LINKSPD_10; 2755 break; 2756 default: 2757 dev_err(&ndev->dev, 2758 "Speed other than 10, 100 or 1Gbps is not supported\n"); 2759 break; 2760 } 2761 2762 axienet_iow(lp, XAE_EMMC_OFFSET, emmc_reg); 2763 2764 fcc_reg = axienet_ior(lp, XAE_FCC_OFFSET); 2765 if (tx_pause) 2766 fcc_reg |= XAE_FCC_FCTX_MASK; 2767 else 2768 fcc_reg &= ~XAE_FCC_FCTX_MASK; 2769 if (rx_pause) 2770 fcc_reg |= XAE_FCC_FCRX_MASK; 2771 else 2772 fcc_reg &= ~XAE_FCC_FCRX_MASK; 2773 axienet_iow(lp, XAE_FCC_OFFSET, fcc_reg); 2774 } 2775 2776 static const struct phylink_mac_ops axienet_phylink_ops = { 2777 .mac_select_pcs = axienet_mac_select_pcs, 2778 .mac_config = axienet_mac_config, 2779 .mac_link_down = axienet_mac_link_down, 2780 .mac_link_up = axienet_mac_link_up, 2781 }; 2782 2783 /** 2784 * axienet_dma_err_handler - Work queue task for Axi DMA Error 2785 * @work: pointer to work_struct 2786 * 2787 * Resets the Axi DMA and Axi Ethernet devices, and reconfigures the 2788 * Tx/Rx BDs. 2789 */ 2790 static void axienet_dma_err_handler(struct work_struct *work) 2791 { 2792 u32 i; 2793 u32 axienet_status; 2794 struct axidma_bd *cur_p; 2795 struct axienet_local *lp = container_of(work, struct axienet_local, 2796 dma_err_task); 2797 struct net_device *ndev = lp->ndev; 2798 2799 /* Don't bother if we are going to stop anyway */ 2800 if (READ_ONCE(lp->stopping)) 2801 return; 2802 2803 napi_disable(&lp->napi_tx); 2804 napi_disable(&lp->napi_rx); 2805 2806 axienet_setoptions(ndev, lp->options & 2807 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 2808 2809 axienet_dma_stop(lp); 2810 netdev_reset_queue(ndev); 2811 2812 axienet_free_tx_bufs(lp); 2813 2814 for (i = 0; i < lp->rx_bd_num; i++) { 2815 cur_p = &lp->rx_bd_v[i]; 2816 cur_p->status = 0; 2817 cur_p->app0 = 0; 2818 cur_p->app1 = 0; 2819 cur_p->app2 = 0; 2820 cur_p->app3 = 0; 2821 cur_p->app4 = 0; 2822 } 2823 2824 lp->tx_bd_ci = 0; 2825 lp->tx_bd_tail = 0; 2826 lp->rx_bd_ci = 0; 2827 2828 axienet_dma_start(lp); 2829 2830 axienet_status = axienet_ior(lp, XAE_RCW1_OFFSET); 2831 axienet_status &= ~XAE_RCW1_RX_MASK; 2832 axienet_iow(lp, XAE_RCW1_OFFSET, axienet_status); 2833 2834 axienet_status = axienet_ior(lp, XAE_IP_OFFSET); 2835 if (axienet_status & XAE_INT_RXRJECT_MASK) 2836 axienet_iow(lp, XAE_IS_OFFSET, XAE_INT_RXRJECT_MASK); 2837 axienet_iow(lp, XAE_IE_OFFSET, lp->eth_irq > 0 ? 2838 XAE_INT_RECV_ERROR_MASK : 0); 2839 axienet_iow(lp, XAE_FCC_OFFSET, XAE_FCC_FCRX_MASK); 2840 2841 /* Sync default options with HW but leave receiver and 2842 * transmitter disabled. 2843 */ 2844 axienet_setoptions(ndev, lp->options & 2845 ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN)); 2846 axienet_set_mac_address(ndev, NULL); 2847 axienet_set_multicast_list(ndev); 2848 napi_enable(&lp->napi_rx); 2849 napi_enable(&lp->napi_tx); 2850 axienet_setoptions(ndev, lp->options); 2851 } 2852 2853 /** 2854 * axienet_probe - Axi Ethernet probe function. 2855 * @pdev: Pointer to platform device structure. 2856 * 2857 * Return: 0, on success 2858 * Non-zero error value on failure. 2859 * 2860 * This is the probe routine for Axi Ethernet driver. This is called before 2861 * any other driver routines are invoked. It allocates and sets up the Ethernet 2862 * device. Parses through device tree and populates fields of 2863 * axienet_local. It registers the Ethernet device. 2864 */ 2865 static int axienet_probe(struct platform_device *pdev) 2866 { 2867 int ret; 2868 struct device_node *np; 2869 struct axienet_local *lp; 2870 struct net_device *ndev; 2871 struct resource *ethres; 2872 u8 mac_addr[ETH_ALEN]; 2873 int addr_width = 32; 2874 u32 value; 2875 2876 ndev = devm_alloc_etherdev(&pdev->dev, sizeof(*lp)); 2877 if (!ndev) 2878 return -ENOMEM; 2879 2880 platform_set_drvdata(pdev, ndev); 2881 2882 SET_NETDEV_DEV(ndev, &pdev->dev); 2883 ndev->features = NETIF_F_SG; 2884 2885 /* MTU range: 64 - 9000 */ 2886 ndev->min_mtu = 64; 2887 ndev->max_mtu = XAE_JUMBO_MTU; 2888 2889 lp = netdev_priv(ndev); 2890 lp->ndev = ndev; 2891 lp->dev = &pdev->dev; 2892 lp->options = XAE_OPTION_DEFAULTS; 2893 lp->rx_bd_num = RX_BD_NUM_DEFAULT; 2894 lp->tx_bd_num = TX_BD_NUM_DEFAULT; 2895 2896 u64_stats_init(&lp->rx_stat_sync); 2897 u64_stats_init(&lp->tx_stat_sync); 2898 2899 mutex_init(&lp->stats_lock); 2900 seqcount_mutex_init(&lp->hw_stats_seqcount, &lp->stats_lock); 2901 INIT_DEFERRABLE_WORK(&lp->stats_work, axienet_refresh_stats); 2902 2903 lp->axi_clk = devm_clk_get_optional_enabled(&pdev->dev, 2904 "s_axi_lite_clk"); 2905 if (!lp->axi_clk) { 2906 /* For backward compatibility, if named AXI clock is not present, 2907 * treat the first clock specified as the AXI clock. 2908 */ 2909 lp->axi_clk = devm_clk_get_optional_enabled(&pdev->dev, NULL); 2910 } 2911 if (IS_ERR(lp->axi_clk)) 2912 return dev_err_probe(&pdev->dev, PTR_ERR(lp->axi_clk), 2913 "could not get AXI clock\n"); 2914 2915 lp->misc_clks[0].id = "axis_clk"; 2916 lp->misc_clks[1].id = "ref_clk"; 2917 lp->misc_clks[2].id = "mgt_clk"; 2918 2919 ret = devm_clk_bulk_get_optional_enable(&pdev->dev, XAE_NUM_MISC_CLOCKS, 2920 lp->misc_clks); 2921 if (ret) 2922 return dev_err_probe(&pdev->dev, ret, 2923 "could not get/enable misc. clocks\n"); 2924 2925 /* Map device registers */ 2926 lp->regs = devm_platform_get_and_ioremap_resource(pdev, 0, ðres); 2927 if (IS_ERR(lp->regs)) 2928 return PTR_ERR(lp->regs); 2929 lp->regs_start = ethres->start; 2930 2931 /* Setup checksum offload, but default to off if not specified */ 2932 lp->features = 0; 2933 2934 if (axienet_ior(lp, XAE_ABILITY_OFFSET) & XAE_ABILITY_STATS) 2935 lp->features |= XAE_FEATURE_STATS; 2936 2937 ret = of_property_read_u32(pdev->dev.of_node, "xlnx,txcsum", &value); 2938 if (!ret) { 2939 switch (value) { 2940 case 1: 2941 lp->features |= XAE_FEATURE_PARTIAL_TX_CSUM; 2942 /* Can checksum any contiguous range */ 2943 ndev->features |= NETIF_F_HW_CSUM; 2944 break; 2945 case 2: 2946 lp->features |= XAE_FEATURE_FULL_TX_CSUM; 2947 /* Can checksum TCP/UDP over IPv4. */ 2948 ndev->features |= NETIF_F_IP_CSUM; 2949 break; 2950 } 2951 } 2952 ret = of_property_read_u32(pdev->dev.of_node, "xlnx,rxcsum", &value); 2953 if (!ret) { 2954 switch (value) { 2955 case 1: 2956 lp->features |= XAE_FEATURE_PARTIAL_RX_CSUM; 2957 ndev->features |= NETIF_F_RXCSUM; 2958 break; 2959 case 2: 2960 lp->features |= XAE_FEATURE_FULL_RX_CSUM; 2961 ndev->features |= NETIF_F_RXCSUM; 2962 break; 2963 } 2964 } 2965 /* For supporting jumbo frames, the Axi Ethernet hardware must have 2966 * a larger Rx/Tx Memory. Typically, the size must be large so that 2967 * we can enable jumbo option and start supporting jumbo frames. 2968 * Here we check for memory allocated for Rx/Tx in the hardware from 2969 * the device-tree and accordingly set flags. 2970 */ 2971 ret = device_property_read_u32(&pdev->dev, "xlnx,rxmem", &lp->rxmem); 2972 if (ret) 2973 return dev_err_probe(&pdev->dev, ret, 2974 "failed to read xlnx,rxmem property\n"); 2975 2976 lp->switch_x_sgmii = of_property_read_bool(pdev->dev.of_node, 2977 "xlnx,switch-x-sgmii"); 2978 2979 /* Start with the proprietary, and broken phy_type */ 2980 ret = of_property_read_u32(pdev->dev.of_node, "xlnx,phy-type", &value); 2981 if (!ret) { 2982 netdev_warn(ndev, "Please upgrade your device tree binary blob to use phy-mode"); 2983 switch (value) { 2984 case XAE_PHY_TYPE_MII: 2985 lp->phy_mode = PHY_INTERFACE_MODE_MII; 2986 break; 2987 case XAE_PHY_TYPE_GMII: 2988 lp->phy_mode = PHY_INTERFACE_MODE_GMII; 2989 break; 2990 case XAE_PHY_TYPE_RGMII_2_0: 2991 lp->phy_mode = PHY_INTERFACE_MODE_RGMII_ID; 2992 break; 2993 case XAE_PHY_TYPE_SGMII: 2994 lp->phy_mode = PHY_INTERFACE_MODE_SGMII; 2995 break; 2996 case XAE_PHY_TYPE_1000BASE_X: 2997 lp->phy_mode = PHY_INTERFACE_MODE_1000BASEX; 2998 break; 2999 default: 3000 return -EINVAL; 3001 } 3002 } else { 3003 ret = device_get_phy_mode(&pdev->dev); 3004 if (ret < 0) 3005 return ret; 3006 lp->phy_mode = ret; 3007 } 3008 if (lp->switch_x_sgmii && lp->phy_mode != PHY_INTERFACE_MODE_SGMII && 3009 lp->phy_mode != PHY_INTERFACE_MODE_1000BASEX) { 3010 dev_err(&pdev->dev, "xlnx,switch-x-sgmii only supported with SGMII or 1000BaseX\n"); 3011 return -EINVAL; 3012 } 3013 3014 if (!of_property_present(pdev->dev.of_node, "dmas")) { 3015 /* Find the DMA node, map the DMA registers, and decode the DMA IRQs */ 3016 np = of_parse_phandle(pdev->dev.of_node, "axistream-connected", 0); 3017 3018 if (np) { 3019 struct resource dmares; 3020 3021 ret = of_address_to_resource(np, 0, &dmares); 3022 if (ret) { 3023 dev_err(&pdev->dev, 3024 "unable to get DMA resource\n"); 3025 of_node_put(np); 3026 return ret; 3027 } 3028 lp->dma_regs = devm_ioremap_resource(&pdev->dev, 3029 &dmares); 3030 lp->rx_irq = irq_of_parse_and_map(np, 1); 3031 lp->tx_irq = irq_of_parse_and_map(np, 0); 3032 of_node_put(np); 3033 lp->eth_irq = platform_get_irq_optional(pdev, 0); 3034 } else { 3035 /* Check for these resources directly on the Ethernet node. */ 3036 lp->dma_regs = devm_platform_get_and_ioremap_resource(pdev, 1, NULL); 3037 lp->rx_irq = platform_get_irq(pdev, 1); 3038 lp->tx_irq = platform_get_irq(pdev, 0); 3039 lp->eth_irq = platform_get_irq_optional(pdev, 2); 3040 } 3041 if (IS_ERR(lp->dma_regs)) { 3042 dev_err(&pdev->dev, "could not map DMA regs\n"); 3043 return PTR_ERR(lp->dma_regs); 3044 } 3045 if (!lp->rx_irq || !lp->tx_irq) { 3046 dev_err(&pdev->dev, "could not determine irqs\n"); 3047 return -EINVAL; 3048 } 3049 if (lp->rx_irq < 0) 3050 return lp->rx_irq; 3051 if (lp->tx_irq < 0) 3052 return lp->tx_irq; 3053 if (lp->eth_irq < 0 && lp->eth_irq != -ENXIO) 3054 return lp->eth_irq; 3055 3056 /* Reset core now that clocks are enabled, prior to accessing MDIO */ 3057 ret = __axienet_device_reset(lp); 3058 if (ret) 3059 return ret; 3060 3061 /* Autodetect the need for 64-bit DMA pointers. 3062 * When the IP is configured for a bus width bigger than 32 bits, 3063 * writing the MSB registers is mandatory, even if they are all 0. 3064 * We can detect this case by writing all 1's to one such register 3065 * and see if that sticks: when the IP is configured for 32 bits 3066 * only, those registers are RES0. 3067 * Those MSB registers were introduced in IP v7.1, which we check first. 3068 */ 3069 if ((axienet_ior(lp, XAE_ID_OFFSET) >> 24) >= 0x9) { 3070 void __iomem *desc = lp->dma_regs + XAXIDMA_TX_CDESC_OFFSET + 4; 3071 3072 iowrite32(0x0, desc); 3073 if (ioread32(desc) == 0) { /* sanity check */ 3074 iowrite32(0xffffffff, desc); 3075 if (ioread32(desc) > 0) { 3076 lp->features |= XAE_FEATURE_DMA_64BIT; 3077 addr_width = 64; 3078 dev_info(&pdev->dev, 3079 "autodetected 64-bit DMA range\n"); 3080 } 3081 iowrite32(0x0, desc); 3082 } 3083 } 3084 if (!IS_ENABLED(CONFIG_64BIT) && lp->features & XAE_FEATURE_DMA_64BIT) { 3085 dev_err(&pdev->dev, "64-bit addressable DMA is not compatible with 32-bit architecture\n"); 3086 return -EINVAL; 3087 } 3088 3089 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_width)); 3090 if (ret) { 3091 dev_err(&pdev->dev, "No suitable DMA available\n"); 3092 return ret; 3093 } 3094 netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll); 3095 netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll); 3096 } else { 3097 struct xilinx_vdma_config cfg; 3098 struct dma_chan *tx_chan; 3099 3100 lp->eth_irq = platform_get_irq_optional(pdev, 0); 3101 if (lp->eth_irq < 0 && lp->eth_irq != -ENXIO) { 3102 return lp->eth_irq; 3103 } 3104 tx_chan = dma_request_chan(lp->dev, "tx_chan0"); 3105 if (IS_ERR(tx_chan)) 3106 return dev_err_probe(lp->dev, PTR_ERR(tx_chan), 3107 "No Ethernet DMA (TX) channel found\n"); 3108 3109 cfg.reset = 1; 3110 /* As name says VDMA but it has support for DMA channel reset */ 3111 ret = xilinx_vdma_channel_set_config(tx_chan, &cfg); 3112 if (ret < 0) { 3113 dev_err(&pdev->dev, "Reset channel failed\n"); 3114 dma_release_channel(tx_chan); 3115 return ret; 3116 } 3117 3118 dma_release_channel(tx_chan); 3119 lp->use_dmaengine = 1; 3120 } 3121 3122 if (lp->use_dmaengine) { 3123 ndev->netdev_ops = &axienet_netdev_dmaengine_ops; 3124 ndev->ethtool_ops = &axienet_ethtool_dmaengine_ops; 3125 } else { 3126 ndev->netdev_ops = &axienet_netdev_ops; 3127 ndev->ethtool_ops = &axienet_ethtool_ops; 3128 } 3129 /* Check for Ethernet core IRQ (optional) */ 3130 if (lp->eth_irq < 0) 3131 dev_info(&pdev->dev, "Ethernet core IRQ not defined\n"); 3132 3133 /* Retrieve the MAC address */ 3134 ret = device_get_mac_address(&pdev->dev, mac_addr); 3135 if (!ret) { 3136 axienet_set_mac_address(ndev, mac_addr); 3137 } else { 3138 dev_warn(&pdev->dev, "could not find MAC address property: %d\n", 3139 ret); 3140 axienet_set_mac_address(ndev, NULL); 3141 } 3142 3143 spin_lock_init(&lp->rx_cr_lock); 3144 spin_lock_init(&lp->tx_cr_lock); 3145 spin_lock_init(&lp->rx_submit_lock); 3146 INIT_WORK(&lp->rx_dim.work, axienet_rx_dim_work); 3147 lp->rx_dim_enabled = true; 3148 lp->rx_dim.profile_ix = 1; 3149 lp->rx_dma_cr = axienet_calc_cr(lp, axienet_dim_coalesce_count_rx(lp), 3150 XAXIDMA_DFT_RX_USEC); 3151 lp->tx_dma_cr = axienet_calc_cr(lp, XAXIDMA_DFT_TX_THRESHOLD, 3152 XAXIDMA_DFT_TX_USEC); 3153 3154 ret = axienet_mdio_setup(lp); 3155 if (ret) 3156 dev_warn(&pdev->dev, 3157 "error registering MDIO bus: %d\n", ret); 3158 3159 if (lp->phy_mode == PHY_INTERFACE_MODE_SGMII || 3160 lp->phy_mode == PHY_INTERFACE_MODE_1000BASEX) { 3161 np = of_parse_phandle(pdev->dev.of_node, "pcs-handle", 0); 3162 if (!np) { 3163 /* Deprecated: Always use "pcs-handle" for pcs_phy. 3164 * Falling back to "phy-handle" here is only for 3165 * backward compatibility with old device trees. 3166 */ 3167 np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0); 3168 } 3169 if (!np) { 3170 dev_err(&pdev->dev, "pcs-handle (preferred) or phy-handle required for 1000BaseX/SGMII\n"); 3171 ret = -EINVAL; 3172 goto cleanup_mdio; 3173 } 3174 lp->pcs_phy = of_mdio_find_device(np); 3175 if (!lp->pcs_phy) { 3176 ret = -EPROBE_DEFER; 3177 of_node_put(np); 3178 goto cleanup_mdio; 3179 } 3180 of_node_put(np); 3181 lp->pcs.ops = &axienet_pcs_ops; 3182 lp->pcs.poll = true; 3183 } 3184 3185 lp->phylink_config.dev = &ndev->dev; 3186 lp->phylink_config.type = PHYLINK_NETDEV; 3187 lp->phylink_config.mac_managed_pm = true; 3188 lp->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | 3189 MAC_10FD | MAC_100FD | MAC_1000FD; 3190 3191 __set_bit(lp->phy_mode, lp->phylink_config.supported_interfaces); 3192 if (lp->switch_x_sgmii) { 3193 __set_bit(PHY_INTERFACE_MODE_1000BASEX, 3194 lp->phylink_config.supported_interfaces); 3195 __set_bit(PHY_INTERFACE_MODE_SGMII, 3196 lp->phylink_config.supported_interfaces); 3197 } 3198 3199 lp->phylink = phylink_create(&lp->phylink_config, pdev->dev.fwnode, 3200 lp->phy_mode, 3201 &axienet_phylink_ops); 3202 if (IS_ERR(lp->phylink)) { 3203 ret = PTR_ERR(lp->phylink); 3204 dev_err(&pdev->dev, "phylink_create error (%i)\n", ret); 3205 goto cleanup_mdio; 3206 } 3207 3208 ret = register_netdev(lp->ndev); 3209 if (ret) { 3210 dev_err(lp->dev, "register_netdev() error (%i)\n", ret); 3211 goto cleanup_phylink; 3212 } 3213 3214 return 0; 3215 3216 cleanup_phylink: 3217 phylink_destroy(lp->phylink); 3218 3219 cleanup_mdio: 3220 if (lp->pcs_phy) 3221 put_device(&lp->pcs_phy->dev); 3222 if (lp->mii_bus) 3223 axienet_mdio_teardown(lp); 3224 return ret; 3225 } 3226 3227 static void axienet_remove(struct platform_device *pdev) 3228 { 3229 struct net_device *ndev = platform_get_drvdata(pdev); 3230 struct axienet_local *lp = netdev_priv(ndev); 3231 3232 unregister_netdev(ndev); 3233 3234 if (lp->phylink) 3235 phylink_destroy(lp->phylink); 3236 3237 if (lp->pcs_phy) 3238 put_device(&lp->pcs_phy->dev); 3239 3240 axienet_mdio_teardown(lp); 3241 } 3242 3243 static void axienet_shutdown(struct platform_device *pdev) 3244 { 3245 struct net_device *ndev = platform_get_drvdata(pdev); 3246 3247 rtnl_lock(); 3248 netif_device_detach(ndev); 3249 3250 if (netif_running(ndev)) 3251 dev_close(ndev); 3252 3253 rtnl_unlock(); 3254 } 3255 3256 static int axienet_suspend(struct device *dev) 3257 { 3258 struct net_device *ndev = dev_get_drvdata(dev); 3259 3260 if (!netif_running(ndev)) 3261 return 0; 3262 3263 netif_device_detach(ndev); 3264 3265 rtnl_lock(); 3266 axienet_stop(ndev); 3267 rtnl_unlock(); 3268 3269 return 0; 3270 } 3271 3272 static int axienet_resume(struct device *dev) 3273 { 3274 struct net_device *ndev = dev_get_drvdata(dev); 3275 3276 if (!netif_running(ndev)) 3277 return 0; 3278 3279 rtnl_lock(); 3280 axienet_open(ndev); 3281 rtnl_unlock(); 3282 3283 netif_device_attach(ndev); 3284 3285 return 0; 3286 } 3287 3288 static DEFINE_SIMPLE_DEV_PM_OPS(axienet_pm_ops, 3289 axienet_suspend, axienet_resume); 3290 3291 static struct platform_driver axienet_driver = { 3292 .probe = axienet_probe, 3293 .remove = axienet_remove, 3294 .shutdown = axienet_shutdown, 3295 .driver = { 3296 .name = AXIENET_DRIVER_NAME, 3297 .pm = &axienet_pm_ops, 3298 .of_match_table = axienet_of_match, 3299 }, 3300 }; 3301 3302 module_platform_driver(axienet_driver); 3303 3304 /* The module is named xilinx_emac, the platform driver xilinx_axienet. A 3305 * device registered by name rather than from firmware advertises a 3306 * platform:xilinx_axienet modalias, which without this matches no module: 3307 * udev cannot autoload the driver and the device stays unbound. 3308 */ 3309 MODULE_ALIAS("platform:" AXIENET_DRIVER_NAME); 3310 3311 MODULE_DESCRIPTION("Xilinx Axi Ethernet driver"); 3312 MODULE_AUTHOR("Xilinx"); 3313 MODULE_LICENSE("GPL"); 3314