1 /******************************************************************************* 2 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 3 ST Ethernet IPs are built around a Synopsys IP Core. 4 5 Copyright(C) 2007-2011 STMicroelectronics Ltd 6 7 This program is free software; you can redistribute it and/or modify it 8 under the terms and conditions of the GNU General Public License, 9 version 2, as published by the Free Software Foundation. 10 11 This program is distributed in the hope it will be useful, but WITHOUT 12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 The full GNU General Public License is included in this distribution in 17 the file called "COPYING". 18 19 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 20 21 Documentation available at: 22 http://www.stlinux.com 23 Support available at: 24 https://bugzilla.stlinux.com/ 25 *******************************************************************************/ 26 27 #include <linux/clk.h> 28 #include <linux/kernel.h> 29 #include <linux/interrupt.h> 30 #include <linux/ip.h> 31 #include <linux/tcp.h> 32 #include <linux/skbuff.h> 33 #include <linux/ethtool.h> 34 #include <linux/if_ether.h> 35 #include <linux/crc32.h> 36 #include <linux/mii.h> 37 #include <linux/if.h> 38 #include <linux/if_vlan.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/slab.h> 41 #include <linux/prefetch.h> 42 #include <linux/pinctrl/consumer.h> 43 #ifdef CONFIG_DEBUG_FS 44 #include <linux/debugfs.h> 45 #include <linux/seq_file.h> 46 #endif /* CONFIG_DEBUG_FS */ 47 #include <linux/net_tstamp.h> 48 #include <net/pkt_cls.h> 49 #include "stmmac_ptp.h" 50 #include "stmmac.h" 51 #include <linux/reset.h> 52 #include <linux/of_mdio.h> 53 #include "dwmac1000.h" 54 #include "dwxgmac2.h" 55 #include "hwif.h" 56 57 #define STMMAC_ALIGN(x) __ALIGN_KERNEL(x, SMP_CACHE_BYTES) 58 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 59 60 /* Module parameters */ 61 #define TX_TIMEO 5000 62 static int watchdog = TX_TIMEO; 63 module_param(watchdog, int, 0644); 64 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 65 66 static int debug = -1; 67 module_param(debug, int, 0644); 68 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 69 70 static int phyaddr = -1; 71 module_param(phyaddr, int, 0444); 72 MODULE_PARM_DESC(phyaddr, "Physical device address"); 73 74 #define STMMAC_TX_THRESH (DMA_TX_SIZE / 4) 75 #define STMMAC_RX_THRESH (DMA_RX_SIZE / 4) 76 77 static int flow_ctrl = FLOW_OFF; 78 module_param(flow_ctrl, int, 0644); 79 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 80 81 static int pause = PAUSE_TIME; 82 module_param(pause, int, 0644); 83 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 84 85 #define TC_DEFAULT 64 86 static int tc = TC_DEFAULT; 87 module_param(tc, int, 0644); 88 MODULE_PARM_DESC(tc, "DMA threshold control value"); 89 90 #define DEFAULT_BUFSIZE 1536 91 static int buf_sz = DEFAULT_BUFSIZE; 92 module_param(buf_sz, int, 0644); 93 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 94 95 #define STMMAC_RX_COPYBREAK 256 96 97 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 98 NETIF_MSG_LINK | NETIF_MSG_IFUP | 99 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 100 101 #define STMMAC_DEFAULT_LPI_TIMER 1000 102 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 103 module_param(eee_timer, int, 0644); 104 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 105 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x)) 106 107 /* By default the driver will use the ring mode to manage tx and rx descriptors, 108 * but allow user to force to use the chain instead of the ring 109 */ 110 static unsigned int chain_mode; 111 module_param(chain_mode, int, 0444); 112 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 113 114 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 115 116 #ifdef CONFIG_DEBUG_FS 117 static int stmmac_init_fs(struct net_device *dev); 118 static void stmmac_exit_fs(struct net_device *dev); 119 #endif 120 121 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x)) 122 123 /** 124 * stmmac_verify_args - verify the driver parameters. 125 * Description: it checks the driver parameters and set a default in case of 126 * errors. 127 */ 128 static void stmmac_verify_args(void) 129 { 130 if (unlikely(watchdog < 0)) 131 watchdog = TX_TIMEO; 132 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 133 buf_sz = DEFAULT_BUFSIZE; 134 if (unlikely(flow_ctrl > 1)) 135 flow_ctrl = FLOW_AUTO; 136 else if (likely(flow_ctrl < 0)) 137 flow_ctrl = FLOW_OFF; 138 if (unlikely((pause < 0) || (pause > 0xffff))) 139 pause = PAUSE_TIME; 140 if (eee_timer < 0) 141 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 142 } 143 144 /** 145 * stmmac_disable_all_queues - Disable all queues 146 * @priv: driver private structure 147 */ 148 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 149 { 150 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 151 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 152 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 153 u32 queue; 154 155 for (queue = 0; queue < maxq; queue++) { 156 struct stmmac_channel *ch = &priv->channel[queue]; 157 158 napi_disable(&ch->napi); 159 } 160 } 161 162 /** 163 * stmmac_enable_all_queues - Enable all queues 164 * @priv: driver private structure 165 */ 166 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 167 { 168 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 169 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 170 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 171 u32 queue; 172 173 for (queue = 0; queue < maxq; queue++) { 174 struct stmmac_channel *ch = &priv->channel[queue]; 175 176 napi_enable(&ch->napi); 177 } 178 } 179 180 /** 181 * stmmac_stop_all_queues - Stop all queues 182 * @priv: driver private structure 183 */ 184 static void stmmac_stop_all_queues(struct stmmac_priv *priv) 185 { 186 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 187 u32 queue; 188 189 for (queue = 0; queue < tx_queues_cnt; queue++) 190 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 191 } 192 193 /** 194 * stmmac_start_all_queues - Start all queues 195 * @priv: driver private structure 196 */ 197 static void stmmac_start_all_queues(struct stmmac_priv *priv) 198 { 199 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 200 u32 queue; 201 202 for (queue = 0; queue < tx_queues_cnt; queue++) 203 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue)); 204 } 205 206 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 207 { 208 if (!test_bit(STMMAC_DOWN, &priv->state) && 209 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 210 queue_work(priv->wq, &priv->service_task); 211 } 212 213 static void stmmac_global_err(struct stmmac_priv *priv) 214 { 215 netif_carrier_off(priv->dev); 216 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 217 stmmac_service_event_schedule(priv); 218 } 219 220 /** 221 * stmmac_clk_csr_set - dynamically set the MDC clock 222 * @priv: driver private structure 223 * Description: this is to dynamically set the MDC clock according to the csr 224 * clock input. 225 * Note: 226 * If a specific clk_csr value is passed from the platform 227 * this means that the CSR Clock Range selection cannot be 228 * changed at run-time and it is fixed (as reported in the driver 229 * documentation). Viceversa the driver will try to set the MDC 230 * clock dynamically according to the actual clock input. 231 */ 232 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 233 { 234 u32 clk_rate; 235 236 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 237 238 /* Platform provided default clk_csr would be assumed valid 239 * for all other cases except for the below mentioned ones. 240 * For values higher than the IEEE 802.3 specified frequency 241 * we can not estimate the proper divider as it is not known 242 * the frequency of clk_csr_i. So we do not change the default 243 * divider. 244 */ 245 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 246 if (clk_rate < CSR_F_35M) 247 priv->clk_csr = STMMAC_CSR_20_35M; 248 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 249 priv->clk_csr = STMMAC_CSR_35_60M; 250 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 251 priv->clk_csr = STMMAC_CSR_60_100M; 252 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 253 priv->clk_csr = STMMAC_CSR_100_150M; 254 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 255 priv->clk_csr = STMMAC_CSR_150_250M; 256 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M)) 257 priv->clk_csr = STMMAC_CSR_250_300M; 258 } 259 260 if (priv->plat->has_sun8i) { 261 if (clk_rate > 160000000) 262 priv->clk_csr = 0x03; 263 else if (clk_rate > 80000000) 264 priv->clk_csr = 0x02; 265 else if (clk_rate > 40000000) 266 priv->clk_csr = 0x01; 267 else 268 priv->clk_csr = 0; 269 } 270 271 if (priv->plat->has_xgmac) { 272 if (clk_rate > 400000000) 273 priv->clk_csr = 0x5; 274 else if (clk_rate > 350000000) 275 priv->clk_csr = 0x4; 276 else if (clk_rate > 300000000) 277 priv->clk_csr = 0x3; 278 else if (clk_rate > 250000000) 279 priv->clk_csr = 0x2; 280 else if (clk_rate > 150000000) 281 priv->clk_csr = 0x1; 282 else 283 priv->clk_csr = 0x0; 284 } 285 } 286 287 static void print_pkt(unsigned char *buf, int len) 288 { 289 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 290 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 291 } 292 293 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 294 { 295 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 296 u32 avail; 297 298 if (tx_q->dirty_tx > tx_q->cur_tx) 299 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 300 else 301 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1; 302 303 return avail; 304 } 305 306 /** 307 * stmmac_rx_dirty - Get RX queue dirty 308 * @priv: driver private structure 309 * @queue: RX queue index 310 */ 311 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 312 { 313 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 314 u32 dirty; 315 316 if (rx_q->dirty_rx <= rx_q->cur_rx) 317 dirty = rx_q->cur_rx - rx_q->dirty_rx; 318 else 319 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx; 320 321 return dirty; 322 } 323 324 /** 325 * stmmac_hw_fix_mac_speed - callback for speed selection 326 * @priv: driver private structure 327 * Description: on some platforms (e.g. ST), some HW system configuration 328 * registers have to be set according to the link speed negotiated. 329 */ 330 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv) 331 { 332 struct net_device *ndev = priv->dev; 333 struct phy_device *phydev = ndev->phydev; 334 335 if (likely(priv->plat->fix_mac_speed)) 336 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed); 337 } 338 339 /** 340 * stmmac_enable_eee_mode - check and enter in LPI mode 341 * @priv: driver private structure 342 * Description: this function is to verify and enter in LPI mode in case of 343 * EEE. 344 */ 345 static void stmmac_enable_eee_mode(struct stmmac_priv *priv) 346 { 347 u32 tx_cnt = priv->plat->tx_queues_to_use; 348 u32 queue; 349 350 /* check if all TX queues have the work finished */ 351 for (queue = 0; queue < tx_cnt; queue++) { 352 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 353 354 if (tx_q->dirty_tx != tx_q->cur_tx) 355 return; /* still unfinished work */ 356 } 357 358 /* Check and enter in LPI mode */ 359 if (!priv->tx_path_in_lpi_mode) 360 stmmac_set_eee_mode(priv, priv->hw, 361 priv->plat->en_tx_lpi_clockgating); 362 } 363 364 /** 365 * stmmac_disable_eee_mode - disable and exit from LPI mode 366 * @priv: driver private structure 367 * Description: this function is to exit and disable EEE in case of 368 * LPI state is true. This is called by the xmit. 369 */ 370 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 371 { 372 stmmac_reset_eee_mode(priv, priv->hw); 373 del_timer_sync(&priv->eee_ctrl_timer); 374 priv->tx_path_in_lpi_mode = false; 375 } 376 377 /** 378 * stmmac_eee_ctrl_timer - EEE TX SW timer. 379 * @arg : data hook 380 * Description: 381 * if there is no data transfer and if we are not in LPI state, 382 * then MAC Transmitter can be moved to LPI state. 383 */ 384 static void stmmac_eee_ctrl_timer(struct timer_list *t) 385 { 386 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 387 388 stmmac_enable_eee_mode(priv); 389 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 390 } 391 392 /** 393 * stmmac_eee_init - init EEE 394 * @priv: driver private structure 395 * Description: 396 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 397 * can also manage EEE, this function enable the LPI state and start related 398 * timer. 399 */ 400 bool stmmac_eee_init(struct stmmac_priv *priv) 401 { 402 struct net_device *ndev = priv->dev; 403 int interface = priv->plat->interface; 404 bool ret = false; 405 406 if ((interface != PHY_INTERFACE_MODE_MII) && 407 (interface != PHY_INTERFACE_MODE_GMII) && 408 !phy_interface_mode_is_rgmii(interface)) 409 goto out; 410 411 /* Using PCS we cannot dial with the phy registers at this stage 412 * so we do not support extra feature like EEE. 413 */ 414 if ((priv->hw->pcs == STMMAC_PCS_RGMII) || 415 (priv->hw->pcs == STMMAC_PCS_TBI) || 416 (priv->hw->pcs == STMMAC_PCS_RTBI)) 417 goto out; 418 419 /* MAC core supports the EEE feature. */ 420 if (priv->dma_cap.eee) { 421 int tx_lpi_timer = priv->tx_lpi_timer; 422 423 /* Check if the PHY supports EEE */ 424 if (phy_init_eee(ndev->phydev, 1)) { 425 /* To manage at run-time if the EEE cannot be supported 426 * anymore (for example because the lp caps have been 427 * changed). 428 * In that case the driver disable own timers. 429 */ 430 mutex_lock(&priv->lock); 431 if (priv->eee_active) { 432 netdev_dbg(priv->dev, "disable EEE\n"); 433 del_timer_sync(&priv->eee_ctrl_timer); 434 stmmac_set_eee_timer(priv, priv->hw, 0, 435 tx_lpi_timer); 436 } 437 priv->eee_active = 0; 438 mutex_unlock(&priv->lock); 439 goto out; 440 } 441 /* Activate the EEE and start timers */ 442 mutex_lock(&priv->lock); 443 if (!priv->eee_active) { 444 priv->eee_active = 1; 445 timer_setup(&priv->eee_ctrl_timer, 446 stmmac_eee_ctrl_timer, 0); 447 mod_timer(&priv->eee_ctrl_timer, 448 STMMAC_LPI_T(eee_timer)); 449 450 stmmac_set_eee_timer(priv, priv->hw, 451 STMMAC_DEFAULT_LIT_LS, tx_lpi_timer); 452 } 453 /* Set HW EEE according to the speed */ 454 stmmac_set_eee_pls(priv, priv->hw, ndev->phydev->link); 455 456 ret = true; 457 mutex_unlock(&priv->lock); 458 459 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 460 } 461 out: 462 return ret; 463 } 464 465 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 466 * @priv: driver private structure 467 * @p : descriptor pointer 468 * @skb : the socket buffer 469 * Description : 470 * This function will read timestamp from the descriptor & pass it to stack. 471 * and also perform some sanity checks. 472 */ 473 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 474 struct dma_desc *p, struct sk_buff *skb) 475 { 476 struct skb_shared_hwtstamps shhwtstamp; 477 u64 ns; 478 479 if (!priv->hwts_tx_en) 480 return; 481 482 /* exit if skb doesn't support hw tstamp */ 483 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 484 return; 485 486 /* check tx tstamp status */ 487 if (stmmac_get_tx_timestamp_status(priv, p)) { 488 /* get the valid tstamp */ 489 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 490 491 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 492 shhwtstamp.hwtstamp = ns_to_ktime(ns); 493 494 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 495 /* pass tstamp to stack */ 496 skb_tstamp_tx(skb, &shhwtstamp); 497 } 498 499 return; 500 } 501 502 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 503 * @priv: driver private structure 504 * @p : descriptor pointer 505 * @np : next descriptor pointer 506 * @skb : the socket buffer 507 * Description : 508 * This function will read received packet's timestamp from the descriptor 509 * and pass it to stack. It also perform some sanity checks. 510 */ 511 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 512 struct dma_desc *np, struct sk_buff *skb) 513 { 514 struct skb_shared_hwtstamps *shhwtstamp = NULL; 515 struct dma_desc *desc = p; 516 u64 ns; 517 518 if (!priv->hwts_rx_en) 519 return; 520 /* For GMAC4, the valid timestamp is from CTX next desc. */ 521 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 522 desc = np; 523 524 /* Check if timestamp is available */ 525 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 526 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 527 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 528 shhwtstamp = skb_hwtstamps(skb); 529 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 530 shhwtstamp->hwtstamp = ns_to_ktime(ns); 531 } else { 532 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 533 } 534 } 535 536 /** 537 * stmmac_hwtstamp_ioctl - control hardware timestamping. 538 * @dev: device pointer. 539 * @ifr: An IOCTL specific structure, that can contain a pointer to 540 * a proprietary structure used to pass information to the driver. 541 * Description: 542 * This function configures the MAC to enable/disable both outgoing(TX) 543 * and incoming(RX) packets time stamping based on user input. 544 * Return Value: 545 * 0 on success and an appropriate -ve integer on failure. 546 */ 547 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr) 548 { 549 struct stmmac_priv *priv = netdev_priv(dev); 550 struct hwtstamp_config config; 551 struct timespec64 now; 552 u64 temp = 0; 553 u32 ptp_v2 = 0; 554 u32 tstamp_all = 0; 555 u32 ptp_over_ipv4_udp = 0; 556 u32 ptp_over_ipv6_udp = 0; 557 u32 ptp_over_ethernet = 0; 558 u32 snap_type_sel = 0; 559 u32 ts_master_en = 0; 560 u32 ts_event_en = 0; 561 u32 value = 0; 562 u32 sec_inc; 563 bool xmac; 564 565 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 566 567 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 568 netdev_alert(priv->dev, "No support for HW time stamping\n"); 569 priv->hwts_tx_en = 0; 570 priv->hwts_rx_en = 0; 571 572 return -EOPNOTSUPP; 573 } 574 575 if (copy_from_user(&config, ifr->ifr_data, 576 sizeof(struct hwtstamp_config))) 577 return -EFAULT; 578 579 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 580 __func__, config.flags, config.tx_type, config.rx_filter); 581 582 /* reserved for future extensions */ 583 if (config.flags) 584 return -EINVAL; 585 586 if (config.tx_type != HWTSTAMP_TX_OFF && 587 config.tx_type != HWTSTAMP_TX_ON) 588 return -ERANGE; 589 590 if (priv->adv_ts) { 591 switch (config.rx_filter) { 592 case HWTSTAMP_FILTER_NONE: 593 /* time stamp no incoming packet at all */ 594 config.rx_filter = HWTSTAMP_FILTER_NONE; 595 break; 596 597 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 598 /* PTP v1, UDP, any kind of event packet */ 599 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 600 /* take time stamp for all event messages */ 601 if (xmac) 602 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 603 else 604 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 605 606 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 607 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 608 break; 609 610 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 611 /* PTP v1, UDP, Sync packet */ 612 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 613 /* take time stamp for SYNC messages only */ 614 ts_event_en = PTP_TCR_TSEVNTENA; 615 616 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 617 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 618 break; 619 620 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 621 /* PTP v1, UDP, Delay_req packet */ 622 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 623 /* take time stamp for Delay_Req messages only */ 624 ts_master_en = PTP_TCR_TSMSTRENA; 625 ts_event_en = PTP_TCR_TSEVNTENA; 626 627 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 628 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 629 break; 630 631 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 632 /* PTP v2, UDP, any kind of event packet */ 633 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 634 ptp_v2 = PTP_TCR_TSVER2ENA; 635 /* take time stamp for all event messages */ 636 if (xmac) 637 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 638 else 639 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 640 641 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 642 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 643 break; 644 645 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 646 /* PTP v2, UDP, Sync packet */ 647 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 648 ptp_v2 = PTP_TCR_TSVER2ENA; 649 /* take time stamp for SYNC messages only */ 650 ts_event_en = PTP_TCR_TSEVNTENA; 651 652 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 653 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 654 break; 655 656 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 657 /* PTP v2, UDP, Delay_req packet */ 658 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 659 ptp_v2 = PTP_TCR_TSVER2ENA; 660 /* take time stamp for Delay_Req messages only */ 661 ts_master_en = PTP_TCR_TSMSTRENA; 662 ts_event_en = PTP_TCR_TSEVNTENA; 663 664 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 665 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 666 break; 667 668 case HWTSTAMP_FILTER_PTP_V2_EVENT: 669 /* PTP v2/802.AS1 any layer, any kind of event packet */ 670 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 671 ptp_v2 = PTP_TCR_TSVER2ENA; 672 /* take time stamp for all event messages */ 673 if (xmac) 674 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 675 else 676 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 677 678 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 679 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 680 ptp_over_ethernet = PTP_TCR_TSIPENA; 681 break; 682 683 case HWTSTAMP_FILTER_PTP_V2_SYNC: 684 /* PTP v2/802.AS1, any layer, Sync packet */ 685 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 686 ptp_v2 = PTP_TCR_TSVER2ENA; 687 /* take time stamp for SYNC messages only */ 688 ts_event_en = PTP_TCR_TSEVNTENA; 689 690 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 691 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 692 ptp_over_ethernet = PTP_TCR_TSIPENA; 693 break; 694 695 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 696 /* PTP v2/802.AS1, any layer, Delay_req packet */ 697 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 698 ptp_v2 = PTP_TCR_TSVER2ENA; 699 /* take time stamp for Delay_Req messages only */ 700 ts_master_en = PTP_TCR_TSMSTRENA; 701 ts_event_en = PTP_TCR_TSEVNTENA; 702 703 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 704 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 705 ptp_over_ethernet = PTP_TCR_TSIPENA; 706 break; 707 708 case HWTSTAMP_FILTER_NTP_ALL: 709 case HWTSTAMP_FILTER_ALL: 710 /* time stamp any incoming packet */ 711 config.rx_filter = HWTSTAMP_FILTER_ALL; 712 tstamp_all = PTP_TCR_TSENALL; 713 break; 714 715 default: 716 return -ERANGE; 717 } 718 } else { 719 switch (config.rx_filter) { 720 case HWTSTAMP_FILTER_NONE: 721 config.rx_filter = HWTSTAMP_FILTER_NONE; 722 break; 723 default: 724 /* PTP v1, UDP, any kind of event packet */ 725 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 726 break; 727 } 728 } 729 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 730 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 731 732 if (!priv->hwts_tx_en && !priv->hwts_rx_en) 733 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0); 734 else { 735 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR | 736 tstamp_all | ptp_v2 | ptp_over_ethernet | 737 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en | 738 ts_master_en | snap_type_sel); 739 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value); 740 741 /* program Sub Second Increment reg */ 742 stmmac_config_sub_second_increment(priv, 743 priv->ptpaddr, priv->plat->clk_ptp_rate, 744 xmac, &sec_inc); 745 temp = div_u64(1000000000ULL, sec_inc); 746 747 /* Store sub second increment and flags for later use */ 748 priv->sub_second_inc = sec_inc; 749 priv->systime_flags = value; 750 751 /* calculate default added value: 752 * formula is : 753 * addend = (2^32)/freq_div_ratio; 754 * where, freq_div_ratio = 1e9ns/sec_inc 755 */ 756 temp = (u64)(temp << 32); 757 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 758 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 759 760 /* initialize system time */ 761 ktime_get_real_ts64(&now); 762 763 /* lower 32 bits of tv_sec are safe until y2106 */ 764 stmmac_init_systime(priv, priv->ptpaddr, 765 (u32)now.tv_sec, now.tv_nsec); 766 } 767 768 return copy_to_user(ifr->ifr_data, &config, 769 sizeof(struct hwtstamp_config)) ? -EFAULT : 0; 770 } 771 772 /** 773 * stmmac_init_ptp - init PTP 774 * @priv: driver private structure 775 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 776 * This is done by looking at the HW cap. register. 777 * This function also registers the ptp driver. 778 */ 779 static int stmmac_init_ptp(struct stmmac_priv *priv) 780 { 781 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 782 783 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 784 return -EOPNOTSUPP; 785 786 priv->adv_ts = 0; 787 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 788 if (xmac && priv->dma_cap.atime_stamp) 789 priv->adv_ts = 1; 790 /* Dwmac 3.x core with extend_desc can support adv_ts */ 791 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 792 priv->adv_ts = 1; 793 794 if (priv->dma_cap.time_stamp) 795 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 796 797 if (priv->adv_ts) 798 netdev_info(priv->dev, 799 "IEEE 1588-2008 Advanced Timestamp supported\n"); 800 801 priv->hwts_tx_en = 0; 802 priv->hwts_rx_en = 0; 803 804 stmmac_ptp_register(priv); 805 806 return 0; 807 } 808 809 static void stmmac_release_ptp(struct stmmac_priv *priv) 810 { 811 if (priv->plat->clk_ptp_ref) 812 clk_disable_unprepare(priv->plat->clk_ptp_ref); 813 stmmac_ptp_unregister(priv); 814 } 815 816 /** 817 * stmmac_mac_flow_ctrl - Configure flow control in all queues 818 * @priv: driver private structure 819 * Description: It is used for configuring the flow control in all queues 820 */ 821 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 822 { 823 u32 tx_cnt = priv->plat->tx_queues_to_use; 824 825 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 826 priv->pause, tx_cnt); 827 } 828 829 /** 830 * stmmac_adjust_link - adjusts the link parameters 831 * @dev: net device structure 832 * Description: this is the helper called by the physical abstraction layer 833 * drivers to communicate the phy link status. According the speed and duplex 834 * this driver can invoke registered glue-logic as well. 835 * It also invoke the eee initialization because it could happen when switch 836 * on different networks (that are eee capable). 837 */ 838 static void stmmac_adjust_link(struct net_device *dev) 839 { 840 struct stmmac_priv *priv = netdev_priv(dev); 841 struct phy_device *phydev = dev->phydev; 842 bool new_state = false; 843 844 if (!phydev) 845 return; 846 847 mutex_lock(&priv->lock); 848 849 if (phydev->link) { 850 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 851 852 /* Now we make sure that we can be in full duplex mode. 853 * If not, we operate in half-duplex mode. */ 854 if (phydev->duplex != priv->oldduplex) { 855 new_state = true; 856 if (!phydev->duplex) 857 ctrl &= ~priv->hw->link.duplex; 858 else 859 ctrl |= priv->hw->link.duplex; 860 priv->oldduplex = phydev->duplex; 861 } 862 /* Flow Control operation */ 863 if (phydev->pause) 864 stmmac_mac_flow_ctrl(priv, phydev->duplex); 865 866 if (phydev->speed != priv->speed) { 867 new_state = true; 868 ctrl &= ~priv->hw->link.speed_mask; 869 switch (phydev->speed) { 870 case SPEED_1000: 871 ctrl |= priv->hw->link.speed1000; 872 break; 873 case SPEED_100: 874 ctrl |= priv->hw->link.speed100; 875 break; 876 case SPEED_10: 877 ctrl |= priv->hw->link.speed10; 878 break; 879 default: 880 netif_warn(priv, link, priv->dev, 881 "broken speed: %d\n", phydev->speed); 882 phydev->speed = SPEED_UNKNOWN; 883 break; 884 } 885 if (phydev->speed != SPEED_UNKNOWN) 886 stmmac_hw_fix_mac_speed(priv); 887 priv->speed = phydev->speed; 888 } 889 890 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 891 892 if (!priv->oldlink) { 893 new_state = true; 894 priv->oldlink = true; 895 } 896 } else if (priv->oldlink) { 897 new_state = true; 898 priv->oldlink = false; 899 priv->speed = SPEED_UNKNOWN; 900 priv->oldduplex = DUPLEX_UNKNOWN; 901 } 902 903 if (new_state && netif_msg_link(priv)) 904 phy_print_status(phydev); 905 906 mutex_unlock(&priv->lock); 907 908 if (phydev->is_pseudo_fixed_link) 909 /* Stop PHY layer to call the hook to adjust the link in case 910 * of a switch is attached to the stmmac driver. 911 */ 912 phydev->irq = PHY_IGNORE_INTERRUPT; 913 else 914 /* At this stage, init the EEE if supported. 915 * Never called in case of fixed_link. 916 */ 917 priv->eee_enabled = stmmac_eee_init(priv); 918 } 919 920 /** 921 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 922 * @priv: driver private structure 923 * Description: this is to verify if the HW supports the PCS. 924 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 925 * configured for the TBI, RTBI, or SGMII PHY interface. 926 */ 927 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 928 { 929 int interface = priv->plat->interface; 930 931 if (priv->dma_cap.pcs) { 932 if ((interface == PHY_INTERFACE_MODE_RGMII) || 933 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 934 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 935 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 936 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 937 priv->hw->pcs = STMMAC_PCS_RGMII; 938 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 939 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 940 priv->hw->pcs = STMMAC_PCS_SGMII; 941 } 942 } 943 } 944 945 /** 946 * stmmac_init_phy - PHY initialization 947 * @dev: net device structure 948 * Description: it initializes the driver's PHY state, and attaches the PHY 949 * to the mac driver. 950 * Return value: 951 * 0 on success 952 */ 953 static int stmmac_init_phy(struct net_device *dev) 954 { 955 struct stmmac_priv *priv = netdev_priv(dev); 956 u32 tx_cnt = priv->plat->tx_queues_to_use; 957 struct phy_device *phydev; 958 char phy_id_fmt[MII_BUS_ID_SIZE + 3]; 959 char bus_id[MII_BUS_ID_SIZE]; 960 int interface = priv->plat->interface; 961 int max_speed = priv->plat->max_speed; 962 priv->oldlink = false; 963 priv->speed = SPEED_UNKNOWN; 964 priv->oldduplex = DUPLEX_UNKNOWN; 965 966 if (priv->plat->phy_node) { 967 phydev = of_phy_connect(dev, priv->plat->phy_node, 968 &stmmac_adjust_link, 0, interface); 969 } else { 970 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x", 971 priv->plat->bus_id); 972 973 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id, 974 priv->plat->phy_addr); 975 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__, 976 phy_id_fmt); 977 978 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link, 979 interface); 980 } 981 982 if (IS_ERR_OR_NULL(phydev)) { 983 netdev_err(priv->dev, "Could not attach to PHY\n"); 984 if (!phydev) 985 return -ENODEV; 986 987 return PTR_ERR(phydev); 988 } 989 990 /* Stop Advertising 1000BASE Capability if interface is not GMII */ 991 if ((interface == PHY_INTERFACE_MODE_MII) || 992 (interface == PHY_INTERFACE_MODE_RMII) || 993 (max_speed < 1000 && max_speed > 0)) 994 phy_set_max_speed(phydev, SPEED_100); 995 996 /* 997 * Half-duplex mode not supported with multiqueue 998 * half-duplex can only works with single queue 999 */ 1000 if (tx_cnt > 1) { 1001 phy_remove_link_mode(phydev, 1002 ETHTOOL_LINK_MODE_10baseT_Half_BIT); 1003 phy_remove_link_mode(phydev, 1004 ETHTOOL_LINK_MODE_100baseT_Half_BIT); 1005 phy_remove_link_mode(phydev, 1006 ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 1007 } 1008 1009 /* 1010 * Broken HW is sometimes missing the pull-up resistor on the 1011 * MDIO line, which results in reads to non-existent devices returning 1012 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent 1013 * device as well. 1014 * Note: phydev->phy_id is the result of reading the UID PHY registers. 1015 */ 1016 if (!priv->plat->phy_node && phydev->phy_id == 0) { 1017 phy_disconnect(phydev); 1018 return -ENODEV; 1019 } 1020 1021 /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid 1022 * subsequent PHY polling, make sure we force a link transition if 1023 * we have a UP/DOWN/UP transition 1024 */ 1025 if (phydev->is_pseudo_fixed_link) 1026 phydev->irq = PHY_POLL; 1027 1028 phy_attached_info(phydev); 1029 return 0; 1030 } 1031 1032 static void stmmac_display_rx_rings(struct stmmac_priv *priv) 1033 { 1034 u32 rx_cnt = priv->plat->rx_queues_to_use; 1035 void *head_rx; 1036 u32 queue; 1037 1038 /* Display RX rings */ 1039 for (queue = 0; queue < rx_cnt; queue++) { 1040 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1041 1042 pr_info("\tRX Queue %u rings\n", queue); 1043 1044 if (priv->extend_desc) 1045 head_rx = (void *)rx_q->dma_erx; 1046 else 1047 head_rx = (void *)rx_q->dma_rx; 1048 1049 /* Display RX ring */ 1050 stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true); 1051 } 1052 } 1053 1054 static void stmmac_display_tx_rings(struct stmmac_priv *priv) 1055 { 1056 u32 tx_cnt = priv->plat->tx_queues_to_use; 1057 void *head_tx; 1058 u32 queue; 1059 1060 /* Display TX rings */ 1061 for (queue = 0; queue < tx_cnt; queue++) { 1062 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1063 1064 pr_info("\tTX Queue %d rings\n", queue); 1065 1066 if (priv->extend_desc) 1067 head_tx = (void *)tx_q->dma_etx; 1068 else 1069 head_tx = (void *)tx_q->dma_tx; 1070 1071 stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false); 1072 } 1073 } 1074 1075 static void stmmac_display_rings(struct stmmac_priv *priv) 1076 { 1077 /* Display RX ring */ 1078 stmmac_display_rx_rings(priv); 1079 1080 /* Display TX ring */ 1081 stmmac_display_tx_rings(priv); 1082 } 1083 1084 static int stmmac_set_bfsize(int mtu, int bufsize) 1085 { 1086 int ret = bufsize; 1087 1088 if (mtu >= BUF_SIZE_4KiB) 1089 ret = BUF_SIZE_8KiB; 1090 else if (mtu >= BUF_SIZE_2KiB) 1091 ret = BUF_SIZE_4KiB; 1092 else if (mtu > DEFAULT_BUFSIZE) 1093 ret = BUF_SIZE_2KiB; 1094 else 1095 ret = DEFAULT_BUFSIZE; 1096 1097 return ret; 1098 } 1099 1100 /** 1101 * stmmac_clear_rx_descriptors - clear RX descriptors 1102 * @priv: driver private structure 1103 * @queue: RX queue index 1104 * Description: this function is called to clear the RX descriptors 1105 * in case of both basic and extended descriptors are used. 1106 */ 1107 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) 1108 { 1109 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1110 int i; 1111 1112 /* Clear the RX descriptors */ 1113 for (i = 0; i < DMA_RX_SIZE; i++) 1114 if (priv->extend_desc) 1115 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1116 priv->use_riwt, priv->mode, 1117 (i == DMA_RX_SIZE - 1)); 1118 else 1119 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1120 priv->use_riwt, priv->mode, 1121 (i == DMA_RX_SIZE - 1)); 1122 } 1123 1124 /** 1125 * stmmac_clear_tx_descriptors - clear tx descriptors 1126 * @priv: driver private structure 1127 * @queue: TX queue index. 1128 * Description: this function is called to clear the TX descriptors 1129 * in case of both basic and extended descriptors are used. 1130 */ 1131 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) 1132 { 1133 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1134 int i; 1135 1136 /* Clear the TX descriptors */ 1137 for (i = 0; i < DMA_TX_SIZE; i++) 1138 if (priv->extend_desc) 1139 stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic, 1140 priv->mode, (i == DMA_TX_SIZE - 1)); 1141 else 1142 stmmac_init_tx_desc(priv, &tx_q->dma_tx[i], 1143 priv->mode, (i == DMA_TX_SIZE - 1)); 1144 } 1145 1146 /** 1147 * stmmac_clear_descriptors - clear descriptors 1148 * @priv: driver private structure 1149 * Description: this function is called to clear the TX and RX descriptors 1150 * in case of both basic and extended descriptors are used. 1151 */ 1152 static void stmmac_clear_descriptors(struct stmmac_priv *priv) 1153 { 1154 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1155 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1156 u32 queue; 1157 1158 /* Clear the RX descriptors */ 1159 for (queue = 0; queue < rx_queue_cnt; queue++) 1160 stmmac_clear_rx_descriptors(priv, queue); 1161 1162 /* Clear the TX descriptors */ 1163 for (queue = 0; queue < tx_queue_cnt; queue++) 1164 stmmac_clear_tx_descriptors(priv, queue); 1165 } 1166 1167 /** 1168 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1169 * @priv: driver private structure 1170 * @p: descriptor pointer 1171 * @i: descriptor index 1172 * @flags: gfp flag 1173 * @queue: RX queue index 1174 * Description: this function is called to allocate a receive buffer, perform 1175 * the DMA mapping and init the descriptor. 1176 */ 1177 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, 1178 int i, gfp_t flags, u32 queue) 1179 { 1180 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1181 struct sk_buff *skb; 1182 1183 skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags); 1184 if (!skb) { 1185 netdev_err(priv->dev, 1186 "%s: Rx init fails; skb is NULL\n", __func__); 1187 return -ENOMEM; 1188 } 1189 rx_q->rx_skbuff[i] = skb; 1190 rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data, 1191 priv->dma_buf_sz, 1192 DMA_FROM_DEVICE); 1193 if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) { 1194 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); 1195 dev_kfree_skb_any(skb); 1196 return -EINVAL; 1197 } 1198 1199 stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[i]); 1200 1201 if (priv->dma_buf_sz == BUF_SIZE_16KiB) 1202 stmmac_init_desc3(priv, p); 1203 1204 return 0; 1205 } 1206 1207 /** 1208 * stmmac_free_rx_buffer - free RX dma buffers 1209 * @priv: private structure 1210 * @queue: RX queue index 1211 * @i: buffer index. 1212 */ 1213 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1214 { 1215 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1216 1217 if (rx_q->rx_skbuff[i]) { 1218 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i], 1219 priv->dma_buf_sz, DMA_FROM_DEVICE); 1220 dev_kfree_skb_any(rx_q->rx_skbuff[i]); 1221 } 1222 rx_q->rx_skbuff[i] = NULL; 1223 } 1224 1225 /** 1226 * stmmac_free_tx_buffer - free RX dma buffers 1227 * @priv: private structure 1228 * @queue: RX queue index 1229 * @i: buffer index. 1230 */ 1231 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1232 { 1233 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1234 1235 if (tx_q->tx_skbuff_dma[i].buf) { 1236 if (tx_q->tx_skbuff_dma[i].map_as_page) 1237 dma_unmap_page(priv->device, 1238 tx_q->tx_skbuff_dma[i].buf, 1239 tx_q->tx_skbuff_dma[i].len, 1240 DMA_TO_DEVICE); 1241 else 1242 dma_unmap_single(priv->device, 1243 tx_q->tx_skbuff_dma[i].buf, 1244 tx_q->tx_skbuff_dma[i].len, 1245 DMA_TO_DEVICE); 1246 } 1247 1248 if (tx_q->tx_skbuff[i]) { 1249 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1250 tx_q->tx_skbuff[i] = NULL; 1251 tx_q->tx_skbuff_dma[i].buf = 0; 1252 tx_q->tx_skbuff_dma[i].map_as_page = false; 1253 } 1254 } 1255 1256 /** 1257 * init_dma_rx_desc_rings - init the RX descriptor rings 1258 * @dev: net device structure 1259 * @flags: gfp flag. 1260 * Description: this function initializes the DMA RX descriptors 1261 * and allocates the socket buffers. It supports the chained and ring 1262 * modes. 1263 */ 1264 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) 1265 { 1266 struct stmmac_priv *priv = netdev_priv(dev); 1267 u32 rx_count = priv->plat->rx_queues_to_use; 1268 int ret = -ENOMEM; 1269 int bfsize = 0; 1270 int queue; 1271 int i; 1272 1273 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); 1274 if (bfsize < 0) 1275 bfsize = 0; 1276 1277 if (bfsize < BUF_SIZE_16KiB) 1278 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); 1279 1280 priv->dma_buf_sz = bfsize; 1281 1282 /* RX INITIALIZATION */ 1283 netif_dbg(priv, probe, priv->dev, 1284 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1285 1286 for (queue = 0; queue < rx_count; queue++) { 1287 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1288 1289 netif_dbg(priv, probe, priv->dev, 1290 "(%s) dma_rx_phy=0x%08x\n", __func__, 1291 (u32)rx_q->dma_rx_phy); 1292 1293 for (i = 0; i < DMA_RX_SIZE; i++) { 1294 struct dma_desc *p; 1295 1296 if (priv->extend_desc) 1297 p = &((rx_q->dma_erx + i)->basic); 1298 else 1299 p = rx_q->dma_rx + i; 1300 1301 ret = stmmac_init_rx_buffers(priv, p, i, flags, 1302 queue); 1303 if (ret) 1304 goto err_init_rx_buffers; 1305 1306 netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n", 1307 rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data, 1308 (unsigned int)rx_q->rx_skbuff_dma[i]); 1309 } 1310 1311 rx_q->cur_rx = 0; 1312 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE); 1313 1314 stmmac_clear_rx_descriptors(priv, queue); 1315 1316 /* Setup the chained descriptor addresses */ 1317 if (priv->mode == STMMAC_CHAIN_MODE) { 1318 if (priv->extend_desc) 1319 stmmac_mode_init(priv, rx_q->dma_erx, 1320 rx_q->dma_rx_phy, DMA_RX_SIZE, 1); 1321 else 1322 stmmac_mode_init(priv, rx_q->dma_rx, 1323 rx_q->dma_rx_phy, DMA_RX_SIZE, 0); 1324 } 1325 } 1326 1327 buf_sz = bfsize; 1328 1329 return 0; 1330 1331 err_init_rx_buffers: 1332 while (queue >= 0) { 1333 while (--i >= 0) 1334 stmmac_free_rx_buffer(priv, queue, i); 1335 1336 if (queue == 0) 1337 break; 1338 1339 i = DMA_RX_SIZE; 1340 queue--; 1341 } 1342 1343 return ret; 1344 } 1345 1346 /** 1347 * init_dma_tx_desc_rings - init the TX descriptor rings 1348 * @dev: net device structure. 1349 * Description: this function initializes the DMA TX descriptors 1350 * and allocates the socket buffers. It supports the chained and ring 1351 * modes. 1352 */ 1353 static int init_dma_tx_desc_rings(struct net_device *dev) 1354 { 1355 struct stmmac_priv *priv = netdev_priv(dev); 1356 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1357 u32 queue; 1358 int i; 1359 1360 for (queue = 0; queue < tx_queue_cnt; queue++) { 1361 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1362 1363 netif_dbg(priv, probe, priv->dev, 1364 "(%s) dma_tx_phy=0x%08x\n", __func__, 1365 (u32)tx_q->dma_tx_phy); 1366 1367 /* Setup the chained descriptor addresses */ 1368 if (priv->mode == STMMAC_CHAIN_MODE) { 1369 if (priv->extend_desc) 1370 stmmac_mode_init(priv, tx_q->dma_etx, 1371 tx_q->dma_tx_phy, DMA_TX_SIZE, 1); 1372 else 1373 stmmac_mode_init(priv, tx_q->dma_tx, 1374 tx_q->dma_tx_phy, DMA_TX_SIZE, 0); 1375 } 1376 1377 for (i = 0; i < DMA_TX_SIZE; i++) { 1378 struct dma_desc *p; 1379 if (priv->extend_desc) 1380 p = &((tx_q->dma_etx + i)->basic); 1381 else 1382 p = tx_q->dma_tx + i; 1383 1384 stmmac_clear_desc(priv, p); 1385 1386 tx_q->tx_skbuff_dma[i].buf = 0; 1387 tx_q->tx_skbuff_dma[i].map_as_page = false; 1388 tx_q->tx_skbuff_dma[i].len = 0; 1389 tx_q->tx_skbuff_dma[i].last_segment = false; 1390 tx_q->tx_skbuff[i] = NULL; 1391 } 1392 1393 tx_q->dirty_tx = 0; 1394 tx_q->cur_tx = 0; 1395 tx_q->mss = 0; 1396 1397 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 1398 } 1399 1400 return 0; 1401 } 1402 1403 /** 1404 * init_dma_desc_rings - init the RX/TX descriptor rings 1405 * @dev: net device structure 1406 * @flags: gfp flag. 1407 * Description: this function initializes the DMA RX/TX descriptors 1408 * and allocates the socket buffers. It supports the chained and ring 1409 * modes. 1410 */ 1411 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) 1412 { 1413 struct stmmac_priv *priv = netdev_priv(dev); 1414 int ret; 1415 1416 ret = init_dma_rx_desc_rings(dev, flags); 1417 if (ret) 1418 return ret; 1419 1420 ret = init_dma_tx_desc_rings(dev); 1421 1422 stmmac_clear_descriptors(priv); 1423 1424 if (netif_msg_hw(priv)) 1425 stmmac_display_rings(priv); 1426 1427 return ret; 1428 } 1429 1430 /** 1431 * dma_free_rx_skbufs - free RX dma buffers 1432 * @priv: private structure 1433 * @queue: RX queue index 1434 */ 1435 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) 1436 { 1437 int i; 1438 1439 for (i = 0; i < DMA_RX_SIZE; i++) 1440 stmmac_free_rx_buffer(priv, queue, i); 1441 } 1442 1443 /** 1444 * dma_free_tx_skbufs - free TX dma buffers 1445 * @priv: private structure 1446 * @queue: TX queue index 1447 */ 1448 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) 1449 { 1450 int i; 1451 1452 for (i = 0; i < DMA_TX_SIZE; i++) 1453 stmmac_free_tx_buffer(priv, queue, i); 1454 } 1455 1456 /** 1457 * free_dma_rx_desc_resources - free RX dma desc resources 1458 * @priv: private structure 1459 */ 1460 static void free_dma_rx_desc_resources(struct stmmac_priv *priv) 1461 { 1462 u32 rx_count = priv->plat->rx_queues_to_use; 1463 u32 queue; 1464 1465 /* Free RX queue resources */ 1466 for (queue = 0; queue < rx_count; queue++) { 1467 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1468 1469 /* Release the DMA RX socket buffers */ 1470 dma_free_rx_skbufs(priv, queue); 1471 1472 /* Free DMA regions of consistent memory previously allocated */ 1473 if (!priv->extend_desc) 1474 dma_free_coherent(priv->device, 1475 DMA_RX_SIZE * sizeof(struct dma_desc), 1476 rx_q->dma_rx, rx_q->dma_rx_phy); 1477 else 1478 dma_free_coherent(priv->device, DMA_RX_SIZE * 1479 sizeof(struct dma_extended_desc), 1480 rx_q->dma_erx, rx_q->dma_rx_phy); 1481 1482 kfree(rx_q->rx_skbuff_dma); 1483 kfree(rx_q->rx_skbuff); 1484 } 1485 } 1486 1487 /** 1488 * free_dma_tx_desc_resources - free TX dma desc resources 1489 * @priv: private structure 1490 */ 1491 static void free_dma_tx_desc_resources(struct stmmac_priv *priv) 1492 { 1493 u32 tx_count = priv->plat->tx_queues_to_use; 1494 u32 queue; 1495 1496 /* Free TX queue resources */ 1497 for (queue = 0; queue < tx_count; queue++) { 1498 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1499 1500 /* Release the DMA TX socket buffers */ 1501 dma_free_tx_skbufs(priv, queue); 1502 1503 /* Free DMA regions of consistent memory previously allocated */ 1504 if (!priv->extend_desc) 1505 dma_free_coherent(priv->device, 1506 DMA_TX_SIZE * sizeof(struct dma_desc), 1507 tx_q->dma_tx, tx_q->dma_tx_phy); 1508 else 1509 dma_free_coherent(priv->device, DMA_TX_SIZE * 1510 sizeof(struct dma_extended_desc), 1511 tx_q->dma_etx, tx_q->dma_tx_phy); 1512 1513 kfree(tx_q->tx_skbuff_dma); 1514 kfree(tx_q->tx_skbuff); 1515 } 1516 } 1517 1518 /** 1519 * alloc_dma_rx_desc_resources - alloc RX resources. 1520 * @priv: private structure 1521 * Description: according to which descriptor can be used (extend or basic) 1522 * this function allocates the resources for TX and RX paths. In case of 1523 * reception, for example, it pre-allocated the RX socket buffer in order to 1524 * allow zero-copy mechanism. 1525 */ 1526 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) 1527 { 1528 u32 rx_count = priv->plat->rx_queues_to_use; 1529 int ret = -ENOMEM; 1530 u32 queue; 1531 1532 /* RX queues buffers and DMA */ 1533 for (queue = 0; queue < rx_count; queue++) { 1534 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1535 1536 rx_q->queue_index = queue; 1537 rx_q->priv_data = priv; 1538 1539 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, 1540 sizeof(dma_addr_t), 1541 GFP_KERNEL); 1542 if (!rx_q->rx_skbuff_dma) 1543 goto err_dma; 1544 1545 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE, 1546 sizeof(struct sk_buff *), 1547 GFP_KERNEL); 1548 if (!rx_q->rx_skbuff) 1549 goto err_dma; 1550 1551 if (priv->extend_desc) { 1552 rx_q->dma_erx = dma_zalloc_coherent(priv->device, 1553 DMA_RX_SIZE * 1554 sizeof(struct 1555 dma_extended_desc), 1556 &rx_q->dma_rx_phy, 1557 GFP_KERNEL); 1558 if (!rx_q->dma_erx) 1559 goto err_dma; 1560 1561 } else { 1562 rx_q->dma_rx = dma_zalloc_coherent(priv->device, 1563 DMA_RX_SIZE * 1564 sizeof(struct 1565 dma_desc), 1566 &rx_q->dma_rx_phy, 1567 GFP_KERNEL); 1568 if (!rx_q->dma_rx) 1569 goto err_dma; 1570 } 1571 } 1572 1573 return 0; 1574 1575 err_dma: 1576 free_dma_rx_desc_resources(priv); 1577 1578 return ret; 1579 } 1580 1581 /** 1582 * alloc_dma_tx_desc_resources - alloc TX resources. 1583 * @priv: private structure 1584 * Description: according to which descriptor can be used (extend or basic) 1585 * this function allocates the resources for TX and RX paths. In case of 1586 * reception, for example, it pre-allocated the RX socket buffer in order to 1587 * allow zero-copy mechanism. 1588 */ 1589 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) 1590 { 1591 u32 tx_count = priv->plat->tx_queues_to_use; 1592 int ret = -ENOMEM; 1593 u32 queue; 1594 1595 /* TX queues buffers and DMA */ 1596 for (queue = 0; queue < tx_count; queue++) { 1597 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1598 1599 tx_q->queue_index = queue; 1600 tx_q->priv_data = priv; 1601 1602 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE, 1603 sizeof(*tx_q->tx_skbuff_dma), 1604 GFP_KERNEL); 1605 if (!tx_q->tx_skbuff_dma) 1606 goto err_dma; 1607 1608 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE, 1609 sizeof(struct sk_buff *), 1610 GFP_KERNEL); 1611 if (!tx_q->tx_skbuff) 1612 goto err_dma; 1613 1614 if (priv->extend_desc) { 1615 tx_q->dma_etx = dma_zalloc_coherent(priv->device, 1616 DMA_TX_SIZE * 1617 sizeof(struct 1618 dma_extended_desc), 1619 &tx_q->dma_tx_phy, 1620 GFP_KERNEL); 1621 if (!tx_q->dma_etx) 1622 goto err_dma; 1623 } else { 1624 tx_q->dma_tx = dma_zalloc_coherent(priv->device, 1625 DMA_TX_SIZE * 1626 sizeof(struct 1627 dma_desc), 1628 &tx_q->dma_tx_phy, 1629 GFP_KERNEL); 1630 if (!tx_q->dma_tx) 1631 goto err_dma; 1632 } 1633 } 1634 1635 return 0; 1636 1637 err_dma: 1638 free_dma_tx_desc_resources(priv); 1639 1640 return ret; 1641 } 1642 1643 /** 1644 * alloc_dma_desc_resources - alloc TX/RX resources. 1645 * @priv: private structure 1646 * Description: according to which descriptor can be used (extend or basic) 1647 * this function allocates the resources for TX and RX paths. In case of 1648 * reception, for example, it pre-allocated the RX socket buffer in order to 1649 * allow zero-copy mechanism. 1650 */ 1651 static int alloc_dma_desc_resources(struct stmmac_priv *priv) 1652 { 1653 /* RX Allocation */ 1654 int ret = alloc_dma_rx_desc_resources(priv); 1655 1656 if (ret) 1657 return ret; 1658 1659 ret = alloc_dma_tx_desc_resources(priv); 1660 1661 return ret; 1662 } 1663 1664 /** 1665 * free_dma_desc_resources - free dma desc resources 1666 * @priv: private structure 1667 */ 1668 static void free_dma_desc_resources(struct stmmac_priv *priv) 1669 { 1670 /* Release the DMA RX socket buffers */ 1671 free_dma_rx_desc_resources(priv); 1672 1673 /* Release the DMA TX socket buffers */ 1674 free_dma_tx_desc_resources(priv); 1675 } 1676 1677 /** 1678 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 1679 * @priv: driver private structure 1680 * Description: It is used for enabling the rx queues in the MAC 1681 */ 1682 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 1683 { 1684 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1685 int queue; 1686 u8 mode; 1687 1688 for (queue = 0; queue < rx_queues_count; queue++) { 1689 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 1690 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 1691 } 1692 } 1693 1694 /** 1695 * stmmac_start_rx_dma - start RX DMA channel 1696 * @priv: driver private structure 1697 * @chan: RX channel index 1698 * Description: 1699 * This starts a RX DMA channel 1700 */ 1701 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 1702 { 1703 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 1704 stmmac_start_rx(priv, priv->ioaddr, chan); 1705 } 1706 1707 /** 1708 * stmmac_start_tx_dma - start TX DMA channel 1709 * @priv: driver private structure 1710 * @chan: TX channel index 1711 * Description: 1712 * This starts a TX DMA channel 1713 */ 1714 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 1715 { 1716 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 1717 stmmac_start_tx(priv, priv->ioaddr, chan); 1718 } 1719 1720 /** 1721 * stmmac_stop_rx_dma - stop RX DMA channel 1722 * @priv: driver private structure 1723 * @chan: RX channel index 1724 * Description: 1725 * This stops a RX DMA channel 1726 */ 1727 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 1728 { 1729 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 1730 stmmac_stop_rx(priv, priv->ioaddr, chan); 1731 } 1732 1733 /** 1734 * stmmac_stop_tx_dma - stop TX DMA channel 1735 * @priv: driver private structure 1736 * @chan: TX channel index 1737 * Description: 1738 * This stops a TX DMA channel 1739 */ 1740 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 1741 { 1742 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 1743 stmmac_stop_tx(priv, priv->ioaddr, chan); 1744 } 1745 1746 /** 1747 * stmmac_start_all_dma - start all RX and TX DMA channels 1748 * @priv: driver private structure 1749 * Description: 1750 * This starts all the RX and TX DMA channels 1751 */ 1752 static void stmmac_start_all_dma(struct stmmac_priv *priv) 1753 { 1754 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1755 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1756 u32 chan = 0; 1757 1758 for (chan = 0; chan < rx_channels_count; chan++) 1759 stmmac_start_rx_dma(priv, chan); 1760 1761 for (chan = 0; chan < tx_channels_count; chan++) 1762 stmmac_start_tx_dma(priv, chan); 1763 } 1764 1765 /** 1766 * stmmac_stop_all_dma - stop all RX and TX DMA channels 1767 * @priv: driver private structure 1768 * Description: 1769 * This stops the RX and TX DMA channels 1770 */ 1771 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 1772 { 1773 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1774 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1775 u32 chan = 0; 1776 1777 for (chan = 0; chan < rx_channels_count; chan++) 1778 stmmac_stop_rx_dma(priv, chan); 1779 1780 for (chan = 0; chan < tx_channels_count; chan++) 1781 stmmac_stop_tx_dma(priv, chan); 1782 } 1783 1784 /** 1785 * stmmac_dma_operation_mode - HW DMA operation mode 1786 * @priv: driver private structure 1787 * Description: it is used for configuring the DMA operation mode register in 1788 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 1789 */ 1790 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 1791 { 1792 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1793 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1794 int rxfifosz = priv->plat->rx_fifo_size; 1795 int txfifosz = priv->plat->tx_fifo_size; 1796 u32 txmode = 0; 1797 u32 rxmode = 0; 1798 u32 chan = 0; 1799 u8 qmode = 0; 1800 1801 if (rxfifosz == 0) 1802 rxfifosz = priv->dma_cap.rx_fifo_size; 1803 if (txfifosz == 0) 1804 txfifosz = priv->dma_cap.tx_fifo_size; 1805 1806 /* Adjust for real per queue fifo size */ 1807 rxfifosz /= rx_channels_count; 1808 txfifosz /= tx_channels_count; 1809 1810 if (priv->plat->force_thresh_dma_mode) { 1811 txmode = tc; 1812 rxmode = tc; 1813 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 1814 /* 1815 * In case of GMAC, SF mode can be enabled 1816 * to perform the TX COE in HW. This depends on: 1817 * 1) TX COE if actually supported 1818 * 2) There is no bugged Jumbo frame support 1819 * that needs to not insert csum in the TDES. 1820 */ 1821 txmode = SF_DMA_MODE; 1822 rxmode = SF_DMA_MODE; 1823 priv->xstats.threshold = SF_DMA_MODE; 1824 } else { 1825 txmode = tc; 1826 rxmode = SF_DMA_MODE; 1827 } 1828 1829 /* configure all channels */ 1830 for (chan = 0; chan < rx_channels_count; chan++) { 1831 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 1832 1833 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 1834 rxfifosz, qmode); 1835 stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz, 1836 chan); 1837 } 1838 1839 for (chan = 0; chan < tx_channels_count; chan++) { 1840 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 1841 1842 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 1843 txfifosz, qmode); 1844 } 1845 } 1846 1847 /** 1848 * stmmac_tx_clean - to manage the transmission completion 1849 * @priv: driver private structure 1850 * @queue: TX queue index 1851 * Description: it reclaims the transmit resources after transmission completes. 1852 */ 1853 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) 1854 { 1855 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1856 unsigned int bytes_compl = 0, pkts_compl = 0; 1857 unsigned int entry, count = 0; 1858 1859 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 1860 1861 priv->xstats.tx_clean++; 1862 1863 entry = tx_q->dirty_tx; 1864 while ((entry != tx_q->cur_tx) && (count < budget)) { 1865 struct sk_buff *skb = tx_q->tx_skbuff[entry]; 1866 struct dma_desc *p; 1867 int status; 1868 1869 if (priv->extend_desc) 1870 p = (struct dma_desc *)(tx_q->dma_etx + entry); 1871 else 1872 p = tx_q->dma_tx + entry; 1873 1874 status = stmmac_tx_status(priv, &priv->dev->stats, 1875 &priv->xstats, p, priv->ioaddr); 1876 /* Check if the descriptor is owned by the DMA */ 1877 if (unlikely(status & tx_dma_own)) 1878 break; 1879 1880 count++; 1881 1882 /* Make sure descriptor fields are read after reading 1883 * the own bit. 1884 */ 1885 dma_rmb(); 1886 1887 /* Just consider the last segment and ...*/ 1888 if (likely(!(status & tx_not_ls))) { 1889 /* ... verify the status error condition */ 1890 if (unlikely(status & tx_err)) { 1891 priv->dev->stats.tx_errors++; 1892 } else { 1893 priv->dev->stats.tx_packets++; 1894 priv->xstats.tx_pkt_n++; 1895 } 1896 stmmac_get_tx_hwtstamp(priv, p, skb); 1897 } 1898 1899 if (likely(tx_q->tx_skbuff_dma[entry].buf)) { 1900 if (tx_q->tx_skbuff_dma[entry].map_as_page) 1901 dma_unmap_page(priv->device, 1902 tx_q->tx_skbuff_dma[entry].buf, 1903 tx_q->tx_skbuff_dma[entry].len, 1904 DMA_TO_DEVICE); 1905 else 1906 dma_unmap_single(priv->device, 1907 tx_q->tx_skbuff_dma[entry].buf, 1908 tx_q->tx_skbuff_dma[entry].len, 1909 DMA_TO_DEVICE); 1910 tx_q->tx_skbuff_dma[entry].buf = 0; 1911 tx_q->tx_skbuff_dma[entry].len = 0; 1912 tx_q->tx_skbuff_dma[entry].map_as_page = false; 1913 } 1914 1915 stmmac_clean_desc3(priv, tx_q, p); 1916 1917 tx_q->tx_skbuff_dma[entry].last_segment = false; 1918 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1919 1920 if (likely(skb != NULL)) { 1921 pkts_compl++; 1922 bytes_compl += skb->len; 1923 dev_consume_skb_any(skb); 1924 tx_q->tx_skbuff[entry] = NULL; 1925 } 1926 1927 stmmac_release_tx_desc(priv, p, priv->mode); 1928 1929 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 1930 } 1931 tx_q->dirty_tx = entry; 1932 1933 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 1934 pkts_compl, bytes_compl); 1935 1936 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 1937 queue))) && 1938 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) { 1939 1940 netif_dbg(priv, tx_done, priv->dev, 1941 "%s: restart transmit\n", __func__); 1942 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 1943 } 1944 1945 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) { 1946 stmmac_enable_eee_mode(priv); 1947 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 1948 } 1949 1950 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 1951 1952 return count; 1953 } 1954 1955 /** 1956 * stmmac_tx_err - to manage the tx error 1957 * @priv: driver private structure 1958 * @chan: channel index 1959 * Description: it cleans the descriptors and restarts the transmission 1960 * in case of transmission errors. 1961 */ 1962 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 1963 { 1964 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 1965 int i; 1966 1967 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 1968 1969 stmmac_stop_tx_dma(priv, chan); 1970 dma_free_tx_skbufs(priv, chan); 1971 for (i = 0; i < DMA_TX_SIZE; i++) 1972 if (priv->extend_desc) 1973 stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic, 1974 priv->mode, (i == DMA_TX_SIZE - 1)); 1975 else 1976 stmmac_init_tx_desc(priv, &tx_q->dma_tx[i], 1977 priv->mode, (i == DMA_TX_SIZE - 1)); 1978 tx_q->dirty_tx = 0; 1979 tx_q->cur_tx = 0; 1980 tx_q->mss = 0; 1981 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); 1982 stmmac_start_tx_dma(priv, chan); 1983 1984 priv->dev->stats.tx_errors++; 1985 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 1986 } 1987 1988 /** 1989 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 1990 * @priv: driver private structure 1991 * @txmode: TX operating mode 1992 * @rxmode: RX operating mode 1993 * @chan: channel index 1994 * Description: it is used for configuring of the DMA operation mode in 1995 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 1996 * mode. 1997 */ 1998 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 1999 u32 rxmode, u32 chan) 2000 { 2001 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2002 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2003 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2004 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2005 int rxfifosz = priv->plat->rx_fifo_size; 2006 int txfifosz = priv->plat->tx_fifo_size; 2007 2008 if (rxfifosz == 0) 2009 rxfifosz = priv->dma_cap.rx_fifo_size; 2010 if (txfifosz == 0) 2011 txfifosz = priv->dma_cap.tx_fifo_size; 2012 2013 /* Adjust for real per queue fifo size */ 2014 rxfifosz /= rx_channels_count; 2015 txfifosz /= tx_channels_count; 2016 2017 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2018 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2019 } 2020 2021 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2022 { 2023 int ret; 2024 2025 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2026 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2027 if (ret && (ret != -EINVAL)) { 2028 stmmac_global_err(priv); 2029 return true; 2030 } 2031 2032 return false; 2033 } 2034 2035 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan) 2036 { 2037 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2038 &priv->xstats, chan); 2039 struct stmmac_channel *ch = &priv->channel[chan]; 2040 bool needs_work = false; 2041 2042 if ((status & handle_rx) && ch->has_rx) { 2043 needs_work = true; 2044 } else { 2045 status &= ~handle_rx; 2046 } 2047 2048 if ((status & handle_tx) && ch->has_tx) { 2049 needs_work = true; 2050 } else { 2051 status &= ~handle_tx; 2052 } 2053 2054 if (needs_work && napi_schedule_prep(&ch->napi)) { 2055 stmmac_disable_dma_irq(priv, priv->ioaddr, chan); 2056 __napi_schedule(&ch->napi); 2057 } 2058 2059 return status; 2060 } 2061 2062 /** 2063 * stmmac_dma_interrupt - DMA ISR 2064 * @priv: driver private structure 2065 * Description: this is the DMA ISR. It is called by the main ISR. 2066 * It calls the dwmac dma routine and schedule poll method in case of some 2067 * work can be done. 2068 */ 2069 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2070 { 2071 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2072 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2073 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2074 tx_channel_count : rx_channel_count; 2075 u32 chan; 2076 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2077 2078 /* Make sure we never check beyond our status buffer. */ 2079 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2080 channels_to_check = ARRAY_SIZE(status); 2081 2082 for (chan = 0; chan < channels_to_check; chan++) 2083 status[chan] = stmmac_napi_check(priv, chan); 2084 2085 for (chan = 0; chan < tx_channel_count; chan++) { 2086 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2087 /* Try to bump up the dma threshold on this failure */ 2088 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && 2089 (tc <= 256)) { 2090 tc += 64; 2091 if (priv->plat->force_thresh_dma_mode) 2092 stmmac_set_dma_operation_mode(priv, 2093 tc, 2094 tc, 2095 chan); 2096 else 2097 stmmac_set_dma_operation_mode(priv, 2098 tc, 2099 SF_DMA_MODE, 2100 chan); 2101 priv->xstats.threshold = tc; 2102 } 2103 } else if (unlikely(status[chan] == tx_hard_error)) { 2104 stmmac_tx_err(priv, chan); 2105 } 2106 } 2107 } 2108 2109 /** 2110 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2111 * @priv: driver private structure 2112 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2113 */ 2114 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2115 { 2116 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2117 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2118 2119 dwmac_mmc_intr_all_mask(priv->mmcaddr); 2120 2121 if (priv->dma_cap.rmon) { 2122 dwmac_mmc_ctrl(priv->mmcaddr, mode); 2123 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2124 } else 2125 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2126 } 2127 2128 /** 2129 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2130 * @priv: driver private structure 2131 * Description: 2132 * new GMAC chip generations have a new register to indicate the 2133 * presence of the optional feature/functions. 2134 * This can be also used to override the value passed through the 2135 * platform and necessary for old MAC10/100 and GMAC chips. 2136 */ 2137 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2138 { 2139 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2140 } 2141 2142 /** 2143 * stmmac_check_ether_addr - check if the MAC addr is valid 2144 * @priv: driver private structure 2145 * Description: 2146 * it is to verify if the MAC address is valid, in case of failures it 2147 * generates a random MAC address 2148 */ 2149 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2150 { 2151 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2152 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0); 2153 if (!is_valid_ether_addr(priv->dev->dev_addr)) 2154 eth_hw_addr_random(priv->dev); 2155 netdev_info(priv->dev, "device MAC address %pM\n", 2156 priv->dev->dev_addr); 2157 } 2158 } 2159 2160 /** 2161 * stmmac_init_dma_engine - DMA init. 2162 * @priv: driver private structure 2163 * Description: 2164 * It inits the DMA invoking the specific MAC/GMAC callback. 2165 * Some DMA parameters can be passed from the platform; 2166 * in case of these are not passed a default is kept for the MAC or GMAC. 2167 */ 2168 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2169 { 2170 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2171 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2172 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2173 struct stmmac_rx_queue *rx_q; 2174 struct stmmac_tx_queue *tx_q; 2175 u32 chan = 0; 2176 int atds = 0; 2177 int ret = 0; 2178 2179 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2180 dev_err(priv->device, "Invalid DMA configuration\n"); 2181 return -EINVAL; 2182 } 2183 2184 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2185 atds = 1; 2186 2187 ret = stmmac_reset(priv, priv->ioaddr); 2188 if (ret) { 2189 dev_err(priv->device, "Failed to reset the dma\n"); 2190 return ret; 2191 } 2192 2193 /* DMA Configuration */ 2194 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); 2195 2196 if (priv->plat->axi) 2197 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 2198 2199 /* DMA RX Channel Configuration */ 2200 for (chan = 0; chan < rx_channels_count; chan++) { 2201 rx_q = &priv->rx_queue[chan]; 2202 2203 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2204 rx_q->dma_rx_phy, chan); 2205 2206 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2207 (DMA_RX_SIZE * sizeof(struct dma_desc)); 2208 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 2209 rx_q->rx_tail_addr, chan); 2210 } 2211 2212 /* DMA TX Channel Configuration */ 2213 for (chan = 0; chan < tx_channels_count; chan++) { 2214 tx_q = &priv->tx_queue[chan]; 2215 2216 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2217 tx_q->dma_tx_phy, chan); 2218 2219 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 2220 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 2221 tx_q->tx_tail_addr, chan); 2222 } 2223 2224 /* DMA CSR Channel configuration */ 2225 for (chan = 0; chan < dma_csr_ch; chan++) 2226 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 2227 2228 return ret; 2229 } 2230 2231 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 2232 { 2233 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2234 2235 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer)); 2236 } 2237 2238 /** 2239 * stmmac_tx_timer - mitigation sw timer for tx. 2240 * @data: data pointer 2241 * Description: 2242 * This is the timer handler to directly invoke the stmmac_tx_clean. 2243 */ 2244 static void stmmac_tx_timer(struct timer_list *t) 2245 { 2246 struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer); 2247 struct stmmac_priv *priv = tx_q->priv_data; 2248 struct stmmac_channel *ch; 2249 2250 ch = &priv->channel[tx_q->queue_index]; 2251 2252 if (likely(napi_schedule_prep(&ch->napi))) 2253 __napi_schedule(&ch->napi); 2254 } 2255 2256 /** 2257 * stmmac_init_tx_coalesce - init tx mitigation options. 2258 * @priv: driver private structure 2259 * Description: 2260 * This inits the transmit coalesce parameters: i.e. timer rate, 2261 * timer handler and default threshold used for enabling the 2262 * interrupt on completion bit. 2263 */ 2264 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv) 2265 { 2266 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2267 u32 chan; 2268 2269 priv->tx_coal_frames = STMMAC_TX_FRAMES; 2270 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER; 2271 2272 for (chan = 0; chan < tx_channel_count; chan++) { 2273 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 2274 2275 timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0); 2276 } 2277 } 2278 2279 static void stmmac_set_rings_length(struct stmmac_priv *priv) 2280 { 2281 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2282 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2283 u32 chan; 2284 2285 /* set TX ring length */ 2286 for (chan = 0; chan < tx_channels_count; chan++) 2287 stmmac_set_tx_ring_len(priv, priv->ioaddr, 2288 (DMA_TX_SIZE - 1), chan); 2289 2290 /* set RX ring length */ 2291 for (chan = 0; chan < rx_channels_count; chan++) 2292 stmmac_set_rx_ring_len(priv, priv->ioaddr, 2293 (DMA_RX_SIZE - 1), chan); 2294 } 2295 2296 /** 2297 * stmmac_set_tx_queue_weight - Set TX queue weight 2298 * @priv: driver private structure 2299 * Description: It is used for setting TX queues weight 2300 */ 2301 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 2302 { 2303 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2304 u32 weight; 2305 u32 queue; 2306 2307 for (queue = 0; queue < tx_queues_count; queue++) { 2308 weight = priv->plat->tx_queues_cfg[queue].weight; 2309 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 2310 } 2311 } 2312 2313 /** 2314 * stmmac_configure_cbs - Configure CBS in TX queue 2315 * @priv: driver private structure 2316 * Description: It is used for configuring CBS in AVB TX queues 2317 */ 2318 static void stmmac_configure_cbs(struct stmmac_priv *priv) 2319 { 2320 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2321 u32 mode_to_use; 2322 u32 queue; 2323 2324 /* queue 0 is reserved for legacy traffic */ 2325 for (queue = 1; queue < tx_queues_count; queue++) { 2326 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 2327 if (mode_to_use == MTL_QUEUE_DCB) 2328 continue; 2329 2330 stmmac_config_cbs(priv, priv->hw, 2331 priv->plat->tx_queues_cfg[queue].send_slope, 2332 priv->plat->tx_queues_cfg[queue].idle_slope, 2333 priv->plat->tx_queues_cfg[queue].high_credit, 2334 priv->plat->tx_queues_cfg[queue].low_credit, 2335 queue); 2336 } 2337 } 2338 2339 /** 2340 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 2341 * @priv: driver private structure 2342 * Description: It is used for mapping RX queues to RX dma channels 2343 */ 2344 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 2345 { 2346 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2347 u32 queue; 2348 u32 chan; 2349 2350 for (queue = 0; queue < rx_queues_count; queue++) { 2351 chan = priv->plat->rx_queues_cfg[queue].chan; 2352 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 2353 } 2354 } 2355 2356 /** 2357 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 2358 * @priv: driver private structure 2359 * Description: It is used for configuring the RX Queue Priority 2360 */ 2361 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 2362 { 2363 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2364 u32 queue; 2365 u32 prio; 2366 2367 for (queue = 0; queue < rx_queues_count; queue++) { 2368 if (!priv->plat->rx_queues_cfg[queue].use_prio) 2369 continue; 2370 2371 prio = priv->plat->rx_queues_cfg[queue].prio; 2372 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 2373 } 2374 } 2375 2376 /** 2377 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 2378 * @priv: driver private structure 2379 * Description: It is used for configuring the TX Queue Priority 2380 */ 2381 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 2382 { 2383 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2384 u32 queue; 2385 u32 prio; 2386 2387 for (queue = 0; queue < tx_queues_count; queue++) { 2388 if (!priv->plat->tx_queues_cfg[queue].use_prio) 2389 continue; 2390 2391 prio = priv->plat->tx_queues_cfg[queue].prio; 2392 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 2393 } 2394 } 2395 2396 /** 2397 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 2398 * @priv: driver private structure 2399 * Description: It is used for configuring the RX queue routing 2400 */ 2401 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 2402 { 2403 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2404 u32 queue; 2405 u8 packet; 2406 2407 for (queue = 0; queue < rx_queues_count; queue++) { 2408 /* no specific packet type routing specified for the queue */ 2409 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 2410 continue; 2411 2412 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 2413 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 2414 } 2415 } 2416 2417 /** 2418 * stmmac_mtl_configuration - Configure MTL 2419 * @priv: driver private structure 2420 * Description: It is used for configurring MTL 2421 */ 2422 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 2423 { 2424 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2425 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2426 2427 if (tx_queues_count > 1) 2428 stmmac_set_tx_queue_weight(priv); 2429 2430 /* Configure MTL RX algorithms */ 2431 if (rx_queues_count > 1) 2432 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 2433 priv->plat->rx_sched_algorithm); 2434 2435 /* Configure MTL TX algorithms */ 2436 if (tx_queues_count > 1) 2437 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 2438 priv->plat->tx_sched_algorithm); 2439 2440 /* Configure CBS in AVB TX queues */ 2441 if (tx_queues_count > 1) 2442 stmmac_configure_cbs(priv); 2443 2444 /* Map RX MTL to DMA channels */ 2445 stmmac_rx_queue_dma_chan_map(priv); 2446 2447 /* Enable MAC RX Queues */ 2448 stmmac_mac_enable_rx_queues(priv); 2449 2450 /* Set RX priorities */ 2451 if (rx_queues_count > 1) 2452 stmmac_mac_config_rx_queues_prio(priv); 2453 2454 /* Set TX priorities */ 2455 if (tx_queues_count > 1) 2456 stmmac_mac_config_tx_queues_prio(priv); 2457 2458 /* Set RX routing */ 2459 if (rx_queues_count > 1) 2460 stmmac_mac_config_rx_queues_routing(priv); 2461 } 2462 2463 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 2464 { 2465 if (priv->dma_cap.asp) { 2466 netdev_info(priv->dev, "Enabling Safety Features\n"); 2467 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp); 2468 } else { 2469 netdev_info(priv->dev, "No Safety Features support found\n"); 2470 } 2471 } 2472 2473 /** 2474 * stmmac_hw_setup - setup mac in a usable state. 2475 * @dev : pointer to the device structure. 2476 * Description: 2477 * this is the main function to setup the HW in a usable state because the 2478 * dma engine is reset, the core registers are configured (e.g. AXI, 2479 * Checksum features, timers). The DMA is ready to start receiving and 2480 * transmitting. 2481 * Return value: 2482 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2483 * file on failure. 2484 */ 2485 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) 2486 { 2487 struct stmmac_priv *priv = netdev_priv(dev); 2488 u32 rx_cnt = priv->plat->rx_queues_to_use; 2489 u32 tx_cnt = priv->plat->tx_queues_to_use; 2490 u32 chan; 2491 int ret; 2492 2493 /* DMA initialization and SW reset */ 2494 ret = stmmac_init_dma_engine(priv); 2495 if (ret < 0) { 2496 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 2497 __func__); 2498 return ret; 2499 } 2500 2501 /* Copy the MAC addr into the HW */ 2502 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 2503 2504 /* PS and related bits will be programmed according to the speed */ 2505 if (priv->hw->pcs) { 2506 int speed = priv->plat->mac_port_sel_speed; 2507 2508 if ((speed == SPEED_10) || (speed == SPEED_100) || 2509 (speed == SPEED_1000)) { 2510 priv->hw->ps = speed; 2511 } else { 2512 dev_warn(priv->device, "invalid port speed\n"); 2513 priv->hw->ps = 0; 2514 } 2515 } 2516 2517 /* Initialize the MAC Core */ 2518 stmmac_core_init(priv, priv->hw, dev); 2519 2520 /* Initialize MTL*/ 2521 stmmac_mtl_configuration(priv); 2522 2523 /* Initialize Safety Features */ 2524 stmmac_safety_feat_configuration(priv); 2525 2526 ret = stmmac_rx_ipc(priv, priv->hw); 2527 if (!ret) { 2528 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 2529 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 2530 priv->hw->rx_csum = 0; 2531 } 2532 2533 /* Enable the MAC Rx/Tx */ 2534 stmmac_mac_set(priv, priv->ioaddr, true); 2535 2536 /* Set the HW DMA mode and the COE */ 2537 stmmac_dma_operation_mode(priv); 2538 2539 stmmac_mmc_setup(priv); 2540 2541 if (init_ptp) { 2542 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 2543 if (ret < 0) 2544 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); 2545 2546 ret = stmmac_init_ptp(priv); 2547 if (ret == -EOPNOTSUPP) 2548 netdev_warn(priv->dev, "PTP not supported by HW\n"); 2549 else if (ret) 2550 netdev_warn(priv->dev, "PTP init failed\n"); 2551 } 2552 2553 priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; 2554 2555 if (priv->use_riwt) { 2556 ret = stmmac_rx_watchdog(priv, priv->ioaddr, MAX_DMA_RIWT, rx_cnt); 2557 if (!ret) 2558 priv->rx_riwt = MAX_DMA_RIWT; 2559 } 2560 2561 if (priv->hw->pcs) 2562 stmmac_pcs_ctrl_ane(priv, priv->hw, 1, priv->hw->ps, 0); 2563 2564 /* set TX and RX rings length */ 2565 stmmac_set_rings_length(priv); 2566 2567 /* Enable TSO */ 2568 if (priv->tso) { 2569 for (chan = 0; chan < tx_cnt; chan++) 2570 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 2571 } 2572 2573 /* Start the ball rolling... */ 2574 stmmac_start_all_dma(priv); 2575 2576 return 0; 2577 } 2578 2579 static void stmmac_hw_teardown(struct net_device *dev) 2580 { 2581 struct stmmac_priv *priv = netdev_priv(dev); 2582 2583 clk_disable_unprepare(priv->plat->clk_ptp_ref); 2584 } 2585 2586 /** 2587 * stmmac_open - open entry point of the driver 2588 * @dev : pointer to the device structure. 2589 * Description: 2590 * This function is the open entry point of the driver. 2591 * Return value: 2592 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2593 * file on failure. 2594 */ 2595 static int stmmac_open(struct net_device *dev) 2596 { 2597 struct stmmac_priv *priv = netdev_priv(dev); 2598 u32 chan; 2599 int ret; 2600 2601 stmmac_check_ether_addr(priv); 2602 2603 if (priv->hw->pcs != STMMAC_PCS_RGMII && 2604 priv->hw->pcs != STMMAC_PCS_TBI && 2605 priv->hw->pcs != STMMAC_PCS_RTBI) { 2606 ret = stmmac_init_phy(dev); 2607 if (ret) { 2608 netdev_err(priv->dev, 2609 "%s: Cannot attach to PHY (error: %d)\n", 2610 __func__, ret); 2611 return ret; 2612 } 2613 } 2614 2615 /* Extra statistics */ 2616 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); 2617 priv->xstats.threshold = tc; 2618 2619 priv->dma_buf_sz = STMMAC_ALIGN(buf_sz); 2620 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 2621 2622 ret = alloc_dma_desc_resources(priv); 2623 if (ret < 0) { 2624 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 2625 __func__); 2626 goto dma_desc_error; 2627 } 2628 2629 ret = init_dma_desc_rings(dev, GFP_KERNEL); 2630 if (ret < 0) { 2631 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 2632 __func__); 2633 goto init_error; 2634 } 2635 2636 ret = stmmac_hw_setup(dev, true); 2637 if (ret < 0) { 2638 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 2639 goto init_error; 2640 } 2641 2642 stmmac_init_tx_coalesce(priv); 2643 2644 if (dev->phydev) 2645 phy_start(dev->phydev); 2646 2647 /* Request the IRQ lines */ 2648 ret = request_irq(dev->irq, stmmac_interrupt, 2649 IRQF_SHARED, dev->name, dev); 2650 if (unlikely(ret < 0)) { 2651 netdev_err(priv->dev, 2652 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 2653 __func__, dev->irq, ret); 2654 goto irq_error; 2655 } 2656 2657 /* Request the Wake IRQ in case of another line is used for WoL */ 2658 if (priv->wol_irq != dev->irq) { 2659 ret = request_irq(priv->wol_irq, stmmac_interrupt, 2660 IRQF_SHARED, dev->name, dev); 2661 if (unlikely(ret < 0)) { 2662 netdev_err(priv->dev, 2663 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 2664 __func__, priv->wol_irq, ret); 2665 goto wolirq_error; 2666 } 2667 } 2668 2669 /* Request the IRQ lines */ 2670 if (priv->lpi_irq > 0) { 2671 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED, 2672 dev->name, dev); 2673 if (unlikely(ret < 0)) { 2674 netdev_err(priv->dev, 2675 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 2676 __func__, priv->lpi_irq, ret); 2677 goto lpiirq_error; 2678 } 2679 } 2680 2681 stmmac_enable_all_queues(priv); 2682 stmmac_start_all_queues(priv); 2683 2684 return 0; 2685 2686 lpiirq_error: 2687 if (priv->wol_irq != dev->irq) 2688 free_irq(priv->wol_irq, dev); 2689 wolirq_error: 2690 free_irq(dev->irq, dev); 2691 irq_error: 2692 if (dev->phydev) 2693 phy_stop(dev->phydev); 2694 2695 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 2696 del_timer_sync(&priv->tx_queue[chan].txtimer); 2697 2698 stmmac_hw_teardown(dev); 2699 init_error: 2700 free_dma_desc_resources(priv); 2701 dma_desc_error: 2702 if (dev->phydev) 2703 phy_disconnect(dev->phydev); 2704 2705 return ret; 2706 } 2707 2708 /** 2709 * stmmac_release - close entry point of the driver 2710 * @dev : device pointer. 2711 * Description: 2712 * This is the stop entry point of the driver. 2713 */ 2714 static int stmmac_release(struct net_device *dev) 2715 { 2716 struct stmmac_priv *priv = netdev_priv(dev); 2717 u32 chan; 2718 2719 if (priv->eee_enabled) 2720 del_timer_sync(&priv->eee_ctrl_timer); 2721 2722 /* Stop and disconnect the PHY */ 2723 if (dev->phydev) { 2724 phy_stop(dev->phydev); 2725 phy_disconnect(dev->phydev); 2726 } 2727 2728 stmmac_stop_all_queues(priv); 2729 2730 stmmac_disable_all_queues(priv); 2731 2732 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 2733 del_timer_sync(&priv->tx_queue[chan].txtimer); 2734 2735 /* Free the IRQ lines */ 2736 free_irq(dev->irq, dev); 2737 if (priv->wol_irq != dev->irq) 2738 free_irq(priv->wol_irq, dev); 2739 if (priv->lpi_irq > 0) 2740 free_irq(priv->lpi_irq, dev); 2741 2742 /* Stop TX/RX DMA and clear the descriptors */ 2743 stmmac_stop_all_dma(priv); 2744 2745 /* Release and free the Rx/Tx resources */ 2746 free_dma_desc_resources(priv); 2747 2748 /* Disable the MAC Rx/Tx */ 2749 stmmac_mac_set(priv, priv->ioaddr, false); 2750 2751 netif_carrier_off(dev); 2752 2753 stmmac_release_ptp(priv); 2754 2755 return 0; 2756 } 2757 2758 /** 2759 * stmmac_tso_allocator - close entry point of the driver 2760 * @priv: driver private structure 2761 * @des: buffer start address 2762 * @total_len: total length to fill in descriptors 2763 * @last_segmant: condition for the last descriptor 2764 * @queue: TX queue index 2765 * Description: 2766 * This function fills descriptor and request new descriptors according to 2767 * buffer length to fill 2768 */ 2769 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des, 2770 int total_len, bool last_segment, u32 queue) 2771 { 2772 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2773 struct dma_desc *desc; 2774 u32 buff_size; 2775 int tmp_len; 2776 2777 tmp_len = total_len; 2778 2779 while (tmp_len > 0) { 2780 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2781 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 2782 desc = tx_q->dma_tx + tx_q->cur_tx; 2783 2784 desc->des0 = cpu_to_le32(des + (total_len - tmp_len)); 2785 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 2786 TSO_MAX_BUFF_SIZE : tmp_len; 2787 2788 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 2789 0, 1, 2790 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 2791 0, 0); 2792 2793 tmp_len -= TSO_MAX_BUFF_SIZE; 2794 } 2795 } 2796 2797 /** 2798 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 2799 * @skb : the socket buffer 2800 * @dev : device pointer 2801 * Description: this is the transmit function that is called on TSO frames 2802 * (support available on GMAC4 and newer chips). 2803 * Diagram below show the ring programming in case of TSO frames: 2804 * 2805 * First Descriptor 2806 * -------- 2807 * | DES0 |---> buffer1 = L2/L3/L4 header 2808 * | DES1 |---> TCP Payload (can continue on next descr...) 2809 * | DES2 |---> buffer 1 and 2 len 2810 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 2811 * -------- 2812 * | 2813 * ... 2814 * | 2815 * -------- 2816 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 2817 * | DES1 | --| 2818 * | DES2 | --> buffer 1 and 2 len 2819 * | DES3 | 2820 * -------- 2821 * 2822 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 2823 */ 2824 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 2825 { 2826 struct dma_desc *desc, *first, *mss_desc = NULL; 2827 struct stmmac_priv *priv = netdev_priv(dev); 2828 int nfrags = skb_shinfo(skb)->nr_frags; 2829 u32 queue = skb_get_queue_mapping(skb); 2830 unsigned int first_entry, des; 2831 struct stmmac_tx_queue *tx_q; 2832 int tmp_pay_len = 0; 2833 u32 pay_len, mss; 2834 u8 proto_hdr_len; 2835 int i; 2836 2837 tx_q = &priv->tx_queue[queue]; 2838 2839 /* Compute header lengths */ 2840 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2841 2842 /* Desc availability based on threshold should be enough safe */ 2843 if (unlikely(stmmac_tx_avail(priv, queue) < 2844 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 2845 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 2846 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 2847 queue)); 2848 /* This is a hard error, log it. */ 2849 netdev_err(priv->dev, 2850 "%s: Tx Ring full when queue awake\n", 2851 __func__); 2852 } 2853 return NETDEV_TX_BUSY; 2854 } 2855 2856 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 2857 2858 mss = skb_shinfo(skb)->gso_size; 2859 2860 /* set new MSS value if needed */ 2861 if (mss != tx_q->mss) { 2862 mss_desc = tx_q->dma_tx + tx_q->cur_tx; 2863 stmmac_set_mss(priv, mss_desc, mss); 2864 tx_q->mss = mss; 2865 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2866 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 2867 } 2868 2869 if (netif_msg_tx_queued(priv)) { 2870 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 2871 __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss); 2872 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 2873 skb->data_len); 2874 } 2875 2876 first_entry = tx_q->cur_tx; 2877 WARN_ON(tx_q->tx_skbuff[first_entry]); 2878 2879 desc = tx_q->dma_tx + first_entry; 2880 first = desc; 2881 2882 /* first descriptor: fill Headers on Buf1 */ 2883 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 2884 DMA_TO_DEVICE); 2885 if (dma_mapping_error(priv->device, des)) 2886 goto dma_map_err; 2887 2888 tx_q->tx_skbuff_dma[first_entry].buf = des; 2889 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 2890 2891 first->des0 = cpu_to_le32(des); 2892 2893 /* Fill start of payload in buff2 of first descriptor */ 2894 if (pay_len) 2895 first->des1 = cpu_to_le32(des + proto_hdr_len); 2896 2897 /* If needed take extra descriptors to fill the remaining payload */ 2898 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 2899 2900 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 2901 2902 /* Prepare fragments */ 2903 for (i = 0; i < nfrags; i++) { 2904 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2905 2906 des = skb_frag_dma_map(priv->device, frag, 0, 2907 skb_frag_size(frag), 2908 DMA_TO_DEVICE); 2909 if (dma_mapping_error(priv->device, des)) 2910 goto dma_map_err; 2911 2912 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 2913 (i == nfrags - 1), queue); 2914 2915 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 2916 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 2917 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 2918 } 2919 2920 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 2921 2922 /* Only the last descriptor gets to point to the skb. */ 2923 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 2924 2925 /* We've used all descriptors we need for this skb, however, 2926 * advance cur_tx so that it references a fresh descriptor. 2927 * ndo_start_xmit will fill this descriptor the next time it's 2928 * called and stmmac_tx_clean may clean up to this descriptor. 2929 */ 2930 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2931 2932 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 2933 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 2934 __func__); 2935 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 2936 } 2937 2938 dev->stats.tx_bytes += skb->len; 2939 priv->xstats.tx_tso_frames++; 2940 priv->xstats.tx_tso_nfrags += nfrags; 2941 2942 /* Manage tx mitigation */ 2943 tx_q->tx_count_frames += nfrags + 1; 2944 if (priv->tx_coal_frames <= tx_q->tx_count_frames) { 2945 stmmac_set_tx_ic(priv, desc); 2946 priv->xstats.tx_set_ic_bit++; 2947 tx_q->tx_count_frames = 0; 2948 } else { 2949 stmmac_tx_timer_arm(priv, queue); 2950 } 2951 2952 skb_tx_timestamp(skb); 2953 2954 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 2955 priv->hwts_tx_en)) { 2956 /* declare that device is doing timestamping */ 2957 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2958 stmmac_enable_tx_timestamp(priv, first); 2959 } 2960 2961 /* Complete the first descriptor before granting the DMA */ 2962 stmmac_prepare_tso_tx_desc(priv, first, 1, 2963 proto_hdr_len, 2964 pay_len, 2965 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 2966 tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); 2967 2968 /* If context desc is used to change MSS */ 2969 if (mss_desc) { 2970 /* Make sure that first descriptor has been completely 2971 * written, including its own bit. This is because MSS is 2972 * actually before first descriptor, so we need to make 2973 * sure that MSS's own bit is the last thing written. 2974 */ 2975 dma_wmb(); 2976 stmmac_set_tx_owner(priv, mss_desc); 2977 } 2978 2979 /* The own bit must be the latest setting done when prepare the 2980 * descriptor and then barrier is needed to make sure that 2981 * all is coherent before granting the DMA engine. 2982 */ 2983 wmb(); 2984 2985 if (netif_msg_pktdata(priv)) { 2986 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 2987 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 2988 tx_q->cur_tx, first, nfrags); 2989 2990 stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0); 2991 2992 pr_info(">>> frame to be transmitted: "); 2993 print_pkt(skb->data, skb_headlen(skb)); 2994 } 2995 2996 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 2997 2998 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc)); 2999 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 3000 3001 return NETDEV_TX_OK; 3002 3003 dma_map_err: 3004 dev_err(priv->device, "Tx dma map failed\n"); 3005 dev_kfree_skb(skb); 3006 priv->dev->stats.tx_dropped++; 3007 return NETDEV_TX_OK; 3008 } 3009 3010 /** 3011 * stmmac_xmit - Tx entry point of the driver 3012 * @skb : the socket buffer 3013 * @dev : device pointer 3014 * Description : this is the tx entry point of the driver. 3015 * It programs the chain or the ring and supports oversized frames 3016 * and SG feature. 3017 */ 3018 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 3019 { 3020 struct stmmac_priv *priv = netdev_priv(dev); 3021 unsigned int nopaged_len = skb_headlen(skb); 3022 int i, csum_insertion = 0, is_jumbo = 0; 3023 u32 queue = skb_get_queue_mapping(skb); 3024 int nfrags = skb_shinfo(skb)->nr_frags; 3025 int entry; 3026 unsigned int first_entry; 3027 struct dma_desc *desc, *first; 3028 struct stmmac_tx_queue *tx_q; 3029 unsigned int enh_desc; 3030 unsigned int des; 3031 3032 tx_q = &priv->tx_queue[queue]; 3033 3034 /* Manage oversized TCP frames for GMAC4 device */ 3035 if (skb_is_gso(skb) && priv->tso) { 3036 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 3037 return stmmac_tso_xmit(skb, dev); 3038 } 3039 3040 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 3041 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 3042 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 3043 queue)); 3044 /* This is a hard error, log it. */ 3045 netdev_err(priv->dev, 3046 "%s: Tx Ring full when queue awake\n", 3047 __func__); 3048 } 3049 return NETDEV_TX_BUSY; 3050 } 3051 3052 if (priv->tx_path_in_lpi_mode) 3053 stmmac_disable_eee_mode(priv); 3054 3055 entry = tx_q->cur_tx; 3056 first_entry = entry; 3057 WARN_ON(tx_q->tx_skbuff[first_entry]); 3058 3059 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 3060 3061 if (likely(priv->extend_desc)) 3062 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3063 else 3064 desc = tx_q->dma_tx + entry; 3065 3066 first = desc; 3067 3068 enh_desc = priv->plat->enh_desc; 3069 /* To program the descriptors according to the size of the frame */ 3070 if (enh_desc) 3071 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 3072 3073 if (unlikely(is_jumbo)) { 3074 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 3075 if (unlikely(entry < 0) && (entry != -EINVAL)) 3076 goto dma_map_err; 3077 } 3078 3079 for (i = 0; i < nfrags; i++) { 3080 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3081 int len = skb_frag_size(frag); 3082 bool last_segment = (i == (nfrags - 1)); 3083 3084 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3085 WARN_ON(tx_q->tx_skbuff[entry]); 3086 3087 if (likely(priv->extend_desc)) 3088 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3089 else 3090 desc = tx_q->dma_tx + entry; 3091 3092 des = skb_frag_dma_map(priv->device, frag, 0, len, 3093 DMA_TO_DEVICE); 3094 if (dma_mapping_error(priv->device, des)) 3095 goto dma_map_err; /* should reuse desc w/o issues */ 3096 3097 tx_q->tx_skbuff_dma[entry].buf = des; 3098 3099 stmmac_set_desc_addr(priv, desc, des); 3100 3101 tx_q->tx_skbuff_dma[entry].map_as_page = true; 3102 tx_q->tx_skbuff_dma[entry].len = len; 3103 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 3104 3105 /* Prepare the descriptor and set the own bit too */ 3106 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 3107 priv->mode, 1, last_segment, skb->len); 3108 } 3109 3110 /* Only the last descriptor gets to point to the skb. */ 3111 tx_q->tx_skbuff[entry] = skb; 3112 3113 /* We've used all descriptors we need for this skb, however, 3114 * advance cur_tx so that it references a fresh descriptor. 3115 * ndo_start_xmit will fill this descriptor the next time it's 3116 * called and stmmac_tx_clean may clean up to this descriptor. 3117 */ 3118 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3119 tx_q->cur_tx = entry; 3120 3121 if (netif_msg_pktdata(priv)) { 3122 void *tx_head; 3123 3124 netdev_dbg(priv->dev, 3125 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 3126 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 3127 entry, first, nfrags); 3128 3129 if (priv->extend_desc) 3130 tx_head = (void *)tx_q->dma_etx; 3131 else 3132 tx_head = (void *)tx_q->dma_tx; 3133 3134 stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false); 3135 3136 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 3137 print_pkt(skb->data, skb->len); 3138 } 3139 3140 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 3141 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 3142 __func__); 3143 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 3144 } 3145 3146 dev->stats.tx_bytes += skb->len; 3147 3148 /* According to the coalesce parameter the IC bit for the latest 3149 * segment is reset and the timer re-started to clean the tx status. 3150 * This approach takes care about the fragments: desc is the first 3151 * element in case of no SG. 3152 */ 3153 tx_q->tx_count_frames += nfrags + 1; 3154 if (priv->tx_coal_frames <= tx_q->tx_count_frames) { 3155 stmmac_set_tx_ic(priv, desc); 3156 priv->xstats.tx_set_ic_bit++; 3157 tx_q->tx_count_frames = 0; 3158 } else { 3159 stmmac_tx_timer_arm(priv, queue); 3160 } 3161 3162 skb_tx_timestamp(skb); 3163 3164 /* Ready to fill the first descriptor and set the OWN bit w/o any 3165 * problems because all the descriptors are actually ready to be 3166 * passed to the DMA engine. 3167 */ 3168 if (likely(!is_jumbo)) { 3169 bool last_segment = (nfrags == 0); 3170 3171 des = dma_map_single(priv->device, skb->data, 3172 nopaged_len, DMA_TO_DEVICE); 3173 if (dma_mapping_error(priv->device, des)) 3174 goto dma_map_err; 3175 3176 tx_q->tx_skbuff_dma[first_entry].buf = des; 3177 3178 stmmac_set_desc_addr(priv, first, des); 3179 3180 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 3181 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 3182 3183 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 3184 priv->hwts_tx_en)) { 3185 /* declare that device is doing timestamping */ 3186 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3187 stmmac_enable_tx_timestamp(priv, first); 3188 } 3189 3190 /* Prepare the first descriptor setting the OWN bit too */ 3191 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 3192 csum_insertion, priv->mode, 1, last_segment, 3193 skb->len); 3194 3195 /* The own bit must be the latest setting done when prepare the 3196 * descriptor and then barrier is needed to make sure that 3197 * all is coherent before granting the DMA engine. 3198 */ 3199 wmb(); 3200 } 3201 3202 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 3203 3204 stmmac_enable_dma_transmission(priv, priv->ioaddr); 3205 3206 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc)); 3207 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 3208 3209 return NETDEV_TX_OK; 3210 3211 dma_map_err: 3212 netdev_err(priv->dev, "Tx DMA map failed\n"); 3213 dev_kfree_skb(skb); 3214 priv->dev->stats.tx_dropped++; 3215 return NETDEV_TX_OK; 3216 } 3217 3218 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 3219 { 3220 struct vlan_ethhdr *veth; 3221 __be16 vlan_proto; 3222 u16 vlanid; 3223 3224 veth = (struct vlan_ethhdr *)skb->data; 3225 vlan_proto = veth->h_vlan_proto; 3226 3227 if ((vlan_proto == htons(ETH_P_8021Q) && 3228 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 3229 (vlan_proto == htons(ETH_P_8021AD) && 3230 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 3231 /* pop the vlan tag */ 3232 vlanid = ntohs(veth->h_vlan_TCI); 3233 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 3234 skb_pull(skb, VLAN_HLEN); 3235 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 3236 } 3237 } 3238 3239 3240 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q) 3241 { 3242 if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH) 3243 return 0; 3244 3245 return 1; 3246 } 3247 3248 /** 3249 * stmmac_rx_refill - refill used skb preallocated buffers 3250 * @priv: driver private structure 3251 * @queue: RX queue index 3252 * Description : this is to reallocate the skb for the reception process 3253 * that is based on zero-copy. 3254 */ 3255 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 3256 { 3257 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3258 int dirty = stmmac_rx_dirty(priv, queue); 3259 unsigned int entry = rx_q->dirty_rx; 3260 3261 int bfsize = priv->dma_buf_sz; 3262 3263 while (dirty-- > 0) { 3264 struct dma_desc *p; 3265 3266 if (priv->extend_desc) 3267 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3268 else 3269 p = rx_q->dma_rx + entry; 3270 3271 if (likely(!rx_q->rx_skbuff[entry])) { 3272 struct sk_buff *skb; 3273 3274 skb = netdev_alloc_skb_ip_align(priv->dev, bfsize); 3275 if (unlikely(!skb)) { 3276 /* so for a while no zero-copy! */ 3277 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH; 3278 if (unlikely(net_ratelimit())) 3279 dev_err(priv->device, 3280 "fail to alloc skb entry %d\n", 3281 entry); 3282 break; 3283 } 3284 3285 rx_q->rx_skbuff[entry] = skb; 3286 rx_q->rx_skbuff_dma[entry] = 3287 dma_map_single(priv->device, skb->data, bfsize, 3288 DMA_FROM_DEVICE); 3289 if (dma_mapping_error(priv->device, 3290 rx_q->rx_skbuff_dma[entry])) { 3291 netdev_err(priv->dev, "Rx DMA map failed\n"); 3292 dev_kfree_skb(skb); 3293 break; 3294 } 3295 3296 stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[entry]); 3297 stmmac_refill_desc3(priv, rx_q, p); 3298 3299 if (rx_q->rx_zeroc_thresh > 0) 3300 rx_q->rx_zeroc_thresh--; 3301 3302 netif_dbg(priv, rx_status, priv->dev, 3303 "refill entry #%d\n", entry); 3304 } 3305 dma_wmb(); 3306 3307 stmmac_set_rx_owner(priv, p, priv->use_riwt); 3308 3309 dma_wmb(); 3310 3311 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE); 3312 } 3313 rx_q->dirty_rx = entry; 3314 } 3315 3316 /** 3317 * stmmac_rx - manage the receive process 3318 * @priv: driver private structure 3319 * @limit: napi bugget 3320 * @queue: RX queue index. 3321 * Description : this the function called by the napi poll method. 3322 * It gets all the frames inside the ring. 3323 */ 3324 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 3325 { 3326 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3327 struct stmmac_channel *ch = &priv->channel[queue]; 3328 unsigned int entry = rx_q->cur_rx; 3329 int coe = priv->hw->rx_csum; 3330 unsigned int next_entry; 3331 unsigned int count = 0; 3332 bool xmac; 3333 3334 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 3335 3336 if (netif_msg_rx_status(priv)) { 3337 void *rx_head; 3338 3339 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 3340 if (priv->extend_desc) 3341 rx_head = (void *)rx_q->dma_erx; 3342 else 3343 rx_head = (void *)rx_q->dma_rx; 3344 3345 stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true); 3346 } 3347 while (count < limit) { 3348 int status; 3349 struct dma_desc *p; 3350 struct dma_desc *np; 3351 3352 if (priv->extend_desc) 3353 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3354 else 3355 p = rx_q->dma_rx + entry; 3356 3357 /* read the status of the incoming frame */ 3358 status = stmmac_rx_status(priv, &priv->dev->stats, 3359 &priv->xstats, p); 3360 /* check if managed by the DMA otherwise go ahead */ 3361 if (unlikely(status & dma_own)) 3362 break; 3363 3364 count++; 3365 3366 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE); 3367 next_entry = rx_q->cur_rx; 3368 3369 if (priv->extend_desc) 3370 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 3371 else 3372 np = rx_q->dma_rx + next_entry; 3373 3374 prefetch(np); 3375 3376 if (priv->extend_desc) 3377 stmmac_rx_extended_status(priv, &priv->dev->stats, 3378 &priv->xstats, rx_q->dma_erx + entry); 3379 if (unlikely(status == discard_frame)) { 3380 priv->dev->stats.rx_errors++; 3381 if (priv->hwts_rx_en && !priv->extend_desc) { 3382 /* DESC2 & DESC3 will be overwritten by device 3383 * with timestamp value, hence reinitialize 3384 * them in stmmac_rx_refill() function so that 3385 * device can reuse it. 3386 */ 3387 dev_kfree_skb_any(rx_q->rx_skbuff[entry]); 3388 rx_q->rx_skbuff[entry] = NULL; 3389 dma_unmap_single(priv->device, 3390 rx_q->rx_skbuff_dma[entry], 3391 priv->dma_buf_sz, 3392 DMA_FROM_DEVICE); 3393 } 3394 } else { 3395 struct sk_buff *skb; 3396 int frame_len; 3397 unsigned int des; 3398 3399 stmmac_get_desc_addr(priv, p, &des); 3400 frame_len = stmmac_get_rx_frame_len(priv, p, coe); 3401 3402 /* If frame length is greater than skb buffer size 3403 * (preallocated during init) then the packet is 3404 * ignored 3405 */ 3406 if (frame_len > priv->dma_buf_sz) { 3407 netdev_err(priv->dev, 3408 "len %d larger than size (%d)\n", 3409 frame_len, priv->dma_buf_sz); 3410 priv->dev->stats.rx_length_errors++; 3411 break; 3412 } 3413 3414 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 3415 * Type frames (LLC/LLC-SNAP) 3416 * 3417 * llc_snap is never checked in GMAC >= 4, so this ACS 3418 * feature is always disabled and packets need to be 3419 * stripped manually. 3420 */ 3421 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00) || 3422 unlikely(status != llc_snap)) 3423 frame_len -= ETH_FCS_LEN; 3424 3425 if (netif_msg_rx_status(priv)) { 3426 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n", 3427 p, entry, des); 3428 netdev_dbg(priv->dev, "frame size %d, COE: %d\n", 3429 frame_len, status); 3430 } 3431 3432 /* The zero-copy is always used for all the sizes 3433 * in case of GMAC4 because it needs 3434 * to refill the used descriptors, always. 3435 */ 3436 if (unlikely(!xmac && 3437 ((frame_len < priv->rx_copybreak) || 3438 stmmac_rx_threshold_count(rx_q)))) { 3439 skb = netdev_alloc_skb_ip_align(priv->dev, 3440 frame_len); 3441 if (unlikely(!skb)) { 3442 if (net_ratelimit()) 3443 dev_warn(priv->device, 3444 "packet dropped\n"); 3445 priv->dev->stats.rx_dropped++; 3446 break; 3447 } 3448 3449 dma_sync_single_for_cpu(priv->device, 3450 rx_q->rx_skbuff_dma 3451 [entry], frame_len, 3452 DMA_FROM_DEVICE); 3453 skb_copy_to_linear_data(skb, 3454 rx_q-> 3455 rx_skbuff[entry]->data, 3456 frame_len); 3457 3458 skb_put(skb, frame_len); 3459 dma_sync_single_for_device(priv->device, 3460 rx_q->rx_skbuff_dma 3461 [entry], frame_len, 3462 DMA_FROM_DEVICE); 3463 } else { 3464 skb = rx_q->rx_skbuff[entry]; 3465 if (unlikely(!skb)) { 3466 netdev_err(priv->dev, 3467 "%s: Inconsistent Rx chain\n", 3468 priv->dev->name); 3469 priv->dev->stats.rx_dropped++; 3470 break; 3471 } 3472 prefetch(skb->data - NET_IP_ALIGN); 3473 rx_q->rx_skbuff[entry] = NULL; 3474 rx_q->rx_zeroc_thresh++; 3475 3476 skb_put(skb, frame_len); 3477 dma_unmap_single(priv->device, 3478 rx_q->rx_skbuff_dma[entry], 3479 priv->dma_buf_sz, 3480 DMA_FROM_DEVICE); 3481 } 3482 3483 if (netif_msg_pktdata(priv)) { 3484 netdev_dbg(priv->dev, "frame received (%dbytes)", 3485 frame_len); 3486 print_pkt(skb->data, frame_len); 3487 } 3488 3489 stmmac_get_rx_hwtstamp(priv, p, np, skb); 3490 3491 stmmac_rx_vlan(priv->dev, skb); 3492 3493 skb->protocol = eth_type_trans(skb, priv->dev); 3494 3495 if (unlikely(!coe)) 3496 skb_checksum_none_assert(skb); 3497 else 3498 skb->ip_summed = CHECKSUM_UNNECESSARY; 3499 3500 napi_gro_receive(&ch->napi, skb); 3501 3502 priv->dev->stats.rx_packets++; 3503 priv->dev->stats.rx_bytes += frame_len; 3504 } 3505 entry = next_entry; 3506 } 3507 3508 stmmac_rx_refill(priv, queue); 3509 3510 priv->xstats.rx_pkt_n += count; 3511 3512 return count; 3513 } 3514 3515 /** 3516 * stmmac_poll - stmmac poll method (NAPI) 3517 * @napi : pointer to the napi structure. 3518 * @budget : maximum number of packets that the current CPU can receive from 3519 * all interfaces. 3520 * Description : 3521 * To look at the incoming frames and clear the tx resources. 3522 */ 3523 static int stmmac_napi_poll(struct napi_struct *napi, int budget) 3524 { 3525 struct stmmac_channel *ch = 3526 container_of(napi, struct stmmac_channel, napi); 3527 struct stmmac_priv *priv = ch->priv_data; 3528 int work_done = 0, work_rem = budget; 3529 u32 chan = ch->index; 3530 3531 priv->xstats.napi_poll++; 3532 3533 if (ch->has_tx) { 3534 int done = stmmac_tx_clean(priv, work_rem, chan); 3535 3536 work_done += done; 3537 work_rem -= done; 3538 } 3539 3540 if (ch->has_rx) { 3541 int done = stmmac_rx(priv, work_rem, chan); 3542 3543 work_done += done; 3544 work_rem -= done; 3545 } 3546 3547 if (work_done < budget && napi_complete_done(napi, work_done)) 3548 stmmac_enable_dma_irq(priv, priv->ioaddr, chan); 3549 3550 return work_done; 3551 } 3552 3553 /** 3554 * stmmac_tx_timeout 3555 * @dev : Pointer to net device structure 3556 * Description: this function is called when a packet transmission fails to 3557 * complete within a reasonable time. The driver will mark the error in the 3558 * netdev structure and arrange for the device to be reset to a sane state 3559 * in order to transmit a new packet. 3560 */ 3561 static void stmmac_tx_timeout(struct net_device *dev) 3562 { 3563 struct stmmac_priv *priv = netdev_priv(dev); 3564 3565 stmmac_global_err(priv); 3566 } 3567 3568 /** 3569 * stmmac_set_rx_mode - entry point for multicast addressing 3570 * @dev : pointer to the device structure 3571 * Description: 3572 * This function is a driver entry point which gets called by the kernel 3573 * whenever multicast addresses must be enabled/disabled. 3574 * Return value: 3575 * void. 3576 */ 3577 static void stmmac_set_rx_mode(struct net_device *dev) 3578 { 3579 struct stmmac_priv *priv = netdev_priv(dev); 3580 3581 stmmac_set_filter(priv, priv->hw, dev); 3582 } 3583 3584 /** 3585 * stmmac_change_mtu - entry point to change MTU size for the device. 3586 * @dev : device pointer. 3587 * @new_mtu : the new MTU size for the device. 3588 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 3589 * to drive packet transmission. Ethernet has an MTU of 1500 octets 3590 * (ETH_DATA_LEN). This value can be changed with ifconfig. 3591 * Return value: 3592 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3593 * file on failure. 3594 */ 3595 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 3596 { 3597 struct stmmac_priv *priv = netdev_priv(dev); 3598 3599 if (netif_running(dev)) { 3600 netdev_err(priv->dev, "must be stopped to change its MTU\n"); 3601 return -EBUSY; 3602 } 3603 3604 dev->mtu = new_mtu; 3605 3606 netdev_update_features(dev); 3607 3608 return 0; 3609 } 3610 3611 static netdev_features_t stmmac_fix_features(struct net_device *dev, 3612 netdev_features_t features) 3613 { 3614 struct stmmac_priv *priv = netdev_priv(dev); 3615 3616 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 3617 features &= ~NETIF_F_RXCSUM; 3618 3619 if (!priv->plat->tx_coe) 3620 features &= ~NETIF_F_CSUM_MASK; 3621 3622 /* Some GMAC devices have a bugged Jumbo frame support that 3623 * needs to have the Tx COE disabled for oversized frames 3624 * (due to limited buffer sizes). In this case we disable 3625 * the TX csum insertion in the TDES and not use SF. 3626 */ 3627 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 3628 features &= ~NETIF_F_CSUM_MASK; 3629 3630 /* Disable tso if asked by ethtool */ 3631 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 3632 if (features & NETIF_F_TSO) 3633 priv->tso = true; 3634 else 3635 priv->tso = false; 3636 } 3637 3638 return features; 3639 } 3640 3641 static int stmmac_set_features(struct net_device *netdev, 3642 netdev_features_t features) 3643 { 3644 struct stmmac_priv *priv = netdev_priv(netdev); 3645 3646 /* Keep the COE Type in case of csum is supporting */ 3647 if (features & NETIF_F_RXCSUM) 3648 priv->hw->rx_csum = priv->plat->rx_coe; 3649 else 3650 priv->hw->rx_csum = 0; 3651 /* No check needed because rx_coe has been set before and it will be 3652 * fixed in case of issue. 3653 */ 3654 stmmac_rx_ipc(priv, priv->hw); 3655 3656 return 0; 3657 } 3658 3659 /** 3660 * stmmac_interrupt - main ISR 3661 * @irq: interrupt number. 3662 * @dev_id: to pass the net device pointer. 3663 * Description: this is the main driver interrupt service routine. 3664 * It can call: 3665 * o DMA service routine (to manage incoming frame reception and transmission 3666 * status) 3667 * o Core interrupts to manage: remote wake-up, management counter, LPI 3668 * interrupts. 3669 */ 3670 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 3671 { 3672 struct net_device *dev = (struct net_device *)dev_id; 3673 struct stmmac_priv *priv = netdev_priv(dev); 3674 u32 rx_cnt = priv->plat->rx_queues_to_use; 3675 u32 tx_cnt = priv->plat->tx_queues_to_use; 3676 u32 queues_count; 3677 u32 queue; 3678 bool xmac; 3679 3680 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 3681 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 3682 3683 if (priv->irq_wake) 3684 pm_wakeup_event(priv->device, 0); 3685 3686 if (unlikely(!dev)) { 3687 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 3688 return IRQ_NONE; 3689 } 3690 3691 /* Check if adapter is up */ 3692 if (test_bit(STMMAC_DOWN, &priv->state)) 3693 return IRQ_HANDLED; 3694 /* Check if a fatal error happened */ 3695 if (stmmac_safety_feat_interrupt(priv)) 3696 return IRQ_HANDLED; 3697 3698 /* To handle GMAC own interrupts */ 3699 if ((priv->plat->has_gmac) || xmac) { 3700 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 3701 int mtl_status; 3702 3703 if (unlikely(status)) { 3704 /* For LPI we need to save the tx status */ 3705 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 3706 priv->tx_path_in_lpi_mode = true; 3707 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 3708 priv->tx_path_in_lpi_mode = false; 3709 } 3710 3711 for (queue = 0; queue < queues_count; queue++) { 3712 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3713 3714 mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw, 3715 queue); 3716 if (mtl_status != -EINVAL) 3717 status |= mtl_status; 3718 3719 if (status & CORE_IRQ_MTL_RX_OVERFLOW) 3720 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 3721 rx_q->rx_tail_addr, 3722 queue); 3723 } 3724 3725 /* PCS link status */ 3726 if (priv->hw->pcs) { 3727 if (priv->xstats.pcs_link) 3728 netif_carrier_on(dev); 3729 else 3730 netif_carrier_off(dev); 3731 } 3732 } 3733 3734 /* To handle DMA interrupts */ 3735 stmmac_dma_interrupt(priv); 3736 3737 return IRQ_HANDLED; 3738 } 3739 3740 #ifdef CONFIG_NET_POLL_CONTROLLER 3741 /* Polling receive - used by NETCONSOLE and other diagnostic tools 3742 * to allow network I/O with interrupts disabled. 3743 */ 3744 static void stmmac_poll_controller(struct net_device *dev) 3745 { 3746 disable_irq(dev->irq); 3747 stmmac_interrupt(dev->irq, dev); 3748 enable_irq(dev->irq); 3749 } 3750 #endif 3751 3752 /** 3753 * stmmac_ioctl - Entry point for the Ioctl 3754 * @dev: Device pointer. 3755 * @rq: An IOCTL specefic structure, that can contain a pointer to 3756 * a proprietary structure used to pass information to the driver. 3757 * @cmd: IOCTL command 3758 * Description: 3759 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 3760 */ 3761 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3762 { 3763 int ret = -EOPNOTSUPP; 3764 3765 if (!netif_running(dev)) 3766 return -EINVAL; 3767 3768 switch (cmd) { 3769 case SIOCGMIIPHY: 3770 case SIOCGMIIREG: 3771 case SIOCSMIIREG: 3772 if (!dev->phydev) 3773 return -EINVAL; 3774 ret = phy_mii_ioctl(dev->phydev, rq, cmd); 3775 break; 3776 case SIOCSHWTSTAMP: 3777 ret = stmmac_hwtstamp_ioctl(dev, rq); 3778 break; 3779 default: 3780 break; 3781 } 3782 3783 return ret; 3784 } 3785 3786 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 3787 void *cb_priv) 3788 { 3789 struct stmmac_priv *priv = cb_priv; 3790 int ret = -EOPNOTSUPP; 3791 3792 stmmac_disable_all_queues(priv); 3793 3794 switch (type) { 3795 case TC_SETUP_CLSU32: 3796 if (tc_cls_can_offload_and_chain0(priv->dev, type_data)) 3797 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 3798 break; 3799 default: 3800 break; 3801 } 3802 3803 stmmac_enable_all_queues(priv); 3804 return ret; 3805 } 3806 3807 static int stmmac_setup_tc_block(struct stmmac_priv *priv, 3808 struct tc_block_offload *f) 3809 { 3810 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 3811 return -EOPNOTSUPP; 3812 3813 switch (f->command) { 3814 case TC_BLOCK_BIND: 3815 return tcf_block_cb_register(f->block, stmmac_setup_tc_block_cb, 3816 priv, priv, f->extack); 3817 case TC_BLOCK_UNBIND: 3818 tcf_block_cb_unregister(f->block, stmmac_setup_tc_block_cb, priv); 3819 return 0; 3820 default: 3821 return -EOPNOTSUPP; 3822 } 3823 } 3824 3825 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 3826 void *type_data) 3827 { 3828 struct stmmac_priv *priv = netdev_priv(ndev); 3829 3830 switch (type) { 3831 case TC_SETUP_BLOCK: 3832 return stmmac_setup_tc_block(priv, type_data); 3833 case TC_SETUP_QDISC_CBS: 3834 return stmmac_tc_setup_cbs(priv, priv, type_data); 3835 default: 3836 return -EOPNOTSUPP; 3837 } 3838 } 3839 3840 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 3841 { 3842 struct stmmac_priv *priv = netdev_priv(ndev); 3843 int ret = 0; 3844 3845 ret = eth_mac_addr(ndev, addr); 3846 if (ret) 3847 return ret; 3848 3849 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 3850 3851 return ret; 3852 } 3853 3854 #ifdef CONFIG_DEBUG_FS 3855 static struct dentry *stmmac_fs_dir; 3856 3857 static void sysfs_display_ring(void *head, int size, int extend_desc, 3858 struct seq_file *seq) 3859 { 3860 int i; 3861 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 3862 struct dma_desc *p = (struct dma_desc *)head; 3863 3864 for (i = 0; i < size; i++) { 3865 if (extend_desc) { 3866 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3867 i, (unsigned int)virt_to_phys(ep), 3868 le32_to_cpu(ep->basic.des0), 3869 le32_to_cpu(ep->basic.des1), 3870 le32_to_cpu(ep->basic.des2), 3871 le32_to_cpu(ep->basic.des3)); 3872 ep++; 3873 } else { 3874 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3875 i, (unsigned int)virt_to_phys(p), 3876 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 3877 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 3878 p++; 3879 } 3880 seq_printf(seq, "\n"); 3881 } 3882 } 3883 3884 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v) 3885 { 3886 struct net_device *dev = seq->private; 3887 struct stmmac_priv *priv = netdev_priv(dev); 3888 u32 rx_count = priv->plat->rx_queues_to_use; 3889 u32 tx_count = priv->plat->tx_queues_to_use; 3890 u32 queue; 3891 3892 if ((dev->flags & IFF_UP) == 0) 3893 return 0; 3894 3895 for (queue = 0; queue < rx_count; queue++) { 3896 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3897 3898 seq_printf(seq, "RX Queue %d:\n", queue); 3899 3900 if (priv->extend_desc) { 3901 seq_printf(seq, "Extended descriptor ring:\n"); 3902 sysfs_display_ring((void *)rx_q->dma_erx, 3903 DMA_RX_SIZE, 1, seq); 3904 } else { 3905 seq_printf(seq, "Descriptor ring:\n"); 3906 sysfs_display_ring((void *)rx_q->dma_rx, 3907 DMA_RX_SIZE, 0, seq); 3908 } 3909 } 3910 3911 for (queue = 0; queue < tx_count; queue++) { 3912 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 3913 3914 seq_printf(seq, "TX Queue %d:\n", queue); 3915 3916 if (priv->extend_desc) { 3917 seq_printf(seq, "Extended descriptor ring:\n"); 3918 sysfs_display_ring((void *)tx_q->dma_etx, 3919 DMA_TX_SIZE, 1, seq); 3920 } else { 3921 seq_printf(seq, "Descriptor ring:\n"); 3922 sysfs_display_ring((void *)tx_q->dma_tx, 3923 DMA_TX_SIZE, 0, seq); 3924 } 3925 } 3926 3927 return 0; 3928 } 3929 3930 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file) 3931 { 3932 return single_open(file, stmmac_sysfs_ring_read, inode->i_private); 3933 } 3934 3935 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */ 3936 3937 static const struct file_operations stmmac_rings_status_fops = { 3938 .owner = THIS_MODULE, 3939 .open = stmmac_sysfs_ring_open, 3940 .read = seq_read, 3941 .llseek = seq_lseek, 3942 .release = single_release, 3943 }; 3944 3945 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v) 3946 { 3947 struct net_device *dev = seq->private; 3948 struct stmmac_priv *priv = netdev_priv(dev); 3949 3950 if (!priv->hw_cap_support) { 3951 seq_printf(seq, "DMA HW features not supported\n"); 3952 return 0; 3953 } 3954 3955 seq_printf(seq, "==============================\n"); 3956 seq_printf(seq, "\tDMA HW features\n"); 3957 seq_printf(seq, "==============================\n"); 3958 3959 seq_printf(seq, "\t10/100 Mbps: %s\n", 3960 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 3961 seq_printf(seq, "\t1000 Mbps: %s\n", 3962 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 3963 seq_printf(seq, "\tHalf duplex: %s\n", 3964 (priv->dma_cap.half_duplex) ? "Y" : "N"); 3965 seq_printf(seq, "\tHash Filter: %s\n", 3966 (priv->dma_cap.hash_filter) ? "Y" : "N"); 3967 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 3968 (priv->dma_cap.multi_addr) ? "Y" : "N"); 3969 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 3970 (priv->dma_cap.pcs) ? "Y" : "N"); 3971 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 3972 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 3973 seq_printf(seq, "\tPMT Remote wake up: %s\n", 3974 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 3975 seq_printf(seq, "\tPMT Magic Frame: %s\n", 3976 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 3977 seq_printf(seq, "\tRMON module: %s\n", 3978 (priv->dma_cap.rmon) ? "Y" : "N"); 3979 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 3980 (priv->dma_cap.time_stamp) ? "Y" : "N"); 3981 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 3982 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 3983 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 3984 (priv->dma_cap.eee) ? "Y" : "N"); 3985 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 3986 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 3987 (priv->dma_cap.tx_coe) ? "Y" : "N"); 3988 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3989 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 3990 (priv->dma_cap.rx_coe) ? "Y" : "N"); 3991 } else { 3992 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 3993 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 3994 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 3995 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 3996 } 3997 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 3998 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 3999 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 4000 priv->dma_cap.number_rx_channel); 4001 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 4002 priv->dma_cap.number_tx_channel); 4003 seq_printf(seq, "\tEnhanced descriptors: %s\n", 4004 (priv->dma_cap.enh_desc) ? "Y" : "N"); 4005 4006 return 0; 4007 } 4008 4009 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file) 4010 { 4011 return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private); 4012 } 4013 4014 static const struct file_operations stmmac_dma_cap_fops = { 4015 .owner = THIS_MODULE, 4016 .open = stmmac_sysfs_dma_cap_open, 4017 .read = seq_read, 4018 .llseek = seq_lseek, 4019 .release = single_release, 4020 }; 4021 4022 static int stmmac_init_fs(struct net_device *dev) 4023 { 4024 struct stmmac_priv *priv = netdev_priv(dev); 4025 4026 /* Create per netdev entries */ 4027 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 4028 4029 if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) { 4030 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n"); 4031 4032 return -ENOMEM; 4033 } 4034 4035 /* Entry to report DMA RX/TX rings */ 4036 priv->dbgfs_rings_status = 4037 debugfs_create_file("descriptors_status", 0444, 4038 priv->dbgfs_dir, dev, 4039 &stmmac_rings_status_fops); 4040 4041 if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) { 4042 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n"); 4043 debugfs_remove_recursive(priv->dbgfs_dir); 4044 4045 return -ENOMEM; 4046 } 4047 4048 /* Entry to report the DMA HW features */ 4049 priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", 0444, 4050 priv->dbgfs_dir, 4051 dev, &stmmac_dma_cap_fops); 4052 4053 if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) { 4054 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n"); 4055 debugfs_remove_recursive(priv->dbgfs_dir); 4056 4057 return -ENOMEM; 4058 } 4059 4060 return 0; 4061 } 4062 4063 static void stmmac_exit_fs(struct net_device *dev) 4064 { 4065 struct stmmac_priv *priv = netdev_priv(dev); 4066 4067 debugfs_remove_recursive(priv->dbgfs_dir); 4068 } 4069 #endif /* CONFIG_DEBUG_FS */ 4070 4071 static const struct net_device_ops stmmac_netdev_ops = { 4072 .ndo_open = stmmac_open, 4073 .ndo_start_xmit = stmmac_xmit, 4074 .ndo_stop = stmmac_release, 4075 .ndo_change_mtu = stmmac_change_mtu, 4076 .ndo_fix_features = stmmac_fix_features, 4077 .ndo_set_features = stmmac_set_features, 4078 .ndo_set_rx_mode = stmmac_set_rx_mode, 4079 .ndo_tx_timeout = stmmac_tx_timeout, 4080 .ndo_do_ioctl = stmmac_ioctl, 4081 .ndo_setup_tc = stmmac_setup_tc, 4082 #ifdef CONFIG_NET_POLL_CONTROLLER 4083 .ndo_poll_controller = stmmac_poll_controller, 4084 #endif 4085 .ndo_set_mac_address = stmmac_set_mac_address, 4086 }; 4087 4088 static void stmmac_reset_subtask(struct stmmac_priv *priv) 4089 { 4090 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 4091 return; 4092 if (test_bit(STMMAC_DOWN, &priv->state)) 4093 return; 4094 4095 netdev_err(priv->dev, "Reset adapter.\n"); 4096 4097 rtnl_lock(); 4098 netif_trans_update(priv->dev); 4099 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 4100 usleep_range(1000, 2000); 4101 4102 set_bit(STMMAC_DOWN, &priv->state); 4103 dev_close(priv->dev); 4104 dev_open(priv->dev); 4105 clear_bit(STMMAC_DOWN, &priv->state); 4106 clear_bit(STMMAC_RESETING, &priv->state); 4107 rtnl_unlock(); 4108 } 4109 4110 static void stmmac_service_task(struct work_struct *work) 4111 { 4112 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 4113 service_task); 4114 4115 stmmac_reset_subtask(priv); 4116 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 4117 } 4118 4119 /** 4120 * stmmac_hw_init - Init the MAC device 4121 * @priv: driver private structure 4122 * Description: this function is to configure the MAC device according to 4123 * some platform parameters or the HW capability register. It prepares the 4124 * driver to use either ring or chain modes and to setup either enhanced or 4125 * normal descriptors. 4126 */ 4127 static int stmmac_hw_init(struct stmmac_priv *priv) 4128 { 4129 int ret; 4130 4131 /* dwmac-sun8i only work in chain mode */ 4132 if (priv->plat->has_sun8i) 4133 chain_mode = 1; 4134 priv->chain_mode = chain_mode; 4135 4136 /* Initialize HW Interface */ 4137 ret = stmmac_hwif_init(priv); 4138 if (ret) 4139 return ret; 4140 4141 /* Get the HW capability (new GMAC newer than 3.50a) */ 4142 priv->hw_cap_support = stmmac_get_hw_features(priv); 4143 if (priv->hw_cap_support) { 4144 dev_info(priv->device, "DMA HW capability register supported\n"); 4145 4146 /* We can override some gmac/dma configuration fields: e.g. 4147 * enh_desc, tx_coe (e.g. that are passed through the 4148 * platform) with the values from the HW capability 4149 * register (if supported). 4150 */ 4151 priv->plat->enh_desc = priv->dma_cap.enh_desc; 4152 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; 4153 priv->hw->pmt = priv->plat->pmt; 4154 4155 /* TXCOE doesn't work in thresh DMA mode */ 4156 if (priv->plat->force_thresh_dma_mode) 4157 priv->plat->tx_coe = 0; 4158 else 4159 priv->plat->tx_coe = priv->dma_cap.tx_coe; 4160 4161 /* In case of GMAC4 rx_coe is from HW cap register. */ 4162 priv->plat->rx_coe = priv->dma_cap.rx_coe; 4163 4164 if (priv->dma_cap.rx_coe_type2) 4165 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 4166 else if (priv->dma_cap.rx_coe_type1) 4167 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 4168 4169 } else { 4170 dev_info(priv->device, "No HW DMA feature register supported\n"); 4171 } 4172 4173 if (priv->plat->rx_coe) { 4174 priv->hw->rx_csum = priv->plat->rx_coe; 4175 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 4176 if (priv->synopsys_id < DWMAC_CORE_4_00) 4177 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 4178 } 4179 if (priv->plat->tx_coe) 4180 dev_info(priv->device, "TX Checksum insertion supported\n"); 4181 4182 if (priv->plat->pmt) { 4183 dev_info(priv->device, "Wake-Up On Lan supported\n"); 4184 device_set_wakeup_capable(priv->device, 1); 4185 } 4186 4187 if (priv->dma_cap.tsoen) 4188 dev_info(priv->device, "TSO supported\n"); 4189 4190 /* Run HW quirks, if any */ 4191 if (priv->hwif_quirks) { 4192 ret = priv->hwif_quirks(priv); 4193 if (ret) 4194 return ret; 4195 } 4196 4197 return 0; 4198 } 4199 4200 /** 4201 * stmmac_dvr_probe 4202 * @device: device pointer 4203 * @plat_dat: platform data pointer 4204 * @res: stmmac resource pointer 4205 * Description: this is the main probe function used to 4206 * call the alloc_etherdev, allocate the priv structure. 4207 * Return: 4208 * returns 0 on success, otherwise errno. 4209 */ 4210 int stmmac_dvr_probe(struct device *device, 4211 struct plat_stmmacenet_data *plat_dat, 4212 struct stmmac_resources *res) 4213 { 4214 struct net_device *ndev = NULL; 4215 struct stmmac_priv *priv; 4216 u32 queue, maxq; 4217 int ret = 0; 4218 4219 ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv), 4220 MTL_MAX_TX_QUEUES, 4221 MTL_MAX_RX_QUEUES); 4222 if (!ndev) 4223 return -ENOMEM; 4224 4225 SET_NETDEV_DEV(ndev, device); 4226 4227 priv = netdev_priv(ndev); 4228 priv->device = device; 4229 priv->dev = ndev; 4230 4231 stmmac_set_ethtool_ops(ndev); 4232 priv->pause = pause; 4233 priv->plat = plat_dat; 4234 priv->ioaddr = res->addr; 4235 priv->dev->base_addr = (unsigned long)res->addr; 4236 4237 priv->dev->irq = res->irq; 4238 priv->wol_irq = res->wol_irq; 4239 priv->lpi_irq = res->lpi_irq; 4240 4241 if (res->mac) 4242 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); 4243 4244 dev_set_drvdata(device, priv->dev); 4245 4246 /* Verify driver arguments */ 4247 stmmac_verify_args(); 4248 4249 /* Allocate workqueue */ 4250 priv->wq = create_singlethread_workqueue("stmmac_wq"); 4251 if (!priv->wq) { 4252 dev_err(priv->device, "failed to create workqueue\n"); 4253 ret = -ENOMEM; 4254 goto error_wq; 4255 } 4256 4257 INIT_WORK(&priv->service_task, stmmac_service_task); 4258 4259 /* Override with kernel parameters if supplied XXX CRS XXX 4260 * this needs to have multiple instances 4261 */ 4262 if ((phyaddr >= 0) && (phyaddr <= 31)) 4263 priv->plat->phy_addr = phyaddr; 4264 4265 if (priv->plat->stmmac_rst) { 4266 ret = reset_control_assert(priv->plat->stmmac_rst); 4267 reset_control_deassert(priv->plat->stmmac_rst); 4268 /* Some reset controllers have only reset callback instead of 4269 * assert + deassert callbacks pair. 4270 */ 4271 if (ret == -ENOTSUPP) 4272 reset_control_reset(priv->plat->stmmac_rst); 4273 } 4274 4275 /* Init MAC and get the capabilities */ 4276 ret = stmmac_hw_init(priv); 4277 if (ret) 4278 goto error_hw_init; 4279 4280 /* Configure real RX and TX queues */ 4281 netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use); 4282 netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use); 4283 4284 ndev->netdev_ops = &stmmac_netdev_ops; 4285 4286 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4287 NETIF_F_RXCSUM; 4288 4289 ret = stmmac_tc_init(priv, priv); 4290 if (!ret) { 4291 ndev->hw_features |= NETIF_F_HW_TC; 4292 } 4293 4294 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 4295 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4296 priv->tso = true; 4297 dev_info(priv->device, "TSO feature enabled\n"); 4298 } 4299 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 4300 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 4301 #ifdef STMMAC_VLAN_TAG_USED 4302 /* Both mac100 and gmac support receive VLAN tag detection */ 4303 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 4304 #endif 4305 priv->msg_enable = netif_msg_init(debug, default_msg_level); 4306 4307 /* MTU range: 46 - hw-specific max */ 4308 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 4309 if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 4310 ndev->max_mtu = JUMBO_LEN; 4311 else if (priv->plat->has_xgmac) 4312 ndev->max_mtu = XGMAC_JUMBO_LEN; 4313 else 4314 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 4315 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 4316 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 4317 */ 4318 if ((priv->plat->maxmtu < ndev->max_mtu) && 4319 (priv->plat->maxmtu >= ndev->min_mtu)) 4320 ndev->max_mtu = priv->plat->maxmtu; 4321 else if (priv->plat->maxmtu < ndev->min_mtu) 4322 dev_warn(priv->device, 4323 "%s: warning: maxmtu having invalid value (%d)\n", 4324 __func__, priv->plat->maxmtu); 4325 4326 if (flow_ctrl) 4327 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 4328 4329 /* Rx Watchdog is available in the COREs newer than the 3.40. 4330 * In some case, for example on bugged HW this feature 4331 * has to be disable and this can be done by passing the 4332 * riwt_off field from the platform. 4333 */ 4334 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 4335 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 4336 priv->use_riwt = 1; 4337 dev_info(priv->device, 4338 "Enable RX Mitigation via HW Watchdog Timer\n"); 4339 } 4340 4341 /* Setup channels NAPI */ 4342 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 4343 4344 for (queue = 0; queue < maxq; queue++) { 4345 struct stmmac_channel *ch = &priv->channel[queue]; 4346 4347 ch->priv_data = priv; 4348 ch->index = queue; 4349 4350 if (queue < priv->plat->rx_queues_to_use) 4351 ch->has_rx = true; 4352 if (queue < priv->plat->tx_queues_to_use) 4353 ch->has_tx = true; 4354 4355 netif_napi_add(ndev, &ch->napi, stmmac_napi_poll, 4356 NAPI_POLL_WEIGHT); 4357 } 4358 4359 mutex_init(&priv->lock); 4360 4361 /* If a specific clk_csr value is passed from the platform 4362 * this means that the CSR Clock Range selection cannot be 4363 * changed at run-time and it is fixed. Viceversa the driver'll try to 4364 * set the MDC clock dynamically according to the csr actual 4365 * clock input. 4366 */ 4367 if (!priv->plat->clk_csr) 4368 stmmac_clk_csr_set(priv); 4369 else 4370 priv->clk_csr = priv->plat->clk_csr; 4371 4372 stmmac_check_pcs_mode(priv); 4373 4374 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4375 priv->hw->pcs != STMMAC_PCS_TBI && 4376 priv->hw->pcs != STMMAC_PCS_RTBI) { 4377 /* MDIO bus Registration */ 4378 ret = stmmac_mdio_register(ndev); 4379 if (ret < 0) { 4380 dev_err(priv->device, 4381 "%s: MDIO bus (id: %d) registration failed", 4382 __func__, priv->plat->bus_id); 4383 goto error_mdio_register; 4384 } 4385 } 4386 4387 ret = register_netdev(ndev); 4388 if (ret) { 4389 dev_err(priv->device, "%s: ERROR %i registering the device\n", 4390 __func__, ret); 4391 goto error_netdev_register; 4392 } 4393 4394 #ifdef CONFIG_DEBUG_FS 4395 ret = stmmac_init_fs(ndev); 4396 if (ret < 0) 4397 netdev_warn(priv->dev, "%s: failed debugFS registration\n", 4398 __func__); 4399 #endif 4400 4401 return ret; 4402 4403 error_netdev_register: 4404 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4405 priv->hw->pcs != STMMAC_PCS_TBI && 4406 priv->hw->pcs != STMMAC_PCS_RTBI) 4407 stmmac_mdio_unregister(ndev); 4408 error_mdio_register: 4409 for (queue = 0; queue < maxq; queue++) { 4410 struct stmmac_channel *ch = &priv->channel[queue]; 4411 4412 netif_napi_del(&ch->napi); 4413 } 4414 error_hw_init: 4415 destroy_workqueue(priv->wq); 4416 error_wq: 4417 free_netdev(ndev); 4418 4419 return ret; 4420 } 4421 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 4422 4423 /** 4424 * stmmac_dvr_remove 4425 * @dev: device pointer 4426 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 4427 * changes the link status, releases the DMA descriptor rings. 4428 */ 4429 int stmmac_dvr_remove(struct device *dev) 4430 { 4431 struct net_device *ndev = dev_get_drvdata(dev); 4432 struct stmmac_priv *priv = netdev_priv(ndev); 4433 4434 netdev_info(priv->dev, "%s: removing driver", __func__); 4435 4436 #ifdef CONFIG_DEBUG_FS 4437 stmmac_exit_fs(ndev); 4438 #endif 4439 stmmac_stop_all_dma(priv); 4440 4441 stmmac_mac_set(priv, priv->ioaddr, false); 4442 netif_carrier_off(ndev); 4443 unregister_netdev(ndev); 4444 if (priv->plat->stmmac_rst) 4445 reset_control_assert(priv->plat->stmmac_rst); 4446 clk_disable_unprepare(priv->plat->pclk); 4447 clk_disable_unprepare(priv->plat->stmmac_clk); 4448 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4449 priv->hw->pcs != STMMAC_PCS_TBI && 4450 priv->hw->pcs != STMMAC_PCS_RTBI) 4451 stmmac_mdio_unregister(ndev); 4452 destroy_workqueue(priv->wq); 4453 mutex_destroy(&priv->lock); 4454 free_netdev(ndev); 4455 4456 return 0; 4457 } 4458 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 4459 4460 /** 4461 * stmmac_suspend - suspend callback 4462 * @dev: device pointer 4463 * Description: this is the function to suspend the device and it is called 4464 * by the platform driver to stop the network queue, release the resources, 4465 * program the PMT register (for WoL), clean and release driver resources. 4466 */ 4467 int stmmac_suspend(struct device *dev) 4468 { 4469 struct net_device *ndev = dev_get_drvdata(dev); 4470 struct stmmac_priv *priv = netdev_priv(ndev); 4471 4472 if (!ndev || !netif_running(ndev)) 4473 return 0; 4474 4475 if (ndev->phydev) 4476 phy_stop(ndev->phydev); 4477 4478 mutex_lock(&priv->lock); 4479 4480 netif_device_detach(ndev); 4481 stmmac_stop_all_queues(priv); 4482 4483 stmmac_disable_all_queues(priv); 4484 4485 /* Stop TX/RX DMA */ 4486 stmmac_stop_all_dma(priv); 4487 4488 /* Enable Power down mode by programming the PMT regs */ 4489 if (device_may_wakeup(priv->device)) { 4490 stmmac_pmt(priv, priv->hw, priv->wolopts); 4491 priv->irq_wake = 1; 4492 } else { 4493 stmmac_mac_set(priv, priv->ioaddr, false); 4494 pinctrl_pm_select_sleep_state(priv->device); 4495 /* Disable clock in case of PWM is off */ 4496 clk_disable(priv->plat->pclk); 4497 clk_disable(priv->plat->stmmac_clk); 4498 } 4499 mutex_unlock(&priv->lock); 4500 4501 priv->oldlink = false; 4502 priv->speed = SPEED_UNKNOWN; 4503 priv->oldduplex = DUPLEX_UNKNOWN; 4504 return 0; 4505 } 4506 EXPORT_SYMBOL_GPL(stmmac_suspend); 4507 4508 /** 4509 * stmmac_reset_queues_param - reset queue parameters 4510 * @dev: device pointer 4511 */ 4512 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 4513 { 4514 u32 rx_cnt = priv->plat->rx_queues_to_use; 4515 u32 tx_cnt = priv->plat->tx_queues_to_use; 4516 u32 queue; 4517 4518 for (queue = 0; queue < rx_cnt; queue++) { 4519 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4520 4521 rx_q->cur_rx = 0; 4522 rx_q->dirty_rx = 0; 4523 } 4524 4525 for (queue = 0; queue < tx_cnt; queue++) { 4526 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 4527 4528 tx_q->cur_tx = 0; 4529 tx_q->dirty_tx = 0; 4530 tx_q->mss = 0; 4531 } 4532 } 4533 4534 /** 4535 * stmmac_resume - resume callback 4536 * @dev: device pointer 4537 * Description: when resume this function is invoked to setup the DMA and CORE 4538 * in a usable state. 4539 */ 4540 int stmmac_resume(struct device *dev) 4541 { 4542 struct net_device *ndev = dev_get_drvdata(dev); 4543 struct stmmac_priv *priv = netdev_priv(ndev); 4544 4545 if (!netif_running(ndev)) 4546 return 0; 4547 4548 /* Power Down bit, into the PM register, is cleared 4549 * automatically as soon as a magic packet or a Wake-up frame 4550 * is received. Anyway, it's better to manually clear 4551 * this bit because it can generate problems while resuming 4552 * from another devices (e.g. serial console). 4553 */ 4554 if (device_may_wakeup(priv->device)) { 4555 mutex_lock(&priv->lock); 4556 stmmac_pmt(priv, priv->hw, 0); 4557 mutex_unlock(&priv->lock); 4558 priv->irq_wake = 0; 4559 } else { 4560 pinctrl_pm_select_default_state(priv->device); 4561 /* enable the clk previously disabled */ 4562 clk_enable(priv->plat->stmmac_clk); 4563 clk_enable(priv->plat->pclk); 4564 /* reset the phy so that it's ready */ 4565 if (priv->mii) 4566 stmmac_mdio_reset(priv->mii); 4567 } 4568 4569 netif_device_attach(ndev); 4570 4571 mutex_lock(&priv->lock); 4572 4573 stmmac_reset_queues_param(priv); 4574 4575 stmmac_clear_descriptors(priv); 4576 4577 stmmac_hw_setup(ndev, false); 4578 stmmac_init_tx_coalesce(priv); 4579 stmmac_set_rx_mode(ndev); 4580 4581 stmmac_enable_all_queues(priv); 4582 4583 stmmac_start_all_queues(priv); 4584 4585 mutex_unlock(&priv->lock); 4586 4587 if (ndev->phydev) 4588 phy_start(ndev->phydev); 4589 4590 return 0; 4591 } 4592 EXPORT_SYMBOL_GPL(stmmac_resume); 4593 4594 #ifndef MODULE 4595 static int __init stmmac_cmdline_opt(char *str) 4596 { 4597 char *opt; 4598 4599 if (!str || !*str) 4600 return -EINVAL; 4601 while ((opt = strsep(&str, ",")) != NULL) { 4602 if (!strncmp(opt, "debug:", 6)) { 4603 if (kstrtoint(opt + 6, 0, &debug)) 4604 goto err; 4605 } else if (!strncmp(opt, "phyaddr:", 8)) { 4606 if (kstrtoint(opt + 8, 0, &phyaddr)) 4607 goto err; 4608 } else if (!strncmp(opt, "buf_sz:", 7)) { 4609 if (kstrtoint(opt + 7, 0, &buf_sz)) 4610 goto err; 4611 } else if (!strncmp(opt, "tc:", 3)) { 4612 if (kstrtoint(opt + 3, 0, &tc)) 4613 goto err; 4614 } else if (!strncmp(opt, "watchdog:", 9)) { 4615 if (kstrtoint(opt + 9, 0, &watchdog)) 4616 goto err; 4617 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 4618 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 4619 goto err; 4620 } else if (!strncmp(opt, "pause:", 6)) { 4621 if (kstrtoint(opt + 6, 0, &pause)) 4622 goto err; 4623 } else if (!strncmp(opt, "eee_timer:", 10)) { 4624 if (kstrtoint(opt + 10, 0, &eee_timer)) 4625 goto err; 4626 } else if (!strncmp(opt, "chain_mode:", 11)) { 4627 if (kstrtoint(opt + 11, 0, &chain_mode)) 4628 goto err; 4629 } 4630 } 4631 return 0; 4632 4633 err: 4634 pr_err("%s: ERROR broken module parameter conversion", __func__); 4635 return -EINVAL; 4636 } 4637 4638 __setup("stmmaceth=", stmmac_cmdline_opt); 4639 #endif /* MODULE */ 4640 4641 static int __init stmmac_init(void) 4642 { 4643 #ifdef CONFIG_DEBUG_FS 4644 /* Create debugfs main directory if it doesn't exist yet */ 4645 if (!stmmac_fs_dir) { 4646 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 4647 4648 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) { 4649 pr_err("ERROR %s, debugfs create directory failed\n", 4650 STMMAC_RESOURCE_NAME); 4651 4652 return -ENOMEM; 4653 } 4654 } 4655 #endif 4656 4657 return 0; 4658 } 4659 4660 static void __exit stmmac_exit(void) 4661 { 4662 #ifdef CONFIG_DEBUG_FS 4663 debugfs_remove_recursive(stmmac_fs_dir); 4664 #endif 4665 } 4666 4667 module_init(stmmac_init) 4668 module_exit(stmmac_exit) 4669 4670 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 4671 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 4672 MODULE_LICENSE("GPL"); 4673