1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/clk.h> 18 #include <linux/kernel.h> 19 #include <linux/interrupt.h> 20 #include <linux/ip.h> 21 #include <linux/tcp.h> 22 #include <linux/skbuff.h> 23 #include <linux/ethtool.h> 24 #include <linux/if_ether.h> 25 #include <linux/crc32.h> 26 #include <linux/mii.h> 27 #include <linux/if.h> 28 #include <linux/if_vlan.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/slab.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/prefetch.h> 33 #include <linux/pinctrl/consumer.h> 34 #ifdef CONFIG_DEBUG_FS 35 #include <linux/debugfs.h> 36 #include <linux/seq_file.h> 37 #endif /* CONFIG_DEBUG_FS */ 38 #include <linux/net_tstamp.h> 39 #include <linux/phylink.h> 40 #include <linux/udp.h> 41 #include <linux/bpf_trace.h> 42 #include <net/page_pool/helpers.h> 43 #include <net/pkt_cls.h> 44 #include <net/xdp_sock_drv.h> 45 #include "stmmac_ptp.h" 46 #include "stmmac.h" 47 #include "stmmac_xdp.h" 48 #include <linux/reset.h> 49 #include <linux/of_mdio.h> 50 #include "dwmac1000.h" 51 #include "dwxgmac2.h" 52 #include "hwif.h" 53 54 /* As long as the interface is active, we keep the timestamping counter enabled 55 * with fine resolution and binary rollover. This avoid non-monotonic behavior 56 * (clock jumps) when changing timestamping settings at runtime. 57 */ 58 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ 59 PTP_TCR_TSCTRLSSR) 60 61 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 62 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 63 64 /* Module parameters */ 65 #define TX_TIMEO 5000 66 static int watchdog = TX_TIMEO; 67 module_param(watchdog, int, 0644); 68 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 69 70 static int debug = -1; 71 module_param(debug, int, 0644); 72 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 73 74 static int phyaddr = -1; 75 module_param(phyaddr, int, 0444); 76 MODULE_PARM_DESC(phyaddr, "Physical device address"); 77 78 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 79 #define STMMAC_RX_THRESH(x) ((x)->dma_conf.dma_rx_size / 4) 80 81 /* Limit to make sure XDP TX and slow path can coexist */ 82 #define STMMAC_XSK_TX_BUDGET_MAX 256 83 #define STMMAC_TX_XSK_AVAIL 16 84 #define STMMAC_RX_FILL_BATCH 16 85 86 #define STMMAC_XDP_PASS 0 87 #define STMMAC_XDP_CONSUMED BIT(0) 88 #define STMMAC_XDP_TX BIT(1) 89 #define STMMAC_XDP_REDIRECT BIT(2) 90 91 static int flow_ctrl = FLOW_AUTO; 92 module_param(flow_ctrl, int, 0644); 93 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 94 95 static int pause = PAUSE_TIME; 96 module_param(pause, int, 0644); 97 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 98 99 #define TC_DEFAULT 64 100 static int tc = TC_DEFAULT; 101 module_param(tc, int, 0644); 102 MODULE_PARM_DESC(tc, "DMA threshold control value"); 103 104 #define DEFAULT_BUFSIZE 1536 105 static int buf_sz = DEFAULT_BUFSIZE; 106 module_param(buf_sz, int, 0644); 107 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 108 109 #define STMMAC_RX_COPYBREAK 256 110 111 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 112 NETIF_MSG_LINK | NETIF_MSG_IFUP | 113 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 114 115 #define STMMAC_DEFAULT_LPI_TIMER 1000 116 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 117 module_param(eee_timer, int, 0644); 118 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 119 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 120 121 /* By default the driver will use the ring mode to manage tx and rx descriptors, 122 * but allow user to force to use the chain instead of the ring 123 */ 124 static unsigned int chain_mode; 125 module_param(chain_mode, int, 0444); 126 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 127 128 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 129 /* For MSI interrupts handling */ 130 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 131 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 132 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 133 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 134 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 135 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 136 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 137 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 138 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 139 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 140 u32 rxmode, u32 chan); 141 142 #ifdef CONFIG_DEBUG_FS 143 static const struct net_device_ops stmmac_netdev_ops; 144 static void stmmac_init_fs(struct net_device *dev); 145 static void stmmac_exit_fs(struct net_device *dev); 146 #endif 147 148 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 149 150 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) 151 { 152 int ret = 0; 153 154 if (enabled) { 155 ret = clk_prepare_enable(priv->plat->stmmac_clk); 156 if (ret) 157 return ret; 158 ret = clk_prepare_enable(priv->plat->pclk); 159 if (ret) { 160 clk_disable_unprepare(priv->plat->stmmac_clk); 161 return ret; 162 } 163 if (priv->plat->clks_config) { 164 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); 165 if (ret) { 166 clk_disable_unprepare(priv->plat->stmmac_clk); 167 clk_disable_unprepare(priv->plat->pclk); 168 return ret; 169 } 170 } 171 } else { 172 clk_disable_unprepare(priv->plat->stmmac_clk); 173 clk_disable_unprepare(priv->plat->pclk); 174 if (priv->plat->clks_config) 175 priv->plat->clks_config(priv->plat->bsp_priv, enabled); 176 } 177 178 return ret; 179 } 180 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); 181 182 /** 183 * stmmac_verify_args - verify the driver parameters. 184 * Description: it checks the driver parameters and set a default in case of 185 * errors. 186 */ 187 static void stmmac_verify_args(void) 188 { 189 if (unlikely(watchdog < 0)) 190 watchdog = TX_TIMEO; 191 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 192 buf_sz = DEFAULT_BUFSIZE; 193 if (unlikely(flow_ctrl > 1)) 194 flow_ctrl = FLOW_AUTO; 195 else if (likely(flow_ctrl < 0)) 196 flow_ctrl = FLOW_OFF; 197 if (unlikely((pause < 0) || (pause > 0xffff))) 198 pause = PAUSE_TIME; 199 if (eee_timer < 0) 200 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 201 } 202 203 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 204 { 205 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 206 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 207 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 208 u32 queue; 209 210 for (queue = 0; queue < maxq; queue++) { 211 struct stmmac_channel *ch = &priv->channel[queue]; 212 213 if (stmmac_xdp_is_enabled(priv) && 214 test_bit(queue, priv->af_xdp_zc_qps)) { 215 napi_disable(&ch->rxtx_napi); 216 continue; 217 } 218 219 if (queue < rx_queues_cnt) 220 napi_disable(&ch->rx_napi); 221 if (queue < tx_queues_cnt) 222 napi_disable(&ch->tx_napi); 223 } 224 } 225 226 /** 227 * stmmac_disable_all_queues - Disable all queues 228 * @priv: driver private structure 229 */ 230 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 231 { 232 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 233 struct stmmac_rx_queue *rx_q; 234 u32 queue; 235 236 /* synchronize_rcu() needed for pending XDP buffers to drain */ 237 for (queue = 0; queue < rx_queues_cnt; queue++) { 238 rx_q = &priv->dma_conf.rx_queue[queue]; 239 if (rx_q->xsk_pool) { 240 synchronize_rcu(); 241 break; 242 } 243 } 244 245 __stmmac_disable_all_queues(priv); 246 } 247 248 /** 249 * stmmac_enable_all_queues - Enable all queues 250 * @priv: driver private structure 251 */ 252 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 253 { 254 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 255 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 256 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 257 u32 queue; 258 259 for (queue = 0; queue < maxq; queue++) { 260 struct stmmac_channel *ch = &priv->channel[queue]; 261 262 if (stmmac_xdp_is_enabled(priv) && 263 test_bit(queue, priv->af_xdp_zc_qps)) { 264 napi_enable(&ch->rxtx_napi); 265 continue; 266 } 267 268 if (queue < rx_queues_cnt) 269 napi_enable(&ch->rx_napi); 270 if (queue < tx_queues_cnt) 271 napi_enable(&ch->tx_napi); 272 } 273 } 274 275 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 276 { 277 if (!test_bit(STMMAC_DOWN, &priv->state) && 278 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 279 queue_work(priv->wq, &priv->service_task); 280 } 281 282 static void stmmac_global_err(struct stmmac_priv *priv) 283 { 284 netif_carrier_off(priv->dev); 285 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 286 stmmac_service_event_schedule(priv); 287 } 288 289 /** 290 * stmmac_clk_csr_set - dynamically set the MDC clock 291 * @priv: driver private structure 292 * Description: this is to dynamically set the MDC clock according to the csr 293 * clock input. 294 * Note: 295 * If a specific clk_csr value is passed from the platform 296 * this means that the CSR Clock Range selection cannot be 297 * changed at run-time and it is fixed (as reported in the driver 298 * documentation). Viceversa the driver will try to set the MDC 299 * clock dynamically according to the actual clock input. 300 */ 301 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 302 { 303 u32 clk_rate; 304 305 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 306 307 /* Platform provided default clk_csr would be assumed valid 308 * for all other cases except for the below mentioned ones. 309 * For values higher than the IEEE 802.3 specified frequency 310 * we can not estimate the proper divider as it is not known 311 * the frequency of clk_csr_i. So we do not change the default 312 * divider. 313 */ 314 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 315 if (clk_rate < CSR_F_35M) 316 priv->clk_csr = STMMAC_CSR_20_35M; 317 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 318 priv->clk_csr = STMMAC_CSR_35_60M; 319 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 320 priv->clk_csr = STMMAC_CSR_60_100M; 321 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 322 priv->clk_csr = STMMAC_CSR_100_150M; 323 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 324 priv->clk_csr = STMMAC_CSR_150_250M; 325 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) 326 priv->clk_csr = STMMAC_CSR_250_300M; 327 } 328 329 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { 330 if (clk_rate > 160000000) 331 priv->clk_csr = 0x03; 332 else if (clk_rate > 80000000) 333 priv->clk_csr = 0x02; 334 else if (clk_rate > 40000000) 335 priv->clk_csr = 0x01; 336 else 337 priv->clk_csr = 0; 338 } 339 340 if (priv->plat->has_xgmac) { 341 if (clk_rate > 400000000) 342 priv->clk_csr = 0x5; 343 else if (clk_rate > 350000000) 344 priv->clk_csr = 0x4; 345 else if (clk_rate > 300000000) 346 priv->clk_csr = 0x3; 347 else if (clk_rate > 250000000) 348 priv->clk_csr = 0x2; 349 else if (clk_rate > 150000000) 350 priv->clk_csr = 0x1; 351 else 352 priv->clk_csr = 0x0; 353 } 354 } 355 356 static void print_pkt(unsigned char *buf, int len) 357 { 358 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 359 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 360 } 361 362 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 363 { 364 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 365 u32 avail; 366 367 if (tx_q->dirty_tx > tx_q->cur_tx) 368 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 369 else 370 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 371 372 return avail; 373 } 374 375 /** 376 * stmmac_rx_dirty - Get RX queue dirty 377 * @priv: driver private structure 378 * @queue: RX queue index 379 */ 380 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 381 { 382 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 383 u32 dirty; 384 385 if (rx_q->dirty_rx <= rx_q->cur_rx) 386 dirty = rx_q->cur_rx - rx_q->dirty_rx; 387 else 388 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 389 390 return dirty; 391 } 392 393 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) 394 { 395 int tx_lpi_timer; 396 397 /* Clear/set the SW EEE timer flag based on LPI ET enablement */ 398 priv->eee_sw_timer_en = en ? 0 : 1; 399 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; 400 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); 401 } 402 403 /** 404 * stmmac_enable_eee_mode - check and enter in LPI mode 405 * @priv: driver private structure 406 * Description: this function is to verify and enter in LPI mode in case of 407 * EEE. 408 */ 409 static int stmmac_enable_eee_mode(struct stmmac_priv *priv) 410 { 411 u32 tx_cnt = priv->plat->tx_queues_to_use; 412 u32 queue; 413 414 /* check if all TX queues have the work finished */ 415 for (queue = 0; queue < tx_cnt; queue++) { 416 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 417 418 if (tx_q->dirty_tx != tx_q->cur_tx) 419 return -EBUSY; /* still unfinished work */ 420 } 421 422 /* Check and enter in LPI mode */ 423 if (!priv->tx_path_in_lpi_mode) 424 stmmac_set_eee_mode(priv, priv->hw, 425 priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); 426 return 0; 427 } 428 429 /** 430 * stmmac_disable_eee_mode - disable and exit from LPI mode 431 * @priv: driver private structure 432 * Description: this function is to exit and disable EEE in case of 433 * LPI state is true. This is called by the xmit. 434 */ 435 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 436 { 437 if (!priv->eee_sw_timer_en) { 438 stmmac_lpi_entry_timer_config(priv, 0); 439 return; 440 } 441 442 stmmac_reset_eee_mode(priv, priv->hw); 443 del_timer_sync(&priv->eee_ctrl_timer); 444 priv->tx_path_in_lpi_mode = false; 445 } 446 447 /** 448 * stmmac_eee_ctrl_timer - EEE TX SW timer. 449 * @t: timer_list struct containing private info 450 * Description: 451 * if there is no data transfer and if we are not in LPI state, 452 * then MAC Transmitter can be moved to LPI state. 453 */ 454 static void stmmac_eee_ctrl_timer(struct timer_list *t) 455 { 456 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 457 458 if (stmmac_enable_eee_mode(priv)) 459 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 460 } 461 462 /** 463 * stmmac_eee_init - init EEE 464 * @priv: driver private structure 465 * Description: 466 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 467 * can also manage EEE, this function enable the LPI state and start related 468 * timer. 469 */ 470 bool stmmac_eee_init(struct stmmac_priv *priv) 471 { 472 int eee_tw_timer = priv->eee_tw_timer; 473 474 /* Using PCS we cannot dial with the phy registers at this stage 475 * so we do not support extra feature like EEE. 476 */ 477 if (priv->hw->pcs == STMMAC_PCS_TBI || 478 priv->hw->pcs == STMMAC_PCS_RTBI) 479 return false; 480 481 /* Check if MAC core supports the EEE feature. */ 482 if (!priv->dma_cap.eee) 483 return false; 484 485 mutex_lock(&priv->lock); 486 487 /* Check if it needs to be deactivated */ 488 if (!priv->eee_active) { 489 if (priv->eee_enabled) { 490 netdev_dbg(priv->dev, "disable EEE\n"); 491 stmmac_lpi_entry_timer_config(priv, 0); 492 del_timer_sync(&priv->eee_ctrl_timer); 493 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); 494 if (priv->hw->xpcs) 495 xpcs_config_eee(priv->hw->xpcs, 496 priv->plat->mult_fact_100ns, 497 false); 498 } 499 mutex_unlock(&priv->lock); 500 return false; 501 } 502 503 if (priv->eee_active && !priv->eee_enabled) { 504 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 505 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 506 eee_tw_timer); 507 if (priv->hw->xpcs) 508 xpcs_config_eee(priv->hw->xpcs, 509 priv->plat->mult_fact_100ns, 510 true); 511 } 512 513 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { 514 del_timer_sync(&priv->eee_ctrl_timer); 515 priv->tx_path_in_lpi_mode = false; 516 stmmac_lpi_entry_timer_config(priv, 1); 517 } else { 518 stmmac_lpi_entry_timer_config(priv, 0); 519 mod_timer(&priv->eee_ctrl_timer, 520 STMMAC_LPI_T(priv->tx_lpi_timer)); 521 } 522 523 mutex_unlock(&priv->lock); 524 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 525 return true; 526 } 527 528 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 529 * @priv: driver private structure 530 * @p : descriptor pointer 531 * @skb : the socket buffer 532 * Description : 533 * This function will read timestamp from the descriptor & pass it to stack. 534 * and also perform some sanity checks. 535 */ 536 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 537 struct dma_desc *p, struct sk_buff *skb) 538 { 539 struct skb_shared_hwtstamps shhwtstamp; 540 bool found = false; 541 u64 ns = 0; 542 543 if (!priv->hwts_tx_en) 544 return; 545 546 /* exit if skb doesn't support hw tstamp */ 547 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 548 return; 549 550 /* check tx tstamp status */ 551 if (stmmac_get_tx_timestamp_status(priv, p)) { 552 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 553 found = true; 554 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 555 found = true; 556 } 557 558 if (found) { 559 ns -= priv->plat->cdc_error_adj; 560 561 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 562 shhwtstamp.hwtstamp = ns_to_ktime(ns); 563 564 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 565 /* pass tstamp to stack */ 566 skb_tstamp_tx(skb, &shhwtstamp); 567 } 568 } 569 570 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 571 * @priv: driver private structure 572 * @p : descriptor pointer 573 * @np : next descriptor pointer 574 * @skb : the socket buffer 575 * Description : 576 * This function will read received packet's timestamp from the descriptor 577 * and pass it to stack. It also perform some sanity checks. 578 */ 579 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 580 struct dma_desc *np, struct sk_buff *skb) 581 { 582 struct skb_shared_hwtstamps *shhwtstamp = NULL; 583 struct dma_desc *desc = p; 584 u64 ns = 0; 585 586 if (!priv->hwts_rx_en) 587 return; 588 /* For GMAC4, the valid timestamp is from CTX next desc. */ 589 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 590 desc = np; 591 592 /* Check if timestamp is available */ 593 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 594 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 595 596 ns -= priv->plat->cdc_error_adj; 597 598 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 599 shhwtstamp = skb_hwtstamps(skb); 600 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 601 shhwtstamp->hwtstamp = ns_to_ktime(ns); 602 } else { 603 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 604 } 605 } 606 607 /** 608 * stmmac_hwtstamp_set - control hardware timestamping. 609 * @dev: device pointer. 610 * @ifr: An IOCTL specific structure, that can contain a pointer to 611 * a proprietary structure used to pass information to the driver. 612 * Description: 613 * This function configures the MAC to enable/disable both outgoing(TX) 614 * and incoming(RX) packets time stamping based on user input. 615 * Return Value: 616 * 0 on success and an appropriate -ve integer on failure. 617 */ 618 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 619 { 620 struct stmmac_priv *priv = netdev_priv(dev); 621 struct hwtstamp_config config; 622 u32 ptp_v2 = 0; 623 u32 tstamp_all = 0; 624 u32 ptp_over_ipv4_udp = 0; 625 u32 ptp_over_ipv6_udp = 0; 626 u32 ptp_over_ethernet = 0; 627 u32 snap_type_sel = 0; 628 u32 ts_master_en = 0; 629 u32 ts_event_en = 0; 630 631 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 632 netdev_alert(priv->dev, "No support for HW time stamping\n"); 633 priv->hwts_tx_en = 0; 634 priv->hwts_rx_en = 0; 635 636 return -EOPNOTSUPP; 637 } 638 639 if (copy_from_user(&config, ifr->ifr_data, 640 sizeof(config))) 641 return -EFAULT; 642 643 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 644 __func__, config.flags, config.tx_type, config.rx_filter); 645 646 if (config.tx_type != HWTSTAMP_TX_OFF && 647 config.tx_type != HWTSTAMP_TX_ON) 648 return -ERANGE; 649 650 if (priv->adv_ts) { 651 switch (config.rx_filter) { 652 case HWTSTAMP_FILTER_NONE: 653 /* time stamp no incoming packet at all */ 654 config.rx_filter = HWTSTAMP_FILTER_NONE; 655 break; 656 657 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 658 /* PTP v1, UDP, any kind of event packet */ 659 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 660 /* 'xmac' hardware can support Sync, Pdelay_Req and 661 * Pdelay_resp by setting bit14 and bits17/16 to 01 662 * This leaves Delay_Req timestamps out. 663 * Enable all events *and* general purpose message 664 * timestamping 665 */ 666 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 667 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 668 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 669 break; 670 671 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 672 /* PTP v1, UDP, Sync packet */ 673 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 674 /* take time stamp for SYNC messages only */ 675 ts_event_en = PTP_TCR_TSEVNTENA; 676 677 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 678 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 679 break; 680 681 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 682 /* PTP v1, UDP, Delay_req packet */ 683 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 684 /* take time stamp for Delay_Req messages only */ 685 ts_master_en = PTP_TCR_TSMSTRENA; 686 ts_event_en = PTP_TCR_TSEVNTENA; 687 688 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 689 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 690 break; 691 692 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 693 /* PTP v2, UDP, any kind of event packet */ 694 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 695 ptp_v2 = PTP_TCR_TSVER2ENA; 696 /* take time stamp for all event messages */ 697 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 698 699 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 700 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 701 break; 702 703 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 704 /* PTP v2, UDP, Sync packet */ 705 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 706 ptp_v2 = PTP_TCR_TSVER2ENA; 707 /* take time stamp for SYNC messages only */ 708 ts_event_en = PTP_TCR_TSEVNTENA; 709 710 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 711 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 712 break; 713 714 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 715 /* PTP v2, UDP, Delay_req packet */ 716 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 717 ptp_v2 = PTP_TCR_TSVER2ENA; 718 /* take time stamp for Delay_Req messages only */ 719 ts_master_en = PTP_TCR_TSMSTRENA; 720 ts_event_en = PTP_TCR_TSEVNTENA; 721 722 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 723 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 724 break; 725 726 case HWTSTAMP_FILTER_PTP_V2_EVENT: 727 /* PTP v2/802.AS1 any layer, any kind of event packet */ 728 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 729 ptp_v2 = PTP_TCR_TSVER2ENA; 730 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 731 if (priv->synopsys_id < DWMAC_CORE_4_10) 732 ts_event_en = PTP_TCR_TSEVNTENA; 733 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 734 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 735 ptp_over_ethernet = PTP_TCR_TSIPENA; 736 break; 737 738 case HWTSTAMP_FILTER_PTP_V2_SYNC: 739 /* PTP v2/802.AS1, any layer, Sync packet */ 740 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 741 ptp_v2 = PTP_TCR_TSVER2ENA; 742 /* take time stamp for SYNC messages only */ 743 ts_event_en = PTP_TCR_TSEVNTENA; 744 745 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 746 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 747 ptp_over_ethernet = PTP_TCR_TSIPENA; 748 break; 749 750 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 751 /* PTP v2/802.AS1, any layer, Delay_req packet */ 752 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 753 ptp_v2 = PTP_TCR_TSVER2ENA; 754 /* take time stamp for Delay_Req messages only */ 755 ts_master_en = PTP_TCR_TSMSTRENA; 756 ts_event_en = PTP_TCR_TSEVNTENA; 757 758 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 759 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 760 ptp_over_ethernet = PTP_TCR_TSIPENA; 761 break; 762 763 case HWTSTAMP_FILTER_NTP_ALL: 764 case HWTSTAMP_FILTER_ALL: 765 /* time stamp any incoming packet */ 766 config.rx_filter = HWTSTAMP_FILTER_ALL; 767 tstamp_all = PTP_TCR_TSENALL; 768 break; 769 770 default: 771 return -ERANGE; 772 } 773 } else { 774 switch (config.rx_filter) { 775 case HWTSTAMP_FILTER_NONE: 776 config.rx_filter = HWTSTAMP_FILTER_NONE; 777 break; 778 default: 779 /* PTP v1, UDP, any kind of event packet */ 780 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 781 break; 782 } 783 } 784 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 785 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 786 787 priv->systime_flags = STMMAC_HWTS_ACTIVE; 788 789 if (priv->hwts_tx_en || priv->hwts_rx_en) { 790 priv->systime_flags |= tstamp_all | ptp_v2 | 791 ptp_over_ethernet | ptp_over_ipv6_udp | 792 ptp_over_ipv4_udp | ts_event_en | 793 ts_master_en | snap_type_sel; 794 } 795 796 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 797 798 memcpy(&priv->tstamp_config, &config, sizeof(config)); 799 800 return copy_to_user(ifr->ifr_data, &config, 801 sizeof(config)) ? -EFAULT : 0; 802 } 803 804 /** 805 * stmmac_hwtstamp_get - read hardware timestamping. 806 * @dev: device pointer. 807 * @ifr: An IOCTL specific structure, that can contain a pointer to 808 * a proprietary structure used to pass information to the driver. 809 * Description: 810 * This function obtain the current hardware timestamping settings 811 * as requested. 812 */ 813 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 814 { 815 struct stmmac_priv *priv = netdev_priv(dev); 816 struct hwtstamp_config *config = &priv->tstamp_config; 817 818 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 819 return -EOPNOTSUPP; 820 821 return copy_to_user(ifr->ifr_data, config, 822 sizeof(*config)) ? -EFAULT : 0; 823 } 824 825 /** 826 * stmmac_init_tstamp_counter - init hardware timestamping counter 827 * @priv: driver private structure 828 * @systime_flags: timestamping flags 829 * Description: 830 * Initialize hardware counter for packet timestamping. 831 * This is valid as long as the interface is open and not suspended. 832 * Will be rerun after resuming from suspend, case in which the timestamping 833 * flags updated by stmmac_hwtstamp_set() also need to be restored. 834 */ 835 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) 836 { 837 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 838 struct timespec64 now; 839 u32 sec_inc = 0; 840 u64 temp = 0; 841 842 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 843 return -EOPNOTSUPP; 844 845 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 846 priv->systime_flags = systime_flags; 847 848 /* program Sub Second Increment reg */ 849 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 850 priv->plat->clk_ptp_rate, 851 xmac, &sec_inc); 852 temp = div_u64(1000000000ULL, sec_inc); 853 854 /* Store sub second increment for later use */ 855 priv->sub_second_inc = sec_inc; 856 857 /* calculate default added value: 858 * formula is : 859 * addend = (2^32)/freq_div_ratio; 860 * where, freq_div_ratio = 1e9ns/sec_inc 861 */ 862 temp = (u64)(temp << 32); 863 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 864 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 865 866 /* initialize system time */ 867 ktime_get_real_ts64(&now); 868 869 /* lower 32 bits of tv_sec are safe until y2106 */ 870 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 871 872 return 0; 873 } 874 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); 875 876 /** 877 * stmmac_init_ptp - init PTP 878 * @priv: driver private structure 879 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 880 * This is done by looking at the HW cap. register. 881 * This function also registers the ptp driver. 882 */ 883 static int stmmac_init_ptp(struct stmmac_priv *priv) 884 { 885 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 886 int ret; 887 888 if (priv->plat->ptp_clk_freq_config) 889 priv->plat->ptp_clk_freq_config(priv); 890 891 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); 892 if (ret) 893 return ret; 894 895 priv->adv_ts = 0; 896 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 897 if (xmac && priv->dma_cap.atime_stamp) 898 priv->adv_ts = 1; 899 /* Dwmac 3.x core with extend_desc can support adv_ts */ 900 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 901 priv->adv_ts = 1; 902 903 if (priv->dma_cap.time_stamp) 904 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 905 906 if (priv->adv_ts) 907 netdev_info(priv->dev, 908 "IEEE 1588-2008 Advanced Timestamp supported\n"); 909 910 priv->hwts_tx_en = 0; 911 priv->hwts_rx_en = 0; 912 913 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 914 stmmac_hwtstamp_correct_latency(priv, priv); 915 916 return 0; 917 } 918 919 static void stmmac_release_ptp(struct stmmac_priv *priv) 920 { 921 clk_disable_unprepare(priv->plat->clk_ptp_ref); 922 stmmac_ptp_unregister(priv); 923 } 924 925 /** 926 * stmmac_mac_flow_ctrl - Configure flow control in all queues 927 * @priv: driver private structure 928 * @duplex: duplex passed to the next function 929 * Description: It is used for configuring the flow control in all queues 930 */ 931 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 932 { 933 u32 tx_cnt = priv->plat->tx_queues_to_use; 934 935 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 936 priv->pause, tx_cnt); 937 } 938 939 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 940 phy_interface_t interface) 941 { 942 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 943 944 if (priv->hw->xpcs) 945 return &priv->hw->xpcs->pcs; 946 947 if (priv->hw->lynx_pcs) 948 return priv->hw->lynx_pcs; 949 950 return NULL; 951 } 952 953 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 954 const struct phylink_link_state *state) 955 { 956 /* Nothing to do, xpcs_config() handles everything */ 957 } 958 959 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) 960 { 961 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 962 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 963 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 964 bool *hs_enable = &fpe_cfg->hs_enable; 965 966 if (is_up && *hs_enable) { 967 stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); 968 } else { 969 *lo_state = FPE_STATE_OFF; 970 *lp_state = FPE_STATE_OFF; 971 } 972 } 973 974 static void stmmac_mac_link_down(struct phylink_config *config, 975 unsigned int mode, phy_interface_t interface) 976 { 977 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 978 979 stmmac_mac_set(priv, priv->ioaddr, false); 980 priv->eee_active = false; 981 priv->tx_lpi_enabled = false; 982 priv->eee_enabled = stmmac_eee_init(priv); 983 stmmac_set_eee_pls(priv, priv->hw, false); 984 985 if (priv->dma_cap.fpesel) 986 stmmac_fpe_link_state_handle(priv, false); 987 } 988 989 static void stmmac_mac_link_up(struct phylink_config *config, 990 struct phy_device *phy, 991 unsigned int mode, phy_interface_t interface, 992 int speed, int duplex, 993 bool tx_pause, bool rx_pause) 994 { 995 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 996 u32 old_ctrl, ctrl; 997 998 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 999 priv->plat->serdes_powerup) 1000 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 1001 1002 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1003 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1004 1005 if (interface == PHY_INTERFACE_MODE_USXGMII) { 1006 switch (speed) { 1007 case SPEED_10000: 1008 ctrl |= priv->hw->link.xgmii.speed10000; 1009 break; 1010 case SPEED_5000: 1011 ctrl |= priv->hw->link.xgmii.speed5000; 1012 break; 1013 case SPEED_2500: 1014 ctrl |= priv->hw->link.xgmii.speed2500; 1015 break; 1016 default: 1017 return; 1018 } 1019 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1020 switch (speed) { 1021 case SPEED_100000: 1022 ctrl |= priv->hw->link.xlgmii.speed100000; 1023 break; 1024 case SPEED_50000: 1025 ctrl |= priv->hw->link.xlgmii.speed50000; 1026 break; 1027 case SPEED_40000: 1028 ctrl |= priv->hw->link.xlgmii.speed40000; 1029 break; 1030 case SPEED_25000: 1031 ctrl |= priv->hw->link.xlgmii.speed25000; 1032 break; 1033 case SPEED_10000: 1034 ctrl |= priv->hw->link.xgmii.speed10000; 1035 break; 1036 case SPEED_2500: 1037 ctrl |= priv->hw->link.speed2500; 1038 break; 1039 case SPEED_1000: 1040 ctrl |= priv->hw->link.speed1000; 1041 break; 1042 default: 1043 return; 1044 } 1045 } else { 1046 switch (speed) { 1047 case SPEED_2500: 1048 ctrl |= priv->hw->link.speed2500; 1049 break; 1050 case SPEED_1000: 1051 ctrl |= priv->hw->link.speed1000; 1052 break; 1053 case SPEED_100: 1054 ctrl |= priv->hw->link.speed100; 1055 break; 1056 case SPEED_10: 1057 ctrl |= priv->hw->link.speed10; 1058 break; 1059 default: 1060 return; 1061 } 1062 } 1063 1064 priv->speed = speed; 1065 1066 if (priv->plat->fix_mac_speed) 1067 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); 1068 1069 if (!duplex) 1070 ctrl &= ~priv->hw->link.duplex; 1071 else 1072 ctrl |= priv->hw->link.duplex; 1073 1074 /* Flow Control operation */ 1075 if (rx_pause && tx_pause) 1076 priv->flow_ctrl = FLOW_AUTO; 1077 else if (rx_pause && !tx_pause) 1078 priv->flow_ctrl = FLOW_RX; 1079 else if (!rx_pause && tx_pause) 1080 priv->flow_ctrl = FLOW_TX; 1081 else 1082 priv->flow_ctrl = FLOW_OFF; 1083 1084 stmmac_mac_flow_ctrl(priv, duplex); 1085 1086 if (ctrl != old_ctrl) 1087 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1088 1089 stmmac_mac_set(priv, priv->ioaddr, true); 1090 if (phy && priv->dma_cap.eee) { 1091 priv->eee_active = 1092 phy_init_eee(phy, !(priv->plat->flags & 1093 STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) >= 0; 1094 priv->eee_enabled = stmmac_eee_init(priv); 1095 priv->tx_lpi_enabled = priv->eee_enabled; 1096 stmmac_set_eee_pls(priv, priv->hw, true); 1097 } 1098 1099 if (priv->dma_cap.fpesel) 1100 stmmac_fpe_link_state_handle(priv, true); 1101 1102 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1103 stmmac_hwtstamp_correct_latency(priv, priv); 1104 } 1105 1106 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1107 .mac_select_pcs = stmmac_mac_select_pcs, 1108 .mac_config = stmmac_mac_config, 1109 .mac_link_down = stmmac_mac_link_down, 1110 .mac_link_up = stmmac_mac_link_up, 1111 }; 1112 1113 /** 1114 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1115 * @priv: driver private structure 1116 * Description: this is to verify if the HW supports the PCS. 1117 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1118 * configured for the TBI, RTBI, or SGMII PHY interface. 1119 */ 1120 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1121 { 1122 int interface = priv->plat->mac_interface; 1123 1124 if (priv->dma_cap.pcs) { 1125 if ((interface == PHY_INTERFACE_MODE_RGMII) || 1126 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 1127 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 1128 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 1129 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 1130 priv->hw->pcs = STMMAC_PCS_RGMII; 1131 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 1132 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1133 priv->hw->pcs = STMMAC_PCS_SGMII; 1134 } 1135 } 1136 } 1137 1138 /** 1139 * stmmac_init_phy - PHY initialization 1140 * @dev: net device structure 1141 * Description: it initializes the driver's PHY state, and attaches the PHY 1142 * to the mac driver. 1143 * Return value: 1144 * 0 on success 1145 */ 1146 static int stmmac_init_phy(struct net_device *dev) 1147 { 1148 struct stmmac_priv *priv = netdev_priv(dev); 1149 struct fwnode_handle *phy_fwnode; 1150 struct fwnode_handle *fwnode; 1151 int ret; 1152 1153 if (!phylink_expects_phy(priv->phylink)) 1154 return 0; 1155 1156 fwnode = priv->plat->port_node; 1157 if (!fwnode) 1158 fwnode = dev_fwnode(priv->device); 1159 1160 if (fwnode) 1161 phy_fwnode = fwnode_get_phy_node(fwnode); 1162 else 1163 phy_fwnode = NULL; 1164 1165 /* Some DT bindings do not set-up the PHY handle. Let's try to 1166 * manually parse it 1167 */ 1168 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1169 int addr = priv->plat->phy_addr; 1170 struct phy_device *phydev; 1171 1172 if (addr < 0) { 1173 netdev_err(priv->dev, "no phy found\n"); 1174 return -ENODEV; 1175 } 1176 1177 phydev = mdiobus_get_phy(priv->mii, addr); 1178 if (!phydev) { 1179 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1180 return -ENODEV; 1181 } 1182 1183 ret = phylink_connect_phy(priv->phylink, phydev); 1184 } else { 1185 fwnode_handle_put(phy_fwnode); 1186 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); 1187 } 1188 1189 if (!priv->plat->pmt) { 1190 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 1191 1192 phylink_ethtool_get_wol(priv->phylink, &wol); 1193 device_set_wakeup_capable(priv->device, !!wol.supported); 1194 device_set_wakeup_enable(priv->device, !!wol.wolopts); 1195 } 1196 1197 return ret; 1198 } 1199 1200 static int stmmac_phy_setup(struct stmmac_priv *priv) 1201 { 1202 struct stmmac_mdio_bus_data *mdio_bus_data; 1203 int mode = priv->plat->phy_interface; 1204 struct fwnode_handle *fwnode; 1205 struct phylink *phylink; 1206 int max_speed; 1207 1208 priv->phylink_config.dev = &priv->dev->dev; 1209 priv->phylink_config.type = PHYLINK_NETDEV; 1210 priv->phylink_config.mac_managed_pm = true; 1211 1212 mdio_bus_data = priv->plat->mdio_bus_data; 1213 if (mdio_bus_data) 1214 priv->phylink_config.ovr_an_inband = 1215 mdio_bus_data->xpcs_an_inband; 1216 1217 /* Set the platform/firmware specified interface mode. Note, phylink 1218 * deals with the PHY interface mode, not the MAC interface mode. 1219 */ 1220 __set_bit(mode, priv->phylink_config.supported_interfaces); 1221 1222 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1223 if (priv->hw->xpcs) 1224 xpcs_get_interfaces(priv->hw->xpcs, 1225 priv->phylink_config.supported_interfaces); 1226 1227 priv->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | 1228 MAC_10FD | MAC_100FD | 1229 MAC_1000FD; 1230 1231 /* Half-Duplex can only work with single queue */ 1232 if (priv->plat->tx_queues_to_use <= 1) 1233 priv->phylink_config.mac_capabilities |= MAC_10HD | MAC_100HD | 1234 MAC_1000HD; 1235 1236 /* Get the MAC specific capabilities */ 1237 stmmac_mac_phylink_get_caps(priv); 1238 1239 max_speed = priv->plat->max_speed; 1240 if (max_speed) 1241 phylink_limit_mac_speed(&priv->phylink_config, max_speed); 1242 1243 fwnode = priv->plat->port_node; 1244 if (!fwnode) 1245 fwnode = dev_fwnode(priv->device); 1246 1247 phylink = phylink_create(&priv->phylink_config, fwnode, 1248 mode, &stmmac_phylink_mac_ops); 1249 if (IS_ERR(phylink)) 1250 return PTR_ERR(phylink); 1251 1252 priv->phylink = phylink; 1253 return 0; 1254 } 1255 1256 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1257 struct stmmac_dma_conf *dma_conf) 1258 { 1259 u32 rx_cnt = priv->plat->rx_queues_to_use; 1260 unsigned int desc_size; 1261 void *head_rx; 1262 u32 queue; 1263 1264 /* Display RX rings */ 1265 for (queue = 0; queue < rx_cnt; queue++) { 1266 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1267 1268 pr_info("\tRX Queue %u rings\n", queue); 1269 1270 if (priv->extend_desc) { 1271 head_rx = (void *)rx_q->dma_erx; 1272 desc_size = sizeof(struct dma_extended_desc); 1273 } else { 1274 head_rx = (void *)rx_q->dma_rx; 1275 desc_size = sizeof(struct dma_desc); 1276 } 1277 1278 /* Display RX ring */ 1279 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1280 rx_q->dma_rx_phy, desc_size); 1281 } 1282 } 1283 1284 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1285 struct stmmac_dma_conf *dma_conf) 1286 { 1287 u32 tx_cnt = priv->plat->tx_queues_to_use; 1288 unsigned int desc_size; 1289 void *head_tx; 1290 u32 queue; 1291 1292 /* Display TX rings */ 1293 for (queue = 0; queue < tx_cnt; queue++) { 1294 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1295 1296 pr_info("\tTX Queue %d rings\n", queue); 1297 1298 if (priv->extend_desc) { 1299 head_tx = (void *)tx_q->dma_etx; 1300 desc_size = sizeof(struct dma_extended_desc); 1301 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1302 head_tx = (void *)tx_q->dma_entx; 1303 desc_size = sizeof(struct dma_edesc); 1304 } else { 1305 head_tx = (void *)tx_q->dma_tx; 1306 desc_size = sizeof(struct dma_desc); 1307 } 1308 1309 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1310 tx_q->dma_tx_phy, desc_size); 1311 } 1312 } 1313 1314 static void stmmac_display_rings(struct stmmac_priv *priv, 1315 struct stmmac_dma_conf *dma_conf) 1316 { 1317 /* Display RX ring */ 1318 stmmac_display_rx_rings(priv, dma_conf); 1319 1320 /* Display TX ring */ 1321 stmmac_display_tx_rings(priv, dma_conf); 1322 } 1323 1324 static int stmmac_set_bfsize(int mtu, int bufsize) 1325 { 1326 int ret = bufsize; 1327 1328 if (mtu >= BUF_SIZE_8KiB) 1329 ret = BUF_SIZE_16KiB; 1330 else if (mtu >= BUF_SIZE_4KiB) 1331 ret = BUF_SIZE_8KiB; 1332 else if (mtu >= BUF_SIZE_2KiB) 1333 ret = BUF_SIZE_4KiB; 1334 else if (mtu > DEFAULT_BUFSIZE) 1335 ret = BUF_SIZE_2KiB; 1336 else 1337 ret = DEFAULT_BUFSIZE; 1338 1339 return ret; 1340 } 1341 1342 /** 1343 * stmmac_clear_rx_descriptors - clear RX descriptors 1344 * @priv: driver private structure 1345 * @dma_conf: structure to take the dma data 1346 * @queue: RX queue index 1347 * Description: this function is called to clear the RX descriptors 1348 * in case of both basic and extended descriptors are used. 1349 */ 1350 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1351 struct stmmac_dma_conf *dma_conf, 1352 u32 queue) 1353 { 1354 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1355 int i; 1356 1357 /* Clear the RX descriptors */ 1358 for (i = 0; i < dma_conf->dma_rx_size; i++) 1359 if (priv->extend_desc) 1360 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1361 priv->use_riwt, priv->mode, 1362 (i == dma_conf->dma_rx_size - 1), 1363 dma_conf->dma_buf_sz); 1364 else 1365 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1366 priv->use_riwt, priv->mode, 1367 (i == dma_conf->dma_rx_size - 1), 1368 dma_conf->dma_buf_sz); 1369 } 1370 1371 /** 1372 * stmmac_clear_tx_descriptors - clear tx descriptors 1373 * @priv: driver private structure 1374 * @dma_conf: structure to take the dma data 1375 * @queue: TX queue index. 1376 * Description: this function is called to clear the TX descriptors 1377 * in case of both basic and extended descriptors are used. 1378 */ 1379 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1380 struct stmmac_dma_conf *dma_conf, 1381 u32 queue) 1382 { 1383 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1384 int i; 1385 1386 /* Clear the TX descriptors */ 1387 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1388 int last = (i == (dma_conf->dma_tx_size - 1)); 1389 struct dma_desc *p; 1390 1391 if (priv->extend_desc) 1392 p = &tx_q->dma_etx[i].basic; 1393 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1394 p = &tx_q->dma_entx[i].basic; 1395 else 1396 p = &tx_q->dma_tx[i]; 1397 1398 stmmac_init_tx_desc(priv, p, priv->mode, last); 1399 } 1400 } 1401 1402 /** 1403 * stmmac_clear_descriptors - clear descriptors 1404 * @priv: driver private structure 1405 * @dma_conf: structure to take the dma data 1406 * Description: this function is called to clear the TX and RX descriptors 1407 * in case of both basic and extended descriptors are used. 1408 */ 1409 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1410 struct stmmac_dma_conf *dma_conf) 1411 { 1412 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1413 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1414 u32 queue; 1415 1416 /* Clear the RX descriptors */ 1417 for (queue = 0; queue < rx_queue_cnt; queue++) 1418 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1419 1420 /* Clear the TX descriptors */ 1421 for (queue = 0; queue < tx_queue_cnt; queue++) 1422 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1423 } 1424 1425 /** 1426 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1427 * @priv: driver private structure 1428 * @dma_conf: structure to take the dma data 1429 * @p: descriptor pointer 1430 * @i: descriptor index 1431 * @flags: gfp flag 1432 * @queue: RX queue index 1433 * Description: this function is called to allocate a receive buffer, perform 1434 * the DMA mapping and init the descriptor. 1435 */ 1436 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1437 struct stmmac_dma_conf *dma_conf, 1438 struct dma_desc *p, 1439 int i, gfp_t flags, u32 queue) 1440 { 1441 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1442 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1443 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1444 1445 if (priv->dma_cap.host_dma_width <= 32) 1446 gfp |= GFP_DMA32; 1447 1448 if (!buf->page) { 1449 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1450 if (!buf->page) 1451 return -ENOMEM; 1452 buf->page_offset = stmmac_rx_offset(priv); 1453 } 1454 1455 if (priv->sph && !buf->sec_page) { 1456 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1457 if (!buf->sec_page) 1458 return -ENOMEM; 1459 1460 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1461 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1462 } else { 1463 buf->sec_page = NULL; 1464 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1465 } 1466 1467 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1468 1469 stmmac_set_desc_addr(priv, p, buf->addr); 1470 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1471 stmmac_init_desc3(priv, p); 1472 1473 return 0; 1474 } 1475 1476 /** 1477 * stmmac_free_rx_buffer - free RX dma buffers 1478 * @priv: private structure 1479 * @rx_q: RX queue 1480 * @i: buffer index. 1481 */ 1482 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1483 struct stmmac_rx_queue *rx_q, 1484 int i) 1485 { 1486 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1487 1488 if (buf->page) 1489 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1490 buf->page = NULL; 1491 1492 if (buf->sec_page) 1493 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1494 buf->sec_page = NULL; 1495 } 1496 1497 /** 1498 * stmmac_free_tx_buffer - free RX dma buffers 1499 * @priv: private structure 1500 * @dma_conf: structure to take the dma data 1501 * @queue: RX queue index 1502 * @i: buffer index. 1503 */ 1504 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1505 struct stmmac_dma_conf *dma_conf, 1506 u32 queue, int i) 1507 { 1508 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1509 1510 if (tx_q->tx_skbuff_dma[i].buf && 1511 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1512 if (tx_q->tx_skbuff_dma[i].map_as_page) 1513 dma_unmap_page(priv->device, 1514 tx_q->tx_skbuff_dma[i].buf, 1515 tx_q->tx_skbuff_dma[i].len, 1516 DMA_TO_DEVICE); 1517 else 1518 dma_unmap_single(priv->device, 1519 tx_q->tx_skbuff_dma[i].buf, 1520 tx_q->tx_skbuff_dma[i].len, 1521 DMA_TO_DEVICE); 1522 } 1523 1524 if (tx_q->xdpf[i] && 1525 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1526 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1527 xdp_return_frame(tx_q->xdpf[i]); 1528 tx_q->xdpf[i] = NULL; 1529 } 1530 1531 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1532 tx_q->xsk_frames_done++; 1533 1534 if (tx_q->tx_skbuff[i] && 1535 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1536 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1537 tx_q->tx_skbuff[i] = NULL; 1538 } 1539 1540 tx_q->tx_skbuff_dma[i].buf = 0; 1541 tx_q->tx_skbuff_dma[i].map_as_page = false; 1542 } 1543 1544 /** 1545 * dma_free_rx_skbufs - free RX dma buffers 1546 * @priv: private structure 1547 * @dma_conf: structure to take the dma data 1548 * @queue: RX queue index 1549 */ 1550 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1551 struct stmmac_dma_conf *dma_conf, 1552 u32 queue) 1553 { 1554 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1555 int i; 1556 1557 for (i = 0; i < dma_conf->dma_rx_size; i++) 1558 stmmac_free_rx_buffer(priv, rx_q, i); 1559 } 1560 1561 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1562 struct stmmac_dma_conf *dma_conf, 1563 u32 queue, gfp_t flags) 1564 { 1565 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1566 int i; 1567 1568 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1569 struct dma_desc *p; 1570 int ret; 1571 1572 if (priv->extend_desc) 1573 p = &((rx_q->dma_erx + i)->basic); 1574 else 1575 p = rx_q->dma_rx + i; 1576 1577 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1578 queue); 1579 if (ret) 1580 return ret; 1581 1582 rx_q->buf_alloc_num++; 1583 } 1584 1585 return 0; 1586 } 1587 1588 /** 1589 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1590 * @priv: private structure 1591 * @dma_conf: structure to take the dma data 1592 * @queue: RX queue index 1593 */ 1594 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1595 struct stmmac_dma_conf *dma_conf, 1596 u32 queue) 1597 { 1598 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1599 int i; 1600 1601 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1602 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1603 1604 if (!buf->xdp) 1605 continue; 1606 1607 xsk_buff_free(buf->xdp); 1608 buf->xdp = NULL; 1609 } 1610 } 1611 1612 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1613 struct stmmac_dma_conf *dma_conf, 1614 u32 queue) 1615 { 1616 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1617 int i; 1618 1619 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1620 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1621 * use this macro to make sure no size violations. 1622 */ 1623 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1624 1625 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1626 struct stmmac_rx_buffer *buf; 1627 dma_addr_t dma_addr; 1628 struct dma_desc *p; 1629 1630 if (priv->extend_desc) 1631 p = (struct dma_desc *)(rx_q->dma_erx + i); 1632 else 1633 p = rx_q->dma_rx + i; 1634 1635 buf = &rx_q->buf_pool[i]; 1636 1637 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1638 if (!buf->xdp) 1639 return -ENOMEM; 1640 1641 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1642 stmmac_set_desc_addr(priv, p, dma_addr); 1643 rx_q->buf_alloc_num++; 1644 } 1645 1646 return 0; 1647 } 1648 1649 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1650 { 1651 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1652 return NULL; 1653 1654 return xsk_get_pool_from_qid(priv->dev, queue); 1655 } 1656 1657 /** 1658 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1659 * @priv: driver private structure 1660 * @dma_conf: structure to take the dma data 1661 * @queue: RX queue index 1662 * @flags: gfp flag. 1663 * Description: this function initializes the DMA RX descriptors 1664 * and allocates the socket buffers. It supports the chained and ring 1665 * modes. 1666 */ 1667 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1668 struct stmmac_dma_conf *dma_conf, 1669 u32 queue, gfp_t flags) 1670 { 1671 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1672 int ret; 1673 1674 netif_dbg(priv, probe, priv->dev, 1675 "(%s) dma_rx_phy=0x%08x\n", __func__, 1676 (u32)rx_q->dma_rx_phy); 1677 1678 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1679 1680 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1681 1682 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1683 1684 if (rx_q->xsk_pool) { 1685 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1686 MEM_TYPE_XSK_BUFF_POOL, 1687 NULL)); 1688 netdev_info(priv->dev, 1689 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1690 rx_q->queue_index); 1691 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1692 } else { 1693 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1694 MEM_TYPE_PAGE_POOL, 1695 rx_q->page_pool)); 1696 netdev_info(priv->dev, 1697 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1698 rx_q->queue_index); 1699 } 1700 1701 if (rx_q->xsk_pool) { 1702 /* RX XDP ZC buffer pool may not be populated, e.g. 1703 * xdpsock TX-only. 1704 */ 1705 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1706 } else { 1707 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1708 if (ret < 0) 1709 return -ENOMEM; 1710 } 1711 1712 /* Setup the chained descriptor addresses */ 1713 if (priv->mode == STMMAC_CHAIN_MODE) { 1714 if (priv->extend_desc) 1715 stmmac_mode_init(priv, rx_q->dma_erx, 1716 rx_q->dma_rx_phy, 1717 dma_conf->dma_rx_size, 1); 1718 else 1719 stmmac_mode_init(priv, rx_q->dma_rx, 1720 rx_q->dma_rx_phy, 1721 dma_conf->dma_rx_size, 0); 1722 } 1723 1724 return 0; 1725 } 1726 1727 static int init_dma_rx_desc_rings(struct net_device *dev, 1728 struct stmmac_dma_conf *dma_conf, 1729 gfp_t flags) 1730 { 1731 struct stmmac_priv *priv = netdev_priv(dev); 1732 u32 rx_count = priv->plat->rx_queues_to_use; 1733 int queue; 1734 int ret; 1735 1736 /* RX INITIALIZATION */ 1737 netif_dbg(priv, probe, priv->dev, 1738 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1739 1740 for (queue = 0; queue < rx_count; queue++) { 1741 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1742 if (ret) 1743 goto err_init_rx_buffers; 1744 } 1745 1746 return 0; 1747 1748 err_init_rx_buffers: 1749 while (queue >= 0) { 1750 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1751 1752 if (rx_q->xsk_pool) 1753 dma_free_rx_xskbufs(priv, dma_conf, queue); 1754 else 1755 dma_free_rx_skbufs(priv, dma_conf, queue); 1756 1757 rx_q->buf_alloc_num = 0; 1758 rx_q->xsk_pool = NULL; 1759 1760 queue--; 1761 } 1762 1763 return ret; 1764 } 1765 1766 /** 1767 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1768 * @priv: driver private structure 1769 * @dma_conf: structure to take the dma data 1770 * @queue: TX queue index 1771 * Description: this function initializes the DMA TX descriptors 1772 * and allocates the socket buffers. It supports the chained and ring 1773 * modes. 1774 */ 1775 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 1776 struct stmmac_dma_conf *dma_conf, 1777 u32 queue) 1778 { 1779 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1780 int i; 1781 1782 netif_dbg(priv, probe, priv->dev, 1783 "(%s) dma_tx_phy=0x%08x\n", __func__, 1784 (u32)tx_q->dma_tx_phy); 1785 1786 /* Setup the chained descriptor addresses */ 1787 if (priv->mode == STMMAC_CHAIN_MODE) { 1788 if (priv->extend_desc) 1789 stmmac_mode_init(priv, tx_q->dma_etx, 1790 tx_q->dma_tx_phy, 1791 dma_conf->dma_tx_size, 1); 1792 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1793 stmmac_mode_init(priv, tx_q->dma_tx, 1794 tx_q->dma_tx_phy, 1795 dma_conf->dma_tx_size, 0); 1796 } 1797 1798 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1799 1800 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1801 struct dma_desc *p; 1802 1803 if (priv->extend_desc) 1804 p = &((tx_q->dma_etx + i)->basic); 1805 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1806 p = &((tx_q->dma_entx + i)->basic); 1807 else 1808 p = tx_q->dma_tx + i; 1809 1810 stmmac_clear_desc(priv, p); 1811 1812 tx_q->tx_skbuff_dma[i].buf = 0; 1813 tx_q->tx_skbuff_dma[i].map_as_page = false; 1814 tx_q->tx_skbuff_dma[i].len = 0; 1815 tx_q->tx_skbuff_dma[i].last_segment = false; 1816 tx_q->tx_skbuff[i] = NULL; 1817 } 1818 1819 return 0; 1820 } 1821 1822 static int init_dma_tx_desc_rings(struct net_device *dev, 1823 struct stmmac_dma_conf *dma_conf) 1824 { 1825 struct stmmac_priv *priv = netdev_priv(dev); 1826 u32 tx_queue_cnt; 1827 u32 queue; 1828 1829 tx_queue_cnt = priv->plat->tx_queues_to_use; 1830 1831 for (queue = 0; queue < tx_queue_cnt; queue++) 1832 __init_dma_tx_desc_rings(priv, dma_conf, queue); 1833 1834 return 0; 1835 } 1836 1837 /** 1838 * init_dma_desc_rings - init the RX/TX descriptor rings 1839 * @dev: net device structure 1840 * @dma_conf: structure to take the dma data 1841 * @flags: gfp flag. 1842 * Description: this function initializes the DMA RX/TX descriptors 1843 * and allocates the socket buffers. It supports the chained and ring 1844 * modes. 1845 */ 1846 static int init_dma_desc_rings(struct net_device *dev, 1847 struct stmmac_dma_conf *dma_conf, 1848 gfp_t flags) 1849 { 1850 struct stmmac_priv *priv = netdev_priv(dev); 1851 int ret; 1852 1853 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 1854 if (ret) 1855 return ret; 1856 1857 ret = init_dma_tx_desc_rings(dev, dma_conf); 1858 1859 stmmac_clear_descriptors(priv, dma_conf); 1860 1861 if (netif_msg_hw(priv)) 1862 stmmac_display_rings(priv, dma_conf); 1863 1864 return ret; 1865 } 1866 1867 /** 1868 * dma_free_tx_skbufs - free TX dma buffers 1869 * @priv: private structure 1870 * @dma_conf: structure to take the dma data 1871 * @queue: TX queue index 1872 */ 1873 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 1874 struct stmmac_dma_conf *dma_conf, 1875 u32 queue) 1876 { 1877 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1878 int i; 1879 1880 tx_q->xsk_frames_done = 0; 1881 1882 for (i = 0; i < dma_conf->dma_tx_size; i++) 1883 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 1884 1885 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 1886 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 1887 tx_q->xsk_frames_done = 0; 1888 tx_q->xsk_pool = NULL; 1889 } 1890 } 1891 1892 /** 1893 * stmmac_free_tx_skbufs - free TX skb buffers 1894 * @priv: private structure 1895 */ 1896 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 1897 { 1898 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1899 u32 queue; 1900 1901 for (queue = 0; queue < tx_queue_cnt; queue++) 1902 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 1903 } 1904 1905 /** 1906 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 1907 * @priv: private structure 1908 * @dma_conf: structure to take the dma data 1909 * @queue: RX queue index 1910 */ 1911 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 1912 struct stmmac_dma_conf *dma_conf, 1913 u32 queue) 1914 { 1915 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1916 1917 /* Release the DMA RX socket buffers */ 1918 if (rx_q->xsk_pool) 1919 dma_free_rx_xskbufs(priv, dma_conf, queue); 1920 else 1921 dma_free_rx_skbufs(priv, dma_conf, queue); 1922 1923 rx_q->buf_alloc_num = 0; 1924 rx_q->xsk_pool = NULL; 1925 1926 /* Free DMA regions of consistent memory previously allocated */ 1927 if (!priv->extend_desc) 1928 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1929 sizeof(struct dma_desc), 1930 rx_q->dma_rx, rx_q->dma_rx_phy); 1931 else 1932 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1933 sizeof(struct dma_extended_desc), 1934 rx_q->dma_erx, rx_q->dma_rx_phy); 1935 1936 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 1937 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 1938 1939 kfree(rx_q->buf_pool); 1940 if (rx_q->page_pool) 1941 page_pool_destroy(rx_q->page_pool); 1942 } 1943 1944 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 1945 struct stmmac_dma_conf *dma_conf) 1946 { 1947 u32 rx_count = priv->plat->rx_queues_to_use; 1948 u32 queue; 1949 1950 /* Free RX queue resources */ 1951 for (queue = 0; queue < rx_count; queue++) 1952 __free_dma_rx_desc_resources(priv, dma_conf, queue); 1953 } 1954 1955 /** 1956 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 1957 * @priv: private structure 1958 * @dma_conf: structure to take the dma data 1959 * @queue: TX queue index 1960 */ 1961 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 1962 struct stmmac_dma_conf *dma_conf, 1963 u32 queue) 1964 { 1965 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1966 size_t size; 1967 void *addr; 1968 1969 /* Release the DMA TX socket buffers */ 1970 dma_free_tx_skbufs(priv, dma_conf, queue); 1971 1972 if (priv->extend_desc) { 1973 size = sizeof(struct dma_extended_desc); 1974 addr = tx_q->dma_etx; 1975 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1976 size = sizeof(struct dma_edesc); 1977 addr = tx_q->dma_entx; 1978 } else { 1979 size = sizeof(struct dma_desc); 1980 addr = tx_q->dma_tx; 1981 } 1982 1983 size *= dma_conf->dma_tx_size; 1984 1985 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 1986 1987 kfree(tx_q->tx_skbuff_dma); 1988 kfree(tx_q->tx_skbuff); 1989 } 1990 1991 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 1992 struct stmmac_dma_conf *dma_conf) 1993 { 1994 u32 tx_count = priv->plat->tx_queues_to_use; 1995 u32 queue; 1996 1997 /* Free TX queue resources */ 1998 for (queue = 0; queue < tx_count; queue++) 1999 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2000 } 2001 2002 /** 2003 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2004 * @priv: private structure 2005 * @dma_conf: structure to take the dma data 2006 * @queue: RX queue index 2007 * Description: according to which descriptor can be used (extend or basic) 2008 * this function allocates the resources for TX and RX paths. In case of 2009 * reception, for example, it pre-allocated the RX socket buffer in order to 2010 * allow zero-copy mechanism. 2011 */ 2012 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2013 struct stmmac_dma_conf *dma_conf, 2014 u32 queue) 2015 { 2016 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2017 struct stmmac_channel *ch = &priv->channel[queue]; 2018 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2019 struct page_pool_params pp_params = { 0 }; 2020 unsigned int num_pages; 2021 unsigned int napi_id; 2022 int ret; 2023 2024 rx_q->queue_index = queue; 2025 rx_q->priv_data = priv; 2026 2027 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2028 pp_params.pool_size = dma_conf->dma_rx_size; 2029 num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); 2030 pp_params.order = ilog2(num_pages); 2031 pp_params.nid = dev_to_node(priv->device); 2032 pp_params.dev = priv->device; 2033 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2034 pp_params.offset = stmmac_rx_offset(priv); 2035 pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages); 2036 2037 rx_q->page_pool = page_pool_create(&pp_params); 2038 if (IS_ERR(rx_q->page_pool)) { 2039 ret = PTR_ERR(rx_q->page_pool); 2040 rx_q->page_pool = NULL; 2041 return ret; 2042 } 2043 2044 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, 2045 sizeof(*rx_q->buf_pool), 2046 GFP_KERNEL); 2047 if (!rx_q->buf_pool) 2048 return -ENOMEM; 2049 2050 if (priv->extend_desc) { 2051 rx_q->dma_erx = dma_alloc_coherent(priv->device, 2052 dma_conf->dma_rx_size * 2053 sizeof(struct dma_extended_desc), 2054 &rx_q->dma_rx_phy, 2055 GFP_KERNEL); 2056 if (!rx_q->dma_erx) 2057 return -ENOMEM; 2058 2059 } else { 2060 rx_q->dma_rx = dma_alloc_coherent(priv->device, 2061 dma_conf->dma_rx_size * 2062 sizeof(struct dma_desc), 2063 &rx_q->dma_rx_phy, 2064 GFP_KERNEL); 2065 if (!rx_q->dma_rx) 2066 return -ENOMEM; 2067 } 2068 2069 if (stmmac_xdp_is_enabled(priv) && 2070 test_bit(queue, priv->af_xdp_zc_qps)) 2071 napi_id = ch->rxtx_napi.napi_id; 2072 else 2073 napi_id = ch->rx_napi.napi_id; 2074 2075 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 2076 rx_q->queue_index, 2077 napi_id); 2078 if (ret) { 2079 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2080 return -EINVAL; 2081 } 2082 2083 return 0; 2084 } 2085 2086 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2087 struct stmmac_dma_conf *dma_conf) 2088 { 2089 u32 rx_count = priv->plat->rx_queues_to_use; 2090 u32 queue; 2091 int ret; 2092 2093 /* RX queues buffers and DMA */ 2094 for (queue = 0; queue < rx_count; queue++) { 2095 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2096 if (ret) 2097 goto err_dma; 2098 } 2099 2100 return 0; 2101 2102 err_dma: 2103 free_dma_rx_desc_resources(priv, dma_conf); 2104 2105 return ret; 2106 } 2107 2108 /** 2109 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2110 * @priv: private structure 2111 * @dma_conf: structure to take the dma data 2112 * @queue: TX queue index 2113 * Description: according to which descriptor can be used (extend or basic) 2114 * this function allocates the resources for TX and RX paths. In case of 2115 * reception, for example, it pre-allocated the RX socket buffer in order to 2116 * allow zero-copy mechanism. 2117 */ 2118 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2119 struct stmmac_dma_conf *dma_conf, 2120 u32 queue) 2121 { 2122 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2123 size_t size; 2124 void *addr; 2125 2126 tx_q->queue_index = queue; 2127 tx_q->priv_data = priv; 2128 2129 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, 2130 sizeof(*tx_q->tx_skbuff_dma), 2131 GFP_KERNEL); 2132 if (!tx_q->tx_skbuff_dma) 2133 return -ENOMEM; 2134 2135 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, 2136 sizeof(struct sk_buff *), 2137 GFP_KERNEL); 2138 if (!tx_q->tx_skbuff) 2139 return -ENOMEM; 2140 2141 if (priv->extend_desc) 2142 size = sizeof(struct dma_extended_desc); 2143 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2144 size = sizeof(struct dma_edesc); 2145 else 2146 size = sizeof(struct dma_desc); 2147 2148 size *= dma_conf->dma_tx_size; 2149 2150 addr = dma_alloc_coherent(priv->device, size, 2151 &tx_q->dma_tx_phy, GFP_KERNEL); 2152 if (!addr) 2153 return -ENOMEM; 2154 2155 if (priv->extend_desc) 2156 tx_q->dma_etx = addr; 2157 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2158 tx_q->dma_entx = addr; 2159 else 2160 tx_q->dma_tx = addr; 2161 2162 return 0; 2163 } 2164 2165 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2166 struct stmmac_dma_conf *dma_conf) 2167 { 2168 u32 tx_count = priv->plat->tx_queues_to_use; 2169 u32 queue; 2170 int ret; 2171 2172 /* TX queues buffers and DMA */ 2173 for (queue = 0; queue < tx_count; queue++) { 2174 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2175 if (ret) 2176 goto err_dma; 2177 } 2178 2179 return 0; 2180 2181 err_dma: 2182 free_dma_tx_desc_resources(priv, dma_conf); 2183 return ret; 2184 } 2185 2186 /** 2187 * alloc_dma_desc_resources - alloc TX/RX resources. 2188 * @priv: private structure 2189 * @dma_conf: structure to take the dma data 2190 * Description: according to which descriptor can be used (extend or basic) 2191 * this function allocates the resources for TX and RX paths. In case of 2192 * reception, for example, it pre-allocated the RX socket buffer in order to 2193 * allow zero-copy mechanism. 2194 */ 2195 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2196 struct stmmac_dma_conf *dma_conf) 2197 { 2198 /* RX Allocation */ 2199 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2200 2201 if (ret) 2202 return ret; 2203 2204 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2205 2206 return ret; 2207 } 2208 2209 /** 2210 * free_dma_desc_resources - free dma desc resources 2211 * @priv: private structure 2212 * @dma_conf: structure to take the dma data 2213 */ 2214 static void free_dma_desc_resources(struct stmmac_priv *priv, 2215 struct stmmac_dma_conf *dma_conf) 2216 { 2217 /* Release the DMA TX socket buffers */ 2218 free_dma_tx_desc_resources(priv, dma_conf); 2219 2220 /* Release the DMA RX socket buffers later 2221 * to ensure all pending XDP_TX buffers are returned. 2222 */ 2223 free_dma_rx_desc_resources(priv, dma_conf); 2224 } 2225 2226 /** 2227 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2228 * @priv: driver private structure 2229 * Description: It is used for enabling the rx queues in the MAC 2230 */ 2231 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2232 { 2233 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2234 int queue; 2235 u8 mode; 2236 2237 for (queue = 0; queue < rx_queues_count; queue++) { 2238 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2239 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2240 } 2241 } 2242 2243 /** 2244 * stmmac_start_rx_dma - start RX DMA channel 2245 * @priv: driver private structure 2246 * @chan: RX channel index 2247 * Description: 2248 * This starts a RX DMA channel 2249 */ 2250 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2251 { 2252 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2253 stmmac_start_rx(priv, priv->ioaddr, chan); 2254 } 2255 2256 /** 2257 * stmmac_start_tx_dma - start TX DMA channel 2258 * @priv: driver private structure 2259 * @chan: TX channel index 2260 * Description: 2261 * This starts a TX DMA channel 2262 */ 2263 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2264 { 2265 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2266 stmmac_start_tx(priv, priv->ioaddr, chan); 2267 } 2268 2269 /** 2270 * stmmac_stop_rx_dma - stop RX DMA channel 2271 * @priv: driver private structure 2272 * @chan: RX channel index 2273 * Description: 2274 * This stops a RX DMA channel 2275 */ 2276 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2277 { 2278 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2279 stmmac_stop_rx(priv, priv->ioaddr, chan); 2280 } 2281 2282 /** 2283 * stmmac_stop_tx_dma - stop TX DMA channel 2284 * @priv: driver private structure 2285 * @chan: TX channel index 2286 * Description: 2287 * This stops a TX DMA channel 2288 */ 2289 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2290 { 2291 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2292 stmmac_stop_tx(priv, priv->ioaddr, chan); 2293 } 2294 2295 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2296 { 2297 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2298 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2299 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2300 u32 chan; 2301 2302 for (chan = 0; chan < dma_csr_ch; chan++) { 2303 struct stmmac_channel *ch = &priv->channel[chan]; 2304 unsigned long flags; 2305 2306 spin_lock_irqsave(&ch->lock, flags); 2307 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2308 spin_unlock_irqrestore(&ch->lock, flags); 2309 } 2310 } 2311 2312 /** 2313 * stmmac_start_all_dma - start all RX and TX DMA channels 2314 * @priv: driver private structure 2315 * Description: 2316 * This starts all the RX and TX DMA channels 2317 */ 2318 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2319 { 2320 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2321 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2322 u32 chan = 0; 2323 2324 for (chan = 0; chan < rx_channels_count; chan++) 2325 stmmac_start_rx_dma(priv, chan); 2326 2327 for (chan = 0; chan < tx_channels_count; chan++) 2328 stmmac_start_tx_dma(priv, chan); 2329 } 2330 2331 /** 2332 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2333 * @priv: driver private structure 2334 * Description: 2335 * This stops the RX and TX DMA channels 2336 */ 2337 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2338 { 2339 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2340 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2341 u32 chan = 0; 2342 2343 for (chan = 0; chan < rx_channels_count; chan++) 2344 stmmac_stop_rx_dma(priv, chan); 2345 2346 for (chan = 0; chan < tx_channels_count; chan++) 2347 stmmac_stop_tx_dma(priv, chan); 2348 } 2349 2350 /** 2351 * stmmac_dma_operation_mode - HW DMA operation mode 2352 * @priv: driver private structure 2353 * Description: it is used for configuring the DMA operation mode register in 2354 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2355 */ 2356 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2357 { 2358 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2359 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2360 int rxfifosz = priv->plat->rx_fifo_size; 2361 int txfifosz = priv->plat->tx_fifo_size; 2362 u32 txmode = 0; 2363 u32 rxmode = 0; 2364 u32 chan = 0; 2365 u8 qmode = 0; 2366 2367 if (rxfifosz == 0) 2368 rxfifosz = priv->dma_cap.rx_fifo_size; 2369 if (txfifosz == 0) 2370 txfifosz = priv->dma_cap.tx_fifo_size; 2371 2372 /* Adjust for real per queue fifo size */ 2373 rxfifosz /= rx_channels_count; 2374 txfifosz /= tx_channels_count; 2375 2376 if (priv->plat->force_thresh_dma_mode) { 2377 txmode = tc; 2378 rxmode = tc; 2379 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2380 /* 2381 * In case of GMAC, SF mode can be enabled 2382 * to perform the TX COE in HW. This depends on: 2383 * 1) TX COE if actually supported 2384 * 2) There is no bugged Jumbo frame support 2385 * that needs to not insert csum in the TDES. 2386 */ 2387 txmode = SF_DMA_MODE; 2388 rxmode = SF_DMA_MODE; 2389 priv->xstats.threshold = SF_DMA_MODE; 2390 } else { 2391 txmode = tc; 2392 rxmode = SF_DMA_MODE; 2393 } 2394 2395 /* configure all channels */ 2396 for (chan = 0; chan < rx_channels_count; chan++) { 2397 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2398 u32 buf_size; 2399 2400 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2401 2402 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2403 rxfifosz, qmode); 2404 2405 if (rx_q->xsk_pool) { 2406 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2407 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2408 buf_size, 2409 chan); 2410 } else { 2411 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2412 priv->dma_conf.dma_buf_sz, 2413 chan); 2414 } 2415 } 2416 2417 for (chan = 0; chan < tx_channels_count; chan++) { 2418 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2419 2420 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2421 txfifosz, qmode); 2422 } 2423 } 2424 2425 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2426 { 2427 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2428 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2429 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2430 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2431 unsigned int entry = tx_q->cur_tx; 2432 struct dma_desc *tx_desc = NULL; 2433 struct xdp_desc xdp_desc; 2434 bool work_done = true; 2435 u32 tx_set_ic_bit = 0; 2436 unsigned long flags; 2437 2438 /* Avoids TX time-out as we are sharing with slow path */ 2439 txq_trans_cond_update(nq); 2440 2441 budget = min(budget, stmmac_tx_avail(priv, queue)); 2442 2443 while (budget-- > 0) { 2444 dma_addr_t dma_addr; 2445 bool set_ic; 2446 2447 /* We are sharing with slow path and stop XSK TX desc submission when 2448 * available TX ring is less than threshold. 2449 */ 2450 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2451 !netif_carrier_ok(priv->dev)) { 2452 work_done = false; 2453 break; 2454 } 2455 2456 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2457 break; 2458 2459 if (likely(priv->extend_desc)) 2460 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2461 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2462 tx_desc = &tx_q->dma_entx[entry].basic; 2463 else 2464 tx_desc = tx_q->dma_tx + entry; 2465 2466 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2467 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2468 2469 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2470 2471 /* To return XDP buffer to XSK pool, we simple call 2472 * xsk_tx_completed(), so we don't need to fill up 2473 * 'buf' and 'xdpf'. 2474 */ 2475 tx_q->tx_skbuff_dma[entry].buf = 0; 2476 tx_q->xdpf[entry] = NULL; 2477 2478 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2479 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2480 tx_q->tx_skbuff_dma[entry].last_segment = true; 2481 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2482 2483 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2484 2485 tx_q->tx_count_frames++; 2486 2487 if (!priv->tx_coal_frames[queue]) 2488 set_ic = false; 2489 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2490 set_ic = true; 2491 else 2492 set_ic = false; 2493 2494 if (set_ic) { 2495 tx_q->tx_count_frames = 0; 2496 stmmac_set_tx_ic(priv, tx_desc); 2497 tx_set_ic_bit++; 2498 } 2499 2500 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2501 true, priv->mode, true, true, 2502 xdp_desc.len); 2503 2504 stmmac_enable_dma_transmission(priv, priv->ioaddr); 2505 2506 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2507 entry = tx_q->cur_tx; 2508 } 2509 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 2510 txq_stats->tx_set_ic_bit += tx_set_ic_bit; 2511 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 2512 2513 if (tx_desc) { 2514 stmmac_flush_tx_descriptors(priv, queue); 2515 xsk_tx_release(pool); 2516 } 2517 2518 /* Return true if all of the 3 conditions are met 2519 * a) TX Budget is still available 2520 * b) work_done = true when XSK TX desc peek is empty (no more 2521 * pending XSK TX for transmission) 2522 */ 2523 return !!budget && work_done; 2524 } 2525 2526 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2527 { 2528 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2529 tc += 64; 2530 2531 if (priv->plat->force_thresh_dma_mode) 2532 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2533 else 2534 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2535 chan); 2536 2537 priv->xstats.threshold = tc; 2538 } 2539 } 2540 2541 /** 2542 * stmmac_tx_clean - to manage the transmission completion 2543 * @priv: driver private structure 2544 * @budget: napi budget limiting this functions packet handling 2545 * @queue: TX queue index 2546 * Description: it reclaims the transmit resources after transmission completes. 2547 */ 2548 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) 2549 { 2550 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2551 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2552 unsigned int bytes_compl = 0, pkts_compl = 0; 2553 unsigned int entry, xmits = 0, count = 0; 2554 u32 tx_packets = 0, tx_errors = 0; 2555 unsigned long flags; 2556 2557 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2558 2559 tx_q->xsk_frames_done = 0; 2560 2561 entry = tx_q->dirty_tx; 2562 2563 /* Try to clean all TX complete frame in 1 shot */ 2564 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2565 struct xdp_frame *xdpf; 2566 struct sk_buff *skb; 2567 struct dma_desc *p; 2568 int status; 2569 2570 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2571 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2572 xdpf = tx_q->xdpf[entry]; 2573 skb = NULL; 2574 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2575 xdpf = NULL; 2576 skb = tx_q->tx_skbuff[entry]; 2577 } else { 2578 xdpf = NULL; 2579 skb = NULL; 2580 } 2581 2582 if (priv->extend_desc) 2583 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2584 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2585 p = &tx_q->dma_entx[entry].basic; 2586 else 2587 p = tx_q->dma_tx + entry; 2588 2589 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2590 /* Check if the descriptor is owned by the DMA */ 2591 if (unlikely(status & tx_dma_own)) 2592 break; 2593 2594 count++; 2595 2596 /* Make sure descriptor fields are read after reading 2597 * the own bit. 2598 */ 2599 dma_rmb(); 2600 2601 /* Just consider the last segment and ...*/ 2602 if (likely(!(status & tx_not_ls))) { 2603 /* ... verify the status error condition */ 2604 if (unlikely(status & tx_err)) { 2605 tx_errors++; 2606 if (unlikely(status & tx_err_bump_tc)) 2607 stmmac_bump_dma_threshold(priv, queue); 2608 } else { 2609 tx_packets++; 2610 } 2611 if (skb) 2612 stmmac_get_tx_hwtstamp(priv, p, skb); 2613 } 2614 2615 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2616 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2617 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2618 dma_unmap_page(priv->device, 2619 tx_q->tx_skbuff_dma[entry].buf, 2620 tx_q->tx_skbuff_dma[entry].len, 2621 DMA_TO_DEVICE); 2622 else 2623 dma_unmap_single(priv->device, 2624 tx_q->tx_skbuff_dma[entry].buf, 2625 tx_q->tx_skbuff_dma[entry].len, 2626 DMA_TO_DEVICE); 2627 tx_q->tx_skbuff_dma[entry].buf = 0; 2628 tx_q->tx_skbuff_dma[entry].len = 0; 2629 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2630 } 2631 2632 stmmac_clean_desc3(priv, tx_q, p); 2633 2634 tx_q->tx_skbuff_dma[entry].last_segment = false; 2635 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2636 2637 if (xdpf && 2638 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2639 xdp_return_frame_rx_napi(xdpf); 2640 tx_q->xdpf[entry] = NULL; 2641 } 2642 2643 if (xdpf && 2644 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2645 xdp_return_frame(xdpf); 2646 tx_q->xdpf[entry] = NULL; 2647 } 2648 2649 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2650 tx_q->xsk_frames_done++; 2651 2652 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2653 if (likely(skb)) { 2654 pkts_compl++; 2655 bytes_compl += skb->len; 2656 dev_consume_skb_any(skb); 2657 tx_q->tx_skbuff[entry] = NULL; 2658 } 2659 } 2660 2661 stmmac_release_tx_desc(priv, p, priv->mode); 2662 2663 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 2664 } 2665 tx_q->dirty_tx = entry; 2666 2667 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2668 pkts_compl, bytes_compl); 2669 2670 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2671 queue))) && 2672 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2673 2674 netif_dbg(priv, tx_done, priv->dev, 2675 "%s: restart transmit\n", __func__); 2676 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2677 } 2678 2679 if (tx_q->xsk_pool) { 2680 bool work_done; 2681 2682 if (tx_q->xsk_frames_done) 2683 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2684 2685 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2686 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2687 2688 /* For XSK TX, we try to send as many as possible. 2689 * If XSK work done (XSK TX desc empty and budget still 2690 * available), return "budget - 1" to reenable TX IRQ. 2691 * Else, return "budget" to make NAPI continue polling. 2692 */ 2693 work_done = stmmac_xdp_xmit_zc(priv, queue, 2694 STMMAC_XSK_TX_BUDGET_MAX); 2695 if (work_done) 2696 xmits = budget - 1; 2697 else 2698 xmits = budget; 2699 } 2700 2701 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && 2702 priv->eee_sw_timer_en) { 2703 if (stmmac_enable_eee_mode(priv)) 2704 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 2705 } 2706 2707 /* We still have pending packets, let's call for a new scheduling */ 2708 if (tx_q->dirty_tx != tx_q->cur_tx) 2709 stmmac_tx_timer_arm(priv, queue); 2710 2711 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 2712 txq_stats->tx_packets += tx_packets; 2713 txq_stats->tx_pkt_n += tx_packets; 2714 txq_stats->tx_clean++; 2715 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 2716 2717 priv->xstats.tx_errors += tx_errors; 2718 2719 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2720 2721 /* Combine decisions from TX clean and XSK TX */ 2722 return max(count, xmits); 2723 } 2724 2725 /** 2726 * stmmac_tx_err - to manage the tx error 2727 * @priv: driver private structure 2728 * @chan: channel index 2729 * Description: it cleans the descriptors and restarts the transmission 2730 * in case of transmission errors. 2731 */ 2732 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2733 { 2734 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2735 2736 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2737 2738 stmmac_stop_tx_dma(priv, chan); 2739 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 2740 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 2741 stmmac_reset_tx_queue(priv, chan); 2742 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2743 tx_q->dma_tx_phy, chan); 2744 stmmac_start_tx_dma(priv, chan); 2745 2746 priv->xstats.tx_errors++; 2747 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2748 } 2749 2750 /** 2751 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2752 * @priv: driver private structure 2753 * @txmode: TX operating mode 2754 * @rxmode: RX operating mode 2755 * @chan: channel index 2756 * Description: it is used for configuring of the DMA operation mode in 2757 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2758 * mode. 2759 */ 2760 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2761 u32 rxmode, u32 chan) 2762 { 2763 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2764 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2765 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2766 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2767 int rxfifosz = priv->plat->rx_fifo_size; 2768 int txfifosz = priv->plat->tx_fifo_size; 2769 2770 if (rxfifosz == 0) 2771 rxfifosz = priv->dma_cap.rx_fifo_size; 2772 if (txfifosz == 0) 2773 txfifosz = priv->dma_cap.tx_fifo_size; 2774 2775 /* Adjust for real per queue fifo size */ 2776 rxfifosz /= rx_channels_count; 2777 txfifosz /= tx_channels_count; 2778 2779 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2780 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2781 } 2782 2783 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2784 { 2785 int ret; 2786 2787 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2788 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2789 if (ret && (ret != -EINVAL)) { 2790 stmmac_global_err(priv); 2791 return true; 2792 } 2793 2794 return false; 2795 } 2796 2797 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 2798 { 2799 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2800 &priv->xstats, chan, dir); 2801 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2802 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2803 struct stmmac_channel *ch = &priv->channel[chan]; 2804 struct napi_struct *rx_napi; 2805 struct napi_struct *tx_napi; 2806 unsigned long flags; 2807 2808 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 2809 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2810 2811 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 2812 if (napi_schedule_prep(rx_napi)) { 2813 spin_lock_irqsave(&ch->lock, flags); 2814 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 2815 spin_unlock_irqrestore(&ch->lock, flags); 2816 __napi_schedule(rx_napi); 2817 } 2818 } 2819 2820 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 2821 if (napi_schedule_prep(tx_napi)) { 2822 spin_lock_irqsave(&ch->lock, flags); 2823 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 2824 spin_unlock_irqrestore(&ch->lock, flags); 2825 __napi_schedule(tx_napi); 2826 } 2827 } 2828 2829 return status; 2830 } 2831 2832 /** 2833 * stmmac_dma_interrupt - DMA ISR 2834 * @priv: driver private structure 2835 * Description: this is the DMA ISR. It is called by the main ISR. 2836 * It calls the dwmac dma routine and schedule poll method in case of some 2837 * work can be done. 2838 */ 2839 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2840 { 2841 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2842 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2843 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2844 tx_channel_count : rx_channel_count; 2845 u32 chan; 2846 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2847 2848 /* Make sure we never check beyond our status buffer. */ 2849 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2850 channels_to_check = ARRAY_SIZE(status); 2851 2852 for (chan = 0; chan < channels_to_check; chan++) 2853 status[chan] = stmmac_napi_check(priv, chan, 2854 DMA_DIR_RXTX); 2855 2856 for (chan = 0; chan < tx_channel_count; chan++) { 2857 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2858 /* Try to bump up the dma threshold on this failure */ 2859 stmmac_bump_dma_threshold(priv, chan); 2860 } else if (unlikely(status[chan] == tx_hard_error)) { 2861 stmmac_tx_err(priv, chan); 2862 } 2863 } 2864 } 2865 2866 /** 2867 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2868 * @priv: driver private structure 2869 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2870 */ 2871 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2872 { 2873 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2874 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2875 2876 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 2877 2878 if (priv->dma_cap.rmon) { 2879 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 2880 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2881 } else 2882 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2883 } 2884 2885 /** 2886 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2887 * @priv: driver private structure 2888 * Description: 2889 * new GMAC chip generations have a new register to indicate the 2890 * presence of the optional feature/functions. 2891 * This can be also used to override the value passed through the 2892 * platform and necessary for old MAC10/100 and GMAC chips. 2893 */ 2894 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2895 { 2896 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2897 } 2898 2899 /** 2900 * stmmac_check_ether_addr - check if the MAC addr is valid 2901 * @priv: driver private structure 2902 * Description: 2903 * it is to verify if the MAC address is valid, in case of failures it 2904 * generates a random MAC address 2905 */ 2906 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2907 { 2908 u8 addr[ETH_ALEN]; 2909 2910 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2911 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 2912 if (is_valid_ether_addr(addr)) 2913 eth_hw_addr_set(priv->dev, addr); 2914 else 2915 eth_hw_addr_random(priv->dev); 2916 dev_info(priv->device, "device MAC address %pM\n", 2917 priv->dev->dev_addr); 2918 } 2919 } 2920 2921 /** 2922 * stmmac_init_dma_engine - DMA init. 2923 * @priv: driver private structure 2924 * Description: 2925 * It inits the DMA invoking the specific MAC/GMAC callback. 2926 * Some DMA parameters can be passed from the platform; 2927 * in case of these are not passed a default is kept for the MAC or GMAC. 2928 */ 2929 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2930 { 2931 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2932 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2933 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2934 struct stmmac_rx_queue *rx_q; 2935 struct stmmac_tx_queue *tx_q; 2936 u32 chan = 0; 2937 int atds = 0; 2938 int ret = 0; 2939 2940 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2941 dev_err(priv->device, "Invalid DMA configuration\n"); 2942 return -EINVAL; 2943 } 2944 2945 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2946 atds = 1; 2947 2948 ret = stmmac_reset(priv, priv->ioaddr); 2949 if (ret) { 2950 dev_err(priv->device, "Failed to reset the dma\n"); 2951 return ret; 2952 } 2953 2954 /* DMA Configuration */ 2955 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); 2956 2957 if (priv->plat->axi) 2958 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 2959 2960 /* DMA CSR Channel configuration */ 2961 for (chan = 0; chan < dma_csr_ch; chan++) { 2962 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 2963 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2964 } 2965 2966 /* DMA RX Channel Configuration */ 2967 for (chan = 0; chan < rx_channels_count; chan++) { 2968 rx_q = &priv->dma_conf.rx_queue[chan]; 2969 2970 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2971 rx_q->dma_rx_phy, chan); 2972 2973 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2974 (rx_q->buf_alloc_num * 2975 sizeof(struct dma_desc)); 2976 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 2977 rx_q->rx_tail_addr, chan); 2978 } 2979 2980 /* DMA TX Channel Configuration */ 2981 for (chan = 0; chan < tx_channels_count; chan++) { 2982 tx_q = &priv->dma_conf.tx_queue[chan]; 2983 2984 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2985 tx_q->dma_tx_phy, chan); 2986 2987 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 2988 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 2989 tx_q->tx_tail_addr, chan); 2990 } 2991 2992 return ret; 2993 } 2994 2995 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 2996 { 2997 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2998 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 2999 3000 if (!tx_coal_timer) 3001 return; 3002 3003 hrtimer_start(&tx_q->txtimer, 3004 STMMAC_COAL_TIMER(tx_coal_timer), 3005 HRTIMER_MODE_REL); 3006 } 3007 3008 /** 3009 * stmmac_tx_timer - mitigation sw timer for tx. 3010 * @t: data pointer 3011 * Description: 3012 * This is the timer handler to directly invoke the stmmac_tx_clean. 3013 */ 3014 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3015 { 3016 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3017 struct stmmac_priv *priv = tx_q->priv_data; 3018 struct stmmac_channel *ch; 3019 struct napi_struct *napi; 3020 3021 ch = &priv->channel[tx_q->queue_index]; 3022 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3023 3024 if (likely(napi_schedule_prep(napi))) { 3025 unsigned long flags; 3026 3027 spin_lock_irqsave(&ch->lock, flags); 3028 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3029 spin_unlock_irqrestore(&ch->lock, flags); 3030 __napi_schedule(napi); 3031 } 3032 3033 return HRTIMER_NORESTART; 3034 } 3035 3036 /** 3037 * stmmac_init_coalesce - init mitigation options. 3038 * @priv: driver private structure 3039 * Description: 3040 * This inits the coalesce parameters: i.e. timer rate, 3041 * timer handler and default threshold used for enabling the 3042 * interrupt on completion bit. 3043 */ 3044 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3045 { 3046 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3047 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3048 u32 chan; 3049 3050 for (chan = 0; chan < tx_channel_count; chan++) { 3051 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3052 3053 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3054 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3055 3056 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3057 tx_q->txtimer.function = stmmac_tx_timer; 3058 } 3059 3060 for (chan = 0; chan < rx_channel_count; chan++) 3061 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3062 } 3063 3064 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3065 { 3066 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3067 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3068 u32 chan; 3069 3070 /* set TX ring length */ 3071 for (chan = 0; chan < tx_channels_count; chan++) 3072 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3073 (priv->dma_conf.dma_tx_size - 1), chan); 3074 3075 /* set RX ring length */ 3076 for (chan = 0; chan < rx_channels_count; chan++) 3077 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3078 (priv->dma_conf.dma_rx_size - 1), chan); 3079 } 3080 3081 /** 3082 * stmmac_set_tx_queue_weight - Set TX queue weight 3083 * @priv: driver private structure 3084 * Description: It is used for setting TX queues weight 3085 */ 3086 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3087 { 3088 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3089 u32 weight; 3090 u32 queue; 3091 3092 for (queue = 0; queue < tx_queues_count; queue++) { 3093 weight = priv->plat->tx_queues_cfg[queue].weight; 3094 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3095 } 3096 } 3097 3098 /** 3099 * stmmac_configure_cbs - Configure CBS in TX queue 3100 * @priv: driver private structure 3101 * Description: It is used for configuring CBS in AVB TX queues 3102 */ 3103 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3104 { 3105 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3106 u32 mode_to_use; 3107 u32 queue; 3108 3109 /* queue 0 is reserved for legacy traffic */ 3110 for (queue = 1; queue < tx_queues_count; queue++) { 3111 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3112 if (mode_to_use == MTL_QUEUE_DCB) 3113 continue; 3114 3115 stmmac_config_cbs(priv, priv->hw, 3116 priv->plat->tx_queues_cfg[queue].send_slope, 3117 priv->plat->tx_queues_cfg[queue].idle_slope, 3118 priv->plat->tx_queues_cfg[queue].high_credit, 3119 priv->plat->tx_queues_cfg[queue].low_credit, 3120 queue); 3121 } 3122 } 3123 3124 /** 3125 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3126 * @priv: driver private structure 3127 * Description: It is used for mapping RX queues to RX dma channels 3128 */ 3129 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3130 { 3131 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3132 u32 queue; 3133 u32 chan; 3134 3135 for (queue = 0; queue < rx_queues_count; queue++) { 3136 chan = priv->plat->rx_queues_cfg[queue].chan; 3137 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3138 } 3139 } 3140 3141 /** 3142 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3143 * @priv: driver private structure 3144 * Description: It is used for configuring the RX Queue Priority 3145 */ 3146 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3147 { 3148 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3149 u32 queue; 3150 u32 prio; 3151 3152 for (queue = 0; queue < rx_queues_count; queue++) { 3153 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3154 continue; 3155 3156 prio = priv->plat->rx_queues_cfg[queue].prio; 3157 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3158 } 3159 } 3160 3161 /** 3162 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3163 * @priv: driver private structure 3164 * Description: It is used for configuring the TX Queue Priority 3165 */ 3166 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3167 { 3168 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3169 u32 queue; 3170 u32 prio; 3171 3172 for (queue = 0; queue < tx_queues_count; queue++) { 3173 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3174 continue; 3175 3176 prio = priv->plat->tx_queues_cfg[queue].prio; 3177 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3178 } 3179 } 3180 3181 /** 3182 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3183 * @priv: driver private structure 3184 * Description: It is used for configuring the RX queue routing 3185 */ 3186 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3187 { 3188 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3189 u32 queue; 3190 u8 packet; 3191 3192 for (queue = 0; queue < rx_queues_count; queue++) { 3193 /* no specific packet type routing specified for the queue */ 3194 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3195 continue; 3196 3197 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3198 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3199 } 3200 } 3201 3202 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3203 { 3204 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3205 priv->rss.enable = false; 3206 return; 3207 } 3208 3209 if (priv->dev->features & NETIF_F_RXHASH) 3210 priv->rss.enable = true; 3211 else 3212 priv->rss.enable = false; 3213 3214 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3215 priv->plat->rx_queues_to_use); 3216 } 3217 3218 /** 3219 * stmmac_mtl_configuration - Configure MTL 3220 * @priv: driver private structure 3221 * Description: It is used for configurring MTL 3222 */ 3223 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3224 { 3225 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3226 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3227 3228 if (tx_queues_count > 1) 3229 stmmac_set_tx_queue_weight(priv); 3230 3231 /* Configure MTL RX algorithms */ 3232 if (rx_queues_count > 1) 3233 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3234 priv->plat->rx_sched_algorithm); 3235 3236 /* Configure MTL TX algorithms */ 3237 if (tx_queues_count > 1) 3238 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3239 priv->plat->tx_sched_algorithm); 3240 3241 /* Configure CBS in AVB TX queues */ 3242 if (tx_queues_count > 1) 3243 stmmac_configure_cbs(priv); 3244 3245 /* Map RX MTL to DMA channels */ 3246 stmmac_rx_queue_dma_chan_map(priv); 3247 3248 /* Enable MAC RX Queues */ 3249 stmmac_mac_enable_rx_queues(priv); 3250 3251 /* Set RX priorities */ 3252 if (rx_queues_count > 1) 3253 stmmac_mac_config_rx_queues_prio(priv); 3254 3255 /* Set TX priorities */ 3256 if (tx_queues_count > 1) 3257 stmmac_mac_config_tx_queues_prio(priv); 3258 3259 /* Set RX routing */ 3260 if (rx_queues_count > 1) 3261 stmmac_mac_config_rx_queues_routing(priv); 3262 3263 /* Receive Side Scaling */ 3264 if (rx_queues_count > 1) 3265 stmmac_mac_config_rss(priv); 3266 } 3267 3268 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3269 { 3270 if (priv->dma_cap.asp) { 3271 netdev_info(priv->dev, "Enabling Safety Features\n"); 3272 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3273 priv->plat->safety_feat_cfg); 3274 } else { 3275 netdev_info(priv->dev, "No Safety Features support found\n"); 3276 } 3277 } 3278 3279 static int stmmac_fpe_start_wq(struct stmmac_priv *priv) 3280 { 3281 char *name; 3282 3283 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 3284 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); 3285 3286 name = priv->wq_name; 3287 sprintf(name, "%s-fpe", priv->dev->name); 3288 3289 priv->fpe_wq = create_singlethread_workqueue(name); 3290 if (!priv->fpe_wq) { 3291 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); 3292 3293 return -ENOMEM; 3294 } 3295 netdev_info(priv->dev, "FPE workqueue start"); 3296 3297 return 0; 3298 } 3299 3300 /** 3301 * stmmac_hw_setup - setup mac in a usable state. 3302 * @dev : pointer to the device structure. 3303 * @ptp_register: register PTP if set 3304 * Description: 3305 * this is the main function to setup the HW in a usable state because the 3306 * dma engine is reset, the core registers are configured (e.g. AXI, 3307 * Checksum features, timers). The DMA is ready to start receiving and 3308 * transmitting. 3309 * Return value: 3310 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3311 * file on failure. 3312 */ 3313 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) 3314 { 3315 struct stmmac_priv *priv = netdev_priv(dev); 3316 u32 rx_cnt = priv->plat->rx_queues_to_use; 3317 u32 tx_cnt = priv->plat->tx_queues_to_use; 3318 bool sph_en; 3319 u32 chan; 3320 int ret; 3321 3322 /* DMA initialization and SW reset */ 3323 ret = stmmac_init_dma_engine(priv); 3324 if (ret < 0) { 3325 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3326 __func__); 3327 return ret; 3328 } 3329 3330 /* Copy the MAC addr into the HW */ 3331 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3332 3333 /* PS and related bits will be programmed according to the speed */ 3334 if (priv->hw->pcs) { 3335 int speed = priv->plat->mac_port_sel_speed; 3336 3337 if ((speed == SPEED_10) || (speed == SPEED_100) || 3338 (speed == SPEED_1000)) { 3339 priv->hw->ps = speed; 3340 } else { 3341 dev_warn(priv->device, "invalid port speed\n"); 3342 priv->hw->ps = 0; 3343 } 3344 } 3345 3346 /* Initialize the MAC Core */ 3347 stmmac_core_init(priv, priv->hw, dev); 3348 3349 /* Initialize MTL*/ 3350 stmmac_mtl_configuration(priv); 3351 3352 /* Initialize Safety Features */ 3353 stmmac_safety_feat_configuration(priv); 3354 3355 ret = stmmac_rx_ipc(priv, priv->hw); 3356 if (!ret) { 3357 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3358 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3359 priv->hw->rx_csum = 0; 3360 } 3361 3362 /* Enable the MAC Rx/Tx */ 3363 stmmac_mac_set(priv, priv->ioaddr, true); 3364 3365 /* Set the HW DMA mode and the COE */ 3366 stmmac_dma_operation_mode(priv); 3367 3368 stmmac_mmc_setup(priv); 3369 3370 if (ptp_register) { 3371 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 3372 if (ret < 0) 3373 netdev_warn(priv->dev, 3374 "failed to enable PTP reference clock: %pe\n", 3375 ERR_PTR(ret)); 3376 } 3377 3378 ret = stmmac_init_ptp(priv); 3379 if (ret == -EOPNOTSUPP) 3380 netdev_info(priv->dev, "PTP not supported by HW\n"); 3381 else if (ret) 3382 netdev_warn(priv->dev, "PTP init failed\n"); 3383 else if (ptp_register) 3384 stmmac_ptp_register(priv); 3385 3386 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; 3387 3388 /* Convert the timer from msec to usec */ 3389 if (!priv->tx_lpi_timer) 3390 priv->tx_lpi_timer = eee_timer * 1000; 3391 3392 if (priv->use_riwt) { 3393 u32 queue; 3394 3395 for (queue = 0; queue < rx_cnt; queue++) { 3396 if (!priv->rx_riwt[queue]) 3397 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3398 3399 stmmac_rx_watchdog(priv, priv->ioaddr, 3400 priv->rx_riwt[queue], queue); 3401 } 3402 } 3403 3404 if (priv->hw->pcs) 3405 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); 3406 3407 /* set TX and RX rings length */ 3408 stmmac_set_rings_length(priv); 3409 3410 /* Enable TSO */ 3411 if (priv->tso) { 3412 for (chan = 0; chan < tx_cnt; chan++) { 3413 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3414 3415 /* TSO and TBS cannot co-exist */ 3416 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3417 continue; 3418 3419 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3420 } 3421 } 3422 3423 /* Enable Split Header */ 3424 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 3425 for (chan = 0; chan < rx_cnt; chan++) 3426 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3427 3428 3429 /* VLAN Tag Insertion */ 3430 if (priv->dma_cap.vlins) 3431 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3432 3433 /* TBS */ 3434 for (chan = 0; chan < tx_cnt; chan++) { 3435 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3436 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3437 3438 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3439 } 3440 3441 /* Configure real RX and TX queues */ 3442 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3443 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3444 3445 /* Start the ball rolling... */ 3446 stmmac_start_all_dma(priv); 3447 3448 if (priv->dma_cap.fpesel) { 3449 stmmac_fpe_start_wq(priv); 3450 3451 if (priv->plat->fpe_cfg->enable) 3452 stmmac_fpe_handshake(priv, true); 3453 } 3454 3455 return 0; 3456 } 3457 3458 static void stmmac_hw_teardown(struct net_device *dev) 3459 { 3460 struct stmmac_priv *priv = netdev_priv(dev); 3461 3462 clk_disable_unprepare(priv->plat->clk_ptp_ref); 3463 } 3464 3465 static void stmmac_free_irq(struct net_device *dev, 3466 enum request_irq_err irq_err, int irq_idx) 3467 { 3468 struct stmmac_priv *priv = netdev_priv(dev); 3469 int j; 3470 3471 switch (irq_err) { 3472 case REQ_IRQ_ERR_ALL: 3473 irq_idx = priv->plat->tx_queues_to_use; 3474 fallthrough; 3475 case REQ_IRQ_ERR_TX: 3476 for (j = irq_idx - 1; j >= 0; j--) { 3477 if (priv->tx_irq[j] > 0) { 3478 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3479 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); 3480 } 3481 } 3482 irq_idx = priv->plat->rx_queues_to_use; 3483 fallthrough; 3484 case REQ_IRQ_ERR_RX: 3485 for (j = irq_idx - 1; j >= 0; j--) { 3486 if (priv->rx_irq[j] > 0) { 3487 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3488 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); 3489 } 3490 } 3491 3492 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3493 free_irq(priv->sfty_ue_irq, dev); 3494 fallthrough; 3495 case REQ_IRQ_ERR_SFTY_UE: 3496 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3497 free_irq(priv->sfty_ce_irq, dev); 3498 fallthrough; 3499 case REQ_IRQ_ERR_SFTY_CE: 3500 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) 3501 free_irq(priv->lpi_irq, dev); 3502 fallthrough; 3503 case REQ_IRQ_ERR_LPI: 3504 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3505 free_irq(priv->wol_irq, dev); 3506 fallthrough; 3507 case REQ_IRQ_ERR_WOL: 3508 free_irq(dev->irq, dev); 3509 fallthrough; 3510 case REQ_IRQ_ERR_MAC: 3511 case REQ_IRQ_ERR_NO: 3512 /* If MAC IRQ request error, no more IRQ to free */ 3513 break; 3514 } 3515 } 3516 3517 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3518 { 3519 struct stmmac_priv *priv = netdev_priv(dev); 3520 enum request_irq_err irq_err; 3521 cpumask_t cpu_mask; 3522 int irq_idx = 0; 3523 char *int_name; 3524 int ret; 3525 int i; 3526 3527 /* For common interrupt */ 3528 int_name = priv->int_name_mac; 3529 sprintf(int_name, "%s:%s", dev->name, "mac"); 3530 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3531 0, int_name, dev); 3532 if (unlikely(ret < 0)) { 3533 netdev_err(priv->dev, 3534 "%s: alloc mac MSI %d (error: %d)\n", 3535 __func__, dev->irq, ret); 3536 irq_err = REQ_IRQ_ERR_MAC; 3537 goto irq_error; 3538 } 3539 3540 /* Request the Wake IRQ in case of another line 3541 * is used for WoL 3542 */ 3543 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3544 int_name = priv->int_name_wol; 3545 sprintf(int_name, "%s:%s", dev->name, "wol"); 3546 ret = request_irq(priv->wol_irq, 3547 stmmac_mac_interrupt, 3548 0, int_name, dev); 3549 if (unlikely(ret < 0)) { 3550 netdev_err(priv->dev, 3551 "%s: alloc wol MSI %d (error: %d)\n", 3552 __func__, priv->wol_irq, ret); 3553 irq_err = REQ_IRQ_ERR_WOL; 3554 goto irq_error; 3555 } 3556 } 3557 3558 /* Request the LPI IRQ in case of another line 3559 * is used for LPI 3560 */ 3561 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3562 int_name = priv->int_name_lpi; 3563 sprintf(int_name, "%s:%s", dev->name, "lpi"); 3564 ret = request_irq(priv->lpi_irq, 3565 stmmac_mac_interrupt, 3566 0, int_name, dev); 3567 if (unlikely(ret < 0)) { 3568 netdev_err(priv->dev, 3569 "%s: alloc lpi MSI %d (error: %d)\n", 3570 __func__, priv->lpi_irq, ret); 3571 irq_err = REQ_IRQ_ERR_LPI; 3572 goto irq_error; 3573 } 3574 } 3575 3576 /* Request the Safety Feature Correctible Error line in 3577 * case of another line is used 3578 */ 3579 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3580 int_name = priv->int_name_sfty_ce; 3581 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3582 ret = request_irq(priv->sfty_ce_irq, 3583 stmmac_safety_interrupt, 3584 0, int_name, dev); 3585 if (unlikely(ret < 0)) { 3586 netdev_err(priv->dev, 3587 "%s: alloc sfty ce MSI %d (error: %d)\n", 3588 __func__, priv->sfty_ce_irq, ret); 3589 irq_err = REQ_IRQ_ERR_SFTY_CE; 3590 goto irq_error; 3591 } 3592 } 3593 3594 /* Request the Safety Feature Uncorrectible Error line in 3595 * case of another line is used 3596 */ 3597 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3598 int_name = priv->int_name_sfty_ue; 3599 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3600 ret = request_irq(priv->sfty_ue_irq, 3601 stmmac_safety_interrupt, 3602 0, int_name, dev); 3603 if (unlikely(ret < 0)) { 3604 netdev_err(priv->dev, 3605 "%s: alloc sfty ue MSI %d (error: %d)\n", 3606 __func__, priv->sfty_ue_irq, ret); 3607 irq_err = REQ_IRQ_ERR_SFTY_UE; 3608 goto irq_error; 3609 } 3610 } 3611 3612 /* Request Rx MSI irq */ 3613 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3614 if (i >= MTL_MAX_RX_QUEUES) 3615 break; 3616 if (priv->rx_irq[i] == 0) 3617 continue; 3618 3619 int_name = priv->int_name_rx_irq[i]; 3620 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3621 ret = request_irq(priv->rx_irq[i], 3622 stmmac_msi_intr_rx, 3623 0, int_name, &priv->dma_conf.rx_queue[i]); 3624 if (unlikely(ret < 0)) { 3625 netdev_err(priv->dev, 3626 "%s: alloc rx-%d MSI %d (error: %d)\n", 3627 __func__, i, priv->rx_irq[i], ret); 3628 irq_err = REQ_IRQ_ERR_RX; 3629 irq_idx = i; 3630 goto irq_error; 3631 } 3632 cpumask_clear(&cpu_mask); 3633 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3634 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); 3635 } 3636 3637 /* Request Tx MSI irq */ 3638 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3639 if (i >= MTL_MAX_TX_QUEUES) 3640 break; 3641 if (priv->tx_irq[i] == 0) 3642 continue; 3643 3644 int_name = priv->int_name_tx_irq[i]; 3645 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3646 ret = request_irq(priv->tx_irq[i], 3647 stmmac_msi_intr_tx, 3648 0, int_name, &priv->dma_conf.tx_queue[i]); 3649 if (unlikely(ret < 0)) { 3650 netdev_err(priv->dev, 3651 "%s: alloc tx-%d MSI %d (error: %d)\n", 3652 __func__, i, priv->tx_irq[i], ret); 3653 irq_err = REQ_IRQ_ERR_TX; 3654 irq_idx = i; 3655 goto irq_error; 3656 } 3657 cpumask_clear(&cpu_mask); 3658 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3659 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); 3660 } 3661 3662 return 0; 3663 3664 irq_error: 3665 stmmac_free_irq(dev, irq_err, irq_idx); 3666 return ret; 3667 } 3668 3669 static int stmmac_request_irq_single(struct net_device *dev) 3670 { 3671 struct stmmac_priv *priv = netdev_priv(dev); 3672 enum request_irq_err irq_err; 3673 int ret; 3674 3675 ret = request_irq(dev->irq, stmmac_interrupt, 3676 IRQF_SHARED, dev->name, dev); 3677 if (unlikely(ret < 0)) { 3678 netdev_err(priv->dev, 3679 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3680 __func__, dev->irq, ret); 3681 irq_err = REQ_IRQ_ERR_MAC; 3682 goto irq_error; 3683 } 3684 3685 /* Request the Wake IRQ in case of another line 3686 * is used for WoL 3687 */ 3688 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3689 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3690 IRQF_SHARED, dev->name, dev); 3691 if (unlikely(ret < 0)) { 3692 netdev_err(priv->dev, 3693 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3694 __func__, priv->wol_irq, ret); 3695 irq_err = REQ_IRQ_ERR_WOL; 3696 goto irq_error; 3697 } 3698 } 3699 3700 /* Request the IRQ lines */ 3701 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3702 ret = request_irq(priv->lpi_irq, stmmac_interrupt, 3703 IRQF_SHARED, dev->name, dev); 3704 if (unlikely(ret < 0)) { 3705 netdev_err(priv->dev, 3706 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 3707 __func__, priv->lpi_irq, ret); 3708 irq_err = REQ_IRQ_ERR_LPI; 3709 goto irq_error; 3710 } 3711 } 3712 3713 return 0; 3714 3715 irq_error: 3716 stmmac_free_irq(dev, irq_err, 0); 3717 return ret; 3718 } 3719 3720 static int stmmac_request_irq(struct net_device *dev) 3721 { 3722 struct stmmac_priv *priv = netdev_priv(dev); 3723 int ret; 3724 3725 /* Request the IRQ lines */ 3726 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 3727 ret = stmmac_request_irq_multi_msi(dev); 3728 else 3729 ret = stmmac_request_irq_single(dev); 3730 3731 return ret; 3732 } 3733 3734 /** 3735 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 3736 * @priv: driver private structure 3737 * @mtu: MTU to setup the dma queue and buf with 3738 * Description: Allocate and generate a dma_conf based on the provided MTU. 3739 * Allocate the Tx/Rx DMA queue and init them. 3740 * Return value: 3741 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 3742 */ 3743 static struct stmmac_dma_conf * 3744 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 3745 { 3746 struct stmmac_dma_conf *dma_conf; 3747 int chan, bfsize, ret; 3748 3749 dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); 3750 if (!dma_conf) { 3751 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 3752 __func__); 3753 return ERR_PTR(-ENOMEM); 3754 } 3755 3756 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 3757 if (bfsize < 0) 3758 bfsize = 0; 3759 3760 if (bfsize < BUF_SIZE_16KiB) 3761 bfsize = stmmac_set_bfsize(mtu, 0); 3762 3763 dma_conf->dma_buf_sz = bfsize; 3764 /* Chose the tx/rx size from the already defined one in the 3765 * priv struct. (if defined) 3766 */ 3767 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 3768 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 3769 3770 if (!dma_conf->dma_tx_size) 3771 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3772 if (!dma_conf->dma_rx_size) 3773 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3774 3775 /* Earlier check for TBS */ 3776 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3777 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 3778 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3779 3780 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3781 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3782 } 3783 3784 ret = alloc_dma_desc_resources(priv, dma_conf); 3785 if (ret < 0) { 3786 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 3787 __func__); 3788 goto alloc_error; 3789 } 3790 3791 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 3792 if (ret < 0) { 3793 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 3794 __func__); 3795 goto init_error; 3796 } 3797 3798 return dma_conf; 3799 3800 init_error: 3801 free_dma_desc_resources(priv, dma_conf); 3802 alloc_error: 3803 kfree(dma_conf); 3804 return ERR_PTR(ret); 3805 } 3806 3807 /** 3808 * __stmmac_open - open entry point of the driver 3809 * @dev : pointer to the device structure. 3810 * @dma_conf : structure to take the dma data 3811 * Description: 3812 * This function is the open entry point of the driver. 3813 * Return value: 3814 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3815 * file on failure. 3816 */ 3817 static int __stmmac_open(struct net_device *dev, 3818 struct stmmac_dma_conf *dma_conf) 3819 { 3820 struct stmmac_priv *priv = netdev_priv(dev); 3821 int mode = priv->plat->phy_interface; 3822 u32 chan; 3823 int ret; 3824 3825 ret = pm_runtime_resume_and_get(priv->device); 3826 if (ret < 0) 3827 return ret; 3828 3829 if (priv->hw->pcs != STMMAC_PCS_TBI && 3830 priv->hw->pcs != STMMAC_PCS_RTBI && 3831 (!priv->hw->xpcs || 3832 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73) && 3833 !priv->hw->lynx_pcs) { 3834 ret = stmmac_init_phy(dev); 3835 if (ret) { 3836 netdev_err(priv->dev, 3837 "%s: Cannot attach to PHY (error: %d)\n", 3838 __func__, ret); 3839 goto init_phy_error; 3840 } 3841 } 3842 3843 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 3844 3845 buf_sz = dma_conf->dma_buf_sz; 3846 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 3847 3848 stmmac_reset_queues_param(priv); 3849 3850 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 3851 priv->plat->serdes_powerup) { 3852 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); 3853 if (ret < 0) { 3854 netdev_err(priv->dev, "%s: Serdes powerup failed\n", 3855 __func__); 3856 goto init_error; 3857 } 3858 } 3859 3860 ret = stmmac_hw_setup(dev, true); 3861 if (ret < 0) { 3862 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 3863 goto init_error; 3864 } 3865 3866 stmmac_init_coalesce(priv); 3867 3868 phylink_start(priv->phylink); 3869 /* We may have called phylink_speed_down before */ 3870 phylink_speed_up(priv->phylink); 3871 3872 ret = stmmac_request_irq(dev); 3873 if (ret) 3874 goto irq_error; 3875 3876 stmmac_enable_all_queues(priv); 3877 netif_tx_start_all_queues(priv->dev); 3878 stmmac_enable_all_dma_irq(priv); 3879 3880 return 0; 3881 3882 irq_error: 3883 phylink_stop(priv->phylink); 3884 3885 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3886 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3887 3888 stmmac_hw_teardown(dev); 3889 init_error: 3890 phylink_disconnect_phy(priv->phylink); 3891 init_phy_error: 3892 pm_runtime_put(priv->device); 3893 return ret; 3894 } 3895 3896 static int stmmac_open(struct net_device *dev) 3897 { 3898 struct stmmac_priv *priv = netdev_priv(dev); 3899 struct stmmac_dma_conf *dma_conf; 3900 int ret; 3901 3902 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 3903 if (IS_ERR(dma_conf)) 3904 return PTR_ERR(dma_conf); 3905 3906 ret = __stmmac_open(dev, dma_conf); 3907 if (ret) 3908 free_dma_desc_resources(priv, dma_conf); 3909 3910 kfree(dma_conf); 3911 return ret; 3912 } 3913 3914 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) 3915 { 3916 set_bit(__FPE_REMOVING, &priv->fpe_task_state); 3917 3918 if (priv->fpe_wq) 3919 destroy_workqueue(priv->fpe_wq); 3920 3921 netdev_info(priv->dev, "FPE workqueue stop"); 3922 } 3923 3924 /** 3925 * stmmac_release - close entry point of the driver 3926 * @dev : device pointer. 3927 * Description: 3928 * This is the stop entry point of the driver. 3929 */ 3930 static int stmmac_release(struct net_device *dev) 3931 { 3932 struct stmmac_priv *priv = netdev_priv(dev); 3933 u32 chan; 3934 3935 if (device_may_wakeup(priv->device)) 3936 phylink_speed_down(priv->phylink, false); 3937 /* Stop and disconnect the PHY */ 3938 phylink_stop(priv->phylink); 3939 phylink_disconnect_phy(priv->phylink); 3940 3941 stmmac_disable_all_queues(priv); 3942 3943 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3944 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3945 3946 netif_tx_disable(dev); 3947 3948 /* Free the IRQ lines */ 3949 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 3950 3951 if (priv->eee_enabled) { 3952 priv->tx_path_in_lpi_mode = false; 3953 del_timer_sync(&priv->eee_ctrl_timer); 3954 } 3955 3956 /* Stop TX/RX DMA and clear the descriptors */ 3957 stmmac_stop_all_dma(priv); 3958 3959 /* Release and free the Rx/Tx resources */ 3960 free_dma_desc_resources(priv, &priv->dma_conf); 3961 3962 /* Disable the MAC Rx/Tx */ 3963 stmmac_mac_set(priv, priv->ioaddr, false); 3964 3965 /* Powerdown Serdes if there is */ 3966 if (priv->plat->serdes_powerdown) 3967 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); 3968 3969 netif_carrier_off(dev); 3970 3971 stmmac_release_ptp(priv); 3972 3973 pm_runtime_put(priv->device); 3974 3975 if (priv->dma_cap.fpesel) 3976 stmmac_fpe_stop_wq(priv); 3977 3978 return 0; 3979 } 3980 3981 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 3982 struct stmmac_tx_queue *tx_q) 3983 { 3984 u16 tag = 0x0, inner_tag = 0x0; 3985 u32 inner_type = 0x0; 3986 struct dma_desc *p; 3987 3988 if (!priv->dma_cap.vlins) 3989 return false; 3990 if (!skb_vlan_tag_present(skb)) 3991 return false; 3992 if (skb->vlan_proto == htons(ETH_P_8021AD)) { 3993 inner_tag = skb_vlan_tag_get(skb); 3994 inner_type = STMMAC_VLAN_INSERT; 3995 } 3996 3997 tag = skb_vlan_tag_get(skb); 3998 3999 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4000 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4001 else 4002 p = &tx_q->dma_tx[tx_q->cur_tx]; 4003 4004 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) 4005 return false; 4006 4007 stmmac_set_tx_owner(priv, p); 4008 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4009 return true; 4010 } 4011 4012 /** 4013 * stmmac_tso_allocator - close entry point of the driver 4014 * @priv: driver private structure 4015 * @des: buffer start address 4016 * @total_len: total length to fill in descriptors 4017 * @last_segment: condition for the last descriptor 4018 * @queue: TX queue index 4019 * Description: 4020 * This function fills descriptor and request new descriptors according to 4021 * buffer length to fill 4022 */ 4023 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4024 int total_len, bool last_segment, u32 queue) 4025 { 4026 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4027 struct dma_desc *desc; 4028 u32 buff_size; 4029 int tmp_len; 4030 4031 tmp_len = total_len; 4032 4033 while (tmp_len > 0) { 4034 dma_addr_t curr_addr; 4035 4036 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4037 priv->dma_conf.dma_tx_size); 4038 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4039 4040 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4041 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4042 else 4043 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4044 4045 curr_addr = des + (total_len - tmp_len); 4046 if (priv->dma_cap.addr64 <= 32) 4047 desc->des0 = cpu_to_le32(curr_addr); 4048 else 4049 stmmac_set_desc_addr(priv, desc, curr_addr); 4050 4051 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4052 TSO_MAX_BUFF_SIZE : tmp_len; 4053 4054 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4055 0, 1, 4056 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4057 0, 0); 4058 4059 tmp_len -= TSO_MAX_BUFF_SIZE; 4060 } 4061 } 4062 4063 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4064 { 4065 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4066 int desc_size; 4067 4068 if (likely(priv->extend_desc)) 4069 desc_size = sizeof(struct dma_extended_desc); 4070 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4071 desc_size = sizeof(struct dma_edesc); 4072 else 4073 desc_size = sizeof(struct dma_desc); 4074 4075 /* The own bit must be the latest setting done when prepare the 4076 * descriptor and then barrier is needed to make sure that 4077 * all is coherent before granting the DMA engine. 4078 */ 4079 wmb(); 4080 4081 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 4082 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 4083 } 4084 4085 /** 4086 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4087 * @skb : the socket buffer 4088 * @dev : device pointer 4089 * Description: this is the transmit function that is called on TSO frames 4090 * (support available on GMAC4 and newer chips). 4091 * Diagram below show the ring programming in case of TSO frames: 4092 * 4093 * First Descriptor 4094 * -------- 4095 * | DES0 |---> buffer1 = L2/L3/L4 header 4096 * | DES1 |---> TCP Payload (can continue on next descr...) 4097 * | DES2 |---> buffer 1 and 2 len 4098 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4099 * -------- 4100 * | 4101 * ... 4102 * | 4103 * -------- 4104 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 4105 * | DES1 | --| 4106 * | DES2 | --> buffer 1 and 2 len 4107 * | DES3 | 4108 * -------- 4109 * 4110 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4111 */ 4112 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4113 { 4114 struct dma_desc *desc, *first, *mss_desc = NULL; 4115 struct stmmac_priv *priv = netdev_priv(dev); 4116 int nfrags = skb_shinfo(skb)->nr_frags; 4117 u32 queue = skb_get_queue_mapping(skb); 4118 unsigned int first_entry, tx_packets; 4119 struct stmmac_txq_stats *txq_stats; 4120 int tmp_pay_len = 0, first_tx; 4121 struct stmmac_tx_queue *tx_q; 4122 bool has_vlan, set_ic; 4123 u8 proto_hdr_len, hdr; 4124 unsigned long flags; 4125 u32 pay_len, mss; 4126 dma_addr_t des; 4127 int i; 4128 4129 tx_q = &priv->dma_conf.tx_queue[queue]; 4130 txq_stats = &priv->xstats.txq_stats[queue]; 4131 first_tx = tx_q->cur_tx; 4132 4133 /* Compute header lengths */ 4134 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 4135 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 4136 hdr = sizeof(struct udphdr); 4137 } else { 4138 proto_hdr_len = skb_tcp_all_headers(skb); 4139 hdr = tcp_hdrlen(skb); 4140 } 4141 4142 /* Desc availability based on threshold should be enough safe */ 4143 if (unlikely(stmmac_tx_avail(priv, queue) < 4144 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4145 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4146 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4147 queue)); 4148 /* This is a hard error, log it. */ 4149 netdev_err(priv->dev, 4150 "%s: Tx Ring full when queue awake\n", 4151 __func__); 4152 } 4153 return NETDEV_TX_BUSY; 4154 } 4155 4156 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4157 4158 mss = skb_shinfo(skb)->gso_size; 4159 4160 /* set new MSS value if needed */ 4161 if (mss != tx_q->mss) { 4162 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4163 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4164 else 4165 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4166 4167 stmmac_set_mss(priv, mss_desc, mss); 4168 tx_q->mss = mss; 4169 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4170 priv->dma_conf.dma_tx_size); 4171 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4172 } 4173 4174 if (netif_msg_tx_queued(priv)) { 4175 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4176 __func__, hdr, proto_hdr_len, pay_len, mss); 4177 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4178 skb->data_len); 4179 } 4180 4181 /* Check if VLAN can be inserted by HW */ 4182 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4183 4184 first_entry = tx_q->cur_tx; 4185 WARN_ON(tx_q->tx_skbuff[first_entry]); 4186 4187 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4188 desc = &tx_q->dma_entx[first_entry].basic; 4189 else 4190 desc = &tx_q->dma_tx[first_entry]; 4191 first = desc; 4192 4193 if (has_vlan) 4194 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4195 4196 /* first descriptor: fill Headers on Buf1 */ 4197 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4198 DMA_TO_DEVICE); 4199 if (dma_mapping_error(priv->device, des)) 4200 goto dma_map_err; 4201 4202 tx_q->tx_skbuff_dma[first_entry].buf = des; 4203 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 4204 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4205 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4206 4207 if (priv->dma_cap.addr64 <= 32) { 4208 first->des0 = cpu_to_le32(des); 4209 4210 /* Fill start of payload in buff2 of first descriptor */ 4211 if (pay_len) 4212 first->des1 = cpu_to_le32(des + proto_hdr_len); 4213 4214 /* If needed take extra descriptors to fill the remaining payload */ 4215 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 4216 } else { 4217 stmmac_set_desc_addr(priv, first, des); 4218 tmp_pay_len = pay_len; 4219 des += proto_hdr_len; 4220 pay_len = 0; 4221 } 4222 4223 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 4224 4225 /* Prepare fragments */ 4226 for (i = 0; i < nfrags; i++) { 4227 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4228 4229 des = skb_frag_dma_map(priv->device, frag, 0, 4230 skb_frag_size(frag), 4231 DMA_TO_DEVICE); 4232 if (dma_mapping_error(priv->device, des)) 4233 goto dma_map_err; 4234 4235 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4236 (i == nfrags - 1), queue); 4237 4238 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4239 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4240 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4241 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4242 } 4243 4244 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4245 4246 /* Only the last descriptor gets to point to the skb. */ 4247 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4248 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4249 4250 /* Manage tx mitigation */ 4251 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4252 tx_q->tx_count_frames += tx_packets; 4253 4254 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4255 set_ic = true; 4256 else if (!priv->tx_coal_frames[queue]) 4257 set_ic = false; 4258 else if (tx_packets > priv->tx_coal_frames[queue]) 4259 set_ic = true; 4260 else if ((tx_q->tx_count_frames % 4261 priv->tx_coal_frames[queue]) < tx_packets) 4262 set_ic = true; 4263 else 4264 set_ic = false; 4265 4266 if (set_ic) { 4267 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4268 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4269 else 4270 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4271 4272 tx_q->tx_count_frames = 0; 4273 stmmac_set_tx_ic(priv, desc); 4274 } 4275 4276 /* We've used all descriptors we need for this skb, however, 4277 * advance cur_tx so that it references a fresh descriptor. 4278 * ndo_start_xmit will fill this descriptor the next time it's 4279 * called and stmmac_tx_clean may clean up to this descriptor. 4280 */ 4281 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4282 4283 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4284 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4285 __func__); 4286 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4287 } 4288 4289 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 4290 txq_stats->tx_bytes += skb->len; 4291 txq_stats->tx_tso_frames++; 4292 txq_stats->tx_tso_nfrags += nfrags; 4293 if (set_ic) 4294 txq_stats->tx_set_ic_bit++; 4295 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 4296 4297 if (priv->sarc_type) 4298 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4299 4300 skb_tx_timestamp(skb); 4301 4302 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4303 priv->hwts_tx_en)) { 4304 /* declare that device is doing timestamping */ 4305 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4306 stmmac_enable_tx_timestamp(priv, first); 4307 } 4308 4309 /* Complete the first descriptor before granting the DMA */ 4310 stmmac_prepare_tso_tx_desc(priv, first, 1, 4311 proto_hdr_len, 4312 pay_len, 4313 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 4314 hdr / 4, (skb->len - proto_hdr_len)); 4315 4316 /* If context desc is used to change MSS */ 4317 if (mss_desc) { 4318 /* Make sure that first descriptor has been completely 4319 * written, including its own bit. This is because MSS is 4320 * actually before first descriptor, so we need to make 4321 * sure that MSS's own bit is the last thing written. 4322 */ 4323 dma_wmb(); 4324 stmmac_set_tx_owner(priv, mss_desc); 4325 } 4326 4327 if (netif_msg_pktdata(priv)) { 4328 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4329 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4330 tx_q->cur_tx, first, nfrags); 4331 pr_info(">>> frame to be transmitted: "); 4332 print_pkt(skb->data, skb_headlen(skb)); 4333 } 4334 4335 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4336 4337 stmmac_flush_tx_descriptors(priv, queue); 4338 stmmac_tx_timer_arm(priv, queue); 4339 4340 return NETDEV_TX_OK; 4341 4342 dma_map_err: 4343 dev_err(priv->device, "Tx dma map failed\n"); 4344 dev_kfree_skb(skb); 4345 priv->xstats.tx_dropped++; 4346 return NETDEV_TX_OK; 4347 } 4348 4349 /** 4350 * stmmac_xmit - Tx entry point of the driver 4351 * @skb : the socket buffer 4352 * @dev : device pointer 4353 * Description : this is the tx entry point of the driver. 4354 * It programs the chain or the ring and supports oversized frames 4355 * and SG feature. 4356 */ 4357 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4358 { 4359 unsigned int first_entry, tx_packets, enh_desc; 4360 struct stmmac_priv *priv = netdev_priv(dev); 4361 unsigned int nopaged_len = skb_headlen(skb); 4362 int i, csum_insertion = 0, is_jumbo = 0; 4363 u32 queue = skb_get_queue_mapping(skb); 4364 int nfrags = skb_shinfo(skb)->nr_frags; 4365 int gso = skb_shinfo(skb)->gso_type; 4366 struct stmmac_txq_stats *txq_stats; 4367 struct dma_edesc *tbs_desc = NULL; 4368 struct dma_desc *desc, *first; 4369 struct stmmac_tx_queue *tx_q; 4370 bool has_vlan, set_ic; 4371 int entry, first_tx; 4372 unsigned long flags; 4373 dma_addr_t des; 4374 4375 tx_q = &priv->dma_conf.tx_queue[queue]; 4376 txq_stats = &priv->xstats.txq_stats[queue]; 4377 first_tx = tx_q->cur_tx; 4378 4379 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4380 stmmac_disable_eee_mode(priv); 4381 4382 /* Manage oversized TCP frames for GMAC4 device */ 4383 if (skb_is_gso(skb) && priv->tso) { 4384 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4385 return stmmac_tso_xmit(skb, dev); 4386 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) 4387 return stmmac_tso_xmit(skb, dev); 4388 } 4389 4390 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4391 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4392 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4393 queue)); 4394 /* This is a hard error, log it. */ 4395 netdev_err(priv->dev, 4396 "%s: Tx Ring full when queue awake\n", 4397 __func__); 4398 } 4399 return NETDEV_TX_BUSY; 4400 } 4401 4402 /* Check if VLAN can be inserted by HW */ 4403 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4404 4405 entry = tx_q->cur_tx; 4406 first_entry = entry; 4407 WARN_ON(tx_q->tx_skbuff[first_entry]); 4408 4409 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4410 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 4411 * queues. In that case, checksum offloading for those queues that don't 4412 * support tx coe needs to fallback to software checksum calculation. 4413 */ 4414 if (csum_insertion && 4415 priv->plat->tx_queues_cfg[queue].coe_unsupported) { 4416 if (unlikely(skb_checksum_help(skb))) 4417 goto dma_map_err; 4418 csum_insertion = !csum_insertion; 4419 } 4420 4421 if (likely(priv->extend_desc)) 4422 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4423 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4424 desc = &tx_q->dma_entx[entry].basic; 4425 else 4426 desc = tx_q->dma_tx + entry; 4427 4428 first = desc; 4429 4430 if (has_vlan) 4431 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4432 4433 enh_desc = priv->plat->enh_desc; 4434 /* To program the descriptors according to the size of the frame */ 4435 if (enh_desc) 4436 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4437 4438 if (unlikely(is_jumbo)) { 4439 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4440 if (unlikely(entry < 0) && (entry != -EINVAL)) 4441 goto dma_map_err; 4442 } 4443 4444 for (i = 0; i < nfrags; i++) { 4445 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4446 int len = skb_frag_size(frag); 4447 bool last_segment = (i == (nfrags - 1)); 4448 4449 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4450 WARN_ON(tx_q->tx_skbuff[entry]); 4451 4452 if (likely(priv->extend_desc)) 4453 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4454 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4455 desc = &tx_q->dma_entx[entry].basic; 4456 else 4457 desc = tx_q->dma_tx + entry; 4458 4459 des = skb_frag_dma_map(priv->device, frag, 0, len, 4460 DMA_TO_DEVICE); 4461 if (dma_mapping_error(priv->device, des)) 4462 goto dma_map_err; /* should reuse desc w/o issues */ 4463 4464 tx_q->tx_skbuff_dma[entry].buf = des; 4465 4466 stmmac_set_desc_addr(priv, desc, des); 4467 4468 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4469 tx_q->tx_skbuff_dma[entry].len = len; 4470 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4471 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4472 4473 /* Prepare the descriptor and set the own bit too */ 4474 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4475 priv->mode, 1, last_segment, skb->len); 4476 } 4477 4478 /* Only the last descriptor gets to point to the skb. */ 4479 tx_q->tx_skbuff[entry] = skb; 4480 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4481 4482 /* According to the coalesce parameter the IC bit for the latest 4483 * segment is reset and the timer re-started to clean the tx status. 4484 * This approach takes care about the fragments: desc is the first 4485 * element in case of no SG. 4486 */ 4487 tx_packets = (entry + 1) - first_tx; 4488 tx_q->tx_count_frames += tx_packets; 4489 4490 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4491 set_ic = true; 4492 else if (!priv->tx_coal_frames[queue]) 4493 set_ic = false; 4494 else if (tx_packets > priv->tx_coal_frames[queue]) 4495 set_ic = true; 4496 else if ((tx_q->tx_count_frames % 4497 priv->tx_coal_frames[queue]) < tx_packets) 4498 set_ic = true; 4499 else 4500 set_ic = false; 4501 4502 if (set_ic) { 4503 if (likely(priv->extend_desc)) 4504 desc = &tx_q->dma_etx[entry].basic; 4505 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4506 desc = &tx_q->dma_entx[entry].basic; 4507 else 4508 desc = &tx_q->dma_tx[entry]; 4509 4510 tx_q->tx_count_frames = 0; 4511 stmmac_set_tx_ic(priv, desc); 4512 } 4513 4514 /* We've used all descriptors we need for this skb, however, 4515 * advance cur_tx so that it references a fresh descriptor. 4516 * ndo_start_xmit will fill this descriptor the next time it's 4517 * called and stmmac_tx_clean may clean up to this descriptor. 4518 */ 4519 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4520 tx_q->cur_tx = entry; 4521 4522 if (netif_msg_pktdata(priv)) { 4523 netdev_dbg(priv->dev, 4524 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4525 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4526 entry, first, nfrags); 4527 4528 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4529 print_pkt(skb->data, skb->len); 4530 } 4531 4532 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4533 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4534 __func__); 4535 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4536 } 4537 4538 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 4539 txq_stats->tx_bytes += skb->len; 4540 if (set_ic) 4541 txq_stats->tx_set_ic_bit++; 4542 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 4543 4544 if (priv->sarc_type) 4545 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4546 4547 skb_tx_timestamp(skb); 4548 4549 /* Ready to fill the first descriptor and set the OWN bit w/o any 4550 * problems because all the descriptors are actually ready to be 4551 * passed to the DMA engine. 4552 */ 4553 if (likely(!is_jumbo)) { 4554 bool last_segment = (nfrags == 0); 4555 4556 des = dma_map_single(priv->device, skb->data, 4557 nopaged_len, DMA_TO_DEVICE); 4558 if (dma_mapping_error(priv->device, des)) 4559 goto dma_map_err; 4560 4561 tx_q->tx_skbuff_dma[first_entry].buf = des; 4562 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4563 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4564 4565 stmmac_set_desc_addr(priv, first, des); 4566 4567 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4568 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4569 4570 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4571 priv->hwts_tx_en)) { 4572 /* declare that device is doing timestamping */ 4573 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4574 stmmac_enable_tx_timestamp(priv, first); 4575 } 4576 4577 /* Prepare the first descriptor setting the OWN bit too */ 4578 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4579 csum_insertion, priv->mode, 0, last_segment, 4580 skb->len); 4581 } 4582 4583 if (tx_q->tbs & STMMAC_TBS_EN) { 4584 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4585 4586 tbs_desc = &tx_q->dma_entx[first_entry]; 4587 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4588 } 4589 4590 stmmac_set_tx_owner(priv, first); 4591 4592 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4593 4594 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4595 4596 stmmac_flush_tx_descriptors(priv, queue); 4597 stmmac_tx_timer_arm(priv, queue); 4598 4599 return NETDEV_TX_OK; 4600 4601 dma_map_err: 4602 netdev_err(priv->dev, "Tx DMA map failed\n"); 4603 dev_kfree_skb(skb); 4604 priv->xstats.tx_dropped++; 4605 return NETDEV_TX_OK; 4606 } 4607 4608 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4609 { 4610 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 4611 __be16 vlan_proto = veth->h_vlan_proto; 4612 u16 vlanid; 4613 4614 if ((vlan_proto == htons(ETH_P_8021Q) && 4615 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4616 (vlan_proto == htons(ETH_P_8021AD) && 4617 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4618 /* pop the vlan tag */ 4619 vlanid = ntohs(veth->h_vlan_TCI); 4620 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4621 skb_pull(skb, VLAN_HLEN); 4622 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4623 } 4624 } 4625 4626 /** 4627 * stmmac_rx_refill - refill used skb preallocated buffers 4628 * @priv: driver private structure 4629 * @queue: RX queue index 4630 * Description : this is to reallocate the skb for the reception process 4631 * that is based on zero-copy. 4632 */ 4633 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4634 { 4635 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4636 int dirty = stmmac_rx_dirty(priv, queue); 4637 unsigned int entry = rx_q->dirty_rx; 4638 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4639 4640 if (priv->dma_cap.host_dma_width <= 32) 4641 gfp |= GFP_DMA32; 4642 4643 while (dirty-- > 0) { 4644 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4645 struct dma_desc *p; 4646 bool use_rx_wd; 4647 4648 if (priv->extend_desc) 4649 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4650 else 4651 p = rx_q->dma_rx + entry; 4652 4653 if (!buf->page) { 4654 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4655 if (!buf->page) 4656 break; 4657 } 4658 4659 if (priv->sph && !buf->sec_page) { 4660 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4661 if (!buf->sec_page) 4662 break; 4663 4664 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4665 } 4666 4667 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4668 4669 stmmac_set_desc_addr(priv, p, buf->addr); 4670 if (priv->sph) 4671 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4672 else 4673 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4674 stmmac_refill_desc3(priv, rx_q, p); 4675 4676 rx_q->rx_count_frames++; 4677 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4678 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4679 rx_q->rx_count_frames = 0; 4680 4681 use_rx_wd = !priv->rx_coal_frames[queue]; 4682 use_rx_wd |= rx_q->rx_count_frames > 0; 4683 if (!priv->use_riwt) 4684 use_rx_wd = false; 4685 4686 dma_wmb(); 4687 stmmac_set_rx_owner(priv, p, use_rx_wd); 4688 4689 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 4690 } 4691 rx_q->dirty_rx = entry; 4692 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4693 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4694 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4695 } 4696 4697 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4698 struct dma_desc *p, 4699 int status, unsigned int len) 4700 { 4701 unsigned int plen = 0, hlen = 0; 4702 int coe = priv->hw->rx_csum; 4703 4704 /* Not first descriptor, buffer is always zero */ 4705 if (priv->sph && len) 4706 return 0; 4707 4708 /* First descriptor, get split header length */ 4709 stmmac_get_rx_header_len(priv, p, &hlen); 4710 if (priv->sph && hlen) { 4711 priv->xstats.rx_split_hdr_pkt_n++; 4712 return hlen; 4713 } 4714 4715 /* First descriptor, not last descriptor and not split header */ 4716 if (status & rx_not_ls) 4717 return priv->dma_conf.dma_buf_sz; 4718 4719 plen = stmmac_get_rx_frame_len(priv, p, coe); 4720 4721 /* First descriptor and last descriptor and not split header */ 4722 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 4723 } 4724 4725 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4726 struct dma_desc *p, 4727 int status, unsigned int len) 4728 { 4729 int coe = priv->hw->rx_csum; 4730 unsigned int plen = 0; 4731 4732 /* Not split header, buffer is not available */ 4733 if (!priv->sph) 4734 return 0; 4735 4736 /* Not last descriptor */ 4737 if (status & rx_not_ls) 4738 return priv->dma_conf.dma_buf_sz; 4739 4740 plen = stmmac_get_rx_frame_len(priv, p, coe); 4741 4742 /* Last descriptor */ 4743 return plen - len; 4744 } 4745 4746 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 4747 struct xdp_frame *xdpf, bool dma_map) 4748 { 4749 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 4750 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4751 unsigned int entry = tx_q->cur_tx; 4752 struct dma_desc *tx_desc; 4753 dma_addr_t dma_addr; 4754 bool set_ic; 4755 4756 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 4757 return STMMAC_XDP_CONSUMED; 4758 4759 if (likely(priv->extend_desc)) 4760 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4761 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4762 tx_desc = &tx_q->dma_entx[entry].basic; 4763 else 4764 tx_desc = tx_q->dma_tx + entry; 4765 4766 if (dma_map) { 4767 dma_addr = dma_map_single(priv->device, xdpf->data, 4768 xdpf->len, DMA_TO_DEVICE); 4769 if (dma_mapping_error(priv->device, dma_addr)) 4770 return STMMAC_XDP_CONSUMED; 4771 4772 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 4773 } else { 4774 struct page *page = virt_to_page(xdpf->data); 4775 4776 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 4777 xdpf->headroom; 4778 dma_sync_single_for_device(priv->device, dma_addr, 4779 xdpf->len, DMA_BIDIRECTIONAL); 4780 4781 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 4782 } 4783 4784 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 4785 tx_q->tx_skbuff_dma[entry].map_as_page = false; 4786 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 4787 tx_q->tx_skbuff_dma[entry].last_segment = true; 4788 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 4789 4790 tx_q->xdpf[entry] = xdpf; 4791 4792 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 4793 4794 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 4795 true, priv->mode, true, true, 4796 xdpf->len); 4797 4798 tx_q->tx_count_frames++; 4799 4800 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 4801 set_ic = true; 4802 else 4803 set_ic = false; 4804 4805 if (set_ic) { 4806 unsigned long flags; 4807 tx_q->tx_count_frames = 0; 4808 stmmac_set_tx_ic(priv, tx_desc); 4809 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 4810 txq_stats->tx_set_ic_bit++; 4811 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 4812 } 4813 4814 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4815 4816 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4817 tx_q->cur_tx = entry; 4818 4819 return STMMAC_XDP_TX; 4820 } 4821 4822 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 4823 int cpu) 4824 { 4825 int index = cpu; 4826 4827 if (unlikely(index < 0)) 4828 index = 0; 4829 4830 while (index >= priv->plat->tx_queues_to_use) 4831 index -= priv->plat->tx_queues_to_use; 4832 4833 return index; 4834 } 4835 4836 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 4837 struct xdp_buff *xdp) 4838 { 4839 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 4840 int cpu = smp_processor_id(); 4841 struct netdev_queue *nq; 4842 int queue; 4843 int res; 4844 4845 if (unlikely(!xdpf)) 4846 return STMMAC_XDP_CONSUMED; 4847 4848 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4849 nq = netdev_get_tx_queue(priv->dev, queue); 4850 4851 __netif_tx_lock(nq, cpu); 4852 /* Avoids TX time-out as we are sharing with slow path */ 4853 txq_trans_cond_update(nq); 4854 4855 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); 4856 if (res == STMMAC_XDP_TX) 4857 stmmac_flush_tx_descriptors(priv, queue); 4858 4859 __netif_tx_unlock(nq); 4860 4861 return res; 4862 } 4863 4864 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 4865 struct bpf_prog *prog, 4866 struct xdp_buff *xdp) 4867 { 4868 u32 act; 4869 int res; 4870 4871 act = bpf_prog_run_xdp(prog, xdp); 4872 switch (act) { 4873 case XDP_PASS: 4874 res = STMMAC_XDP_PASS; 4875 break; 4876 case XDP_TX: 4877 res = stmmac_xdp_xmit_back(priv, xdp); 4878 break; 4879 case XDP_REDIRECT: 4880 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 4881 res = STMMAC_XDP_CONSUMED; 4882 else 4883 res = STMMAC_XDP_REDIRECT; 4884 break; 4885 default: 4886 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 4887 fallthrough; 4888 case XDP_ABORTED: 4889 trace_xdp_exception(priv->dev, prog, act); 4890 fallthrough; 4891 case XDP_DROP: 4892 res = STMMAC_XDP_CONSUMED; 4893 break; 4894 } 4895 4896 return res; 4897 } 4898 4899 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 4900 struct xdp_buff *xdp) 4901 { 4902 struct bpf_prog *prog; 4903 int res; 4904 4905 prog = READ_ONCE(priv->xdp_prog); 4906 if (!prog) { 4907 res = STMMAC_XDP_PASS; 4908 goto out; 4909 } 4910 4911 res = __stmmac_xdp_run_prog(priv, prog, xdp); 4912 out: 4913 return ERR_PTR(-res); 4914 } 4915 4916 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 4917 int xdp_status) 4918 { 4919 int cpu = smp_processor_id(); 4920 int queue; 4921 4922 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4923 4924 if (xdp_status & STMMAC_XDP_TX) 4925 stmmac_tx_timer_arm(priv, queue); 4926 4927 if (xdp_status & STMMAC_XDP_REDIRECT) 4928 xdp_do_flush(); 4929 } 4930 4931 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 4932 struct xdp_buff *xdp) 4933 { 4934 unsigned int metasize = xdp->data - xdp->data_meta; 4935 unsigned int datasize = xdp->data_end - xdp->data; 4936 struct sk_buff *skb; 4937 4938 skb = __napi_alloc_skb(&ch->rxtx_napi, 4939 xdp->data_end - xdp->data_hard_start, 4940 GFP_ATOMIC | __GFP_NOWARN); 4941 if (unlikely(!skb)) 4942 return NULL; 4943 4944 skb_reserve(skb, xdp->data - xdp->data_hard_start); 4945 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 4946 if (metasize) 4947 skb_metadata_set(skb, metasize); 4948 4949 return skb; 4950 } 4951 4952 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 4953 struct dma_desc *p, struct dma_desc *np, 4954 struct xdp_buff *xdp) 4955 { 4956 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 4957 struct stmmac_channel *ch = &priv->channel[queue]; 4958 unsigned int len = xdp->data_end - xdp->data; 4959 enum pkt_hash_types hash_type; 4960 int coe = priv->hw->rx_csum; 4961 unsigned long flags; 4962 struct sk_buff *skb; 4963 u32 hash; 4964 4965 skb = stmmac_construct_skb_zc(ch, xdp); 4966 if (!skb) { 4967 priv->xstats.rx_dropped++; 4968 return; 4969 } 4970 4971 stmmac_get_rx_hwtstamp(priv, p, np, skb); 4972 stmmac_rx_vlan(priv->dev, skb); 4973 skb->protocol = eth_type_trans(skb, priv->dev); 4974 4975 if (unlikely(!coe)) 4976 skb_checksum_none_assert(skb); 4977 else 4978 skb->ip_summed = CHECKSUM_UNNECESSARY; 4979 4980 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 4981 skb_set_hash(skb, hash, hash_type); 4982 4983 skb_record_rx_queue(skb, queue); 4984 napi_gro_receive(&ch->rxtx_napi, skb); 4985 4986 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 4987 rxq_stats->rx_pkt_n++; 4988 rxq_stats->rx_bytes += len; 4989 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 4990 } 4991 4992 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 4993 { 4994 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4995 unsigned int entry = rx_q->dirty_rx; 4996 struct dma_desc *rx_desc = NULL; 4997 bool ret = true; 4998 4999 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5000 5001 while (budget-- > 0 && entry != rx_q->cur_rx) { 5002 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5003 dma_addr_t dma_addr; 5004 bool use_rx_wd; 5005 5006 if (!buf->xdp) { 5007 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5008 if (!buf->xdp) { 5009 ret = false; 5010 break; 5011 } 5012 } 5013 5014 if (priv->extend_desc) 5015 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 5016 else 5017 rx_desc = rx_q->dma_rx + entry; 5018 5019 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5020 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5021 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5022 stmmac_refill_desc3(priv, rx_q, rx_desc); 5023 5024 rx_q->rx_count_frames++; 5025 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5026 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5027 rx_q->rx_count_frames = 0; 5028 5029 use_rx_wd = !priv->rx_coal_frames[queue]; 5030 use_rx_wd |= rx_q->rx_count_frames > 0; 5031 if (!priv->use_riwt) 5032 use_rx_wd = false; 5033 5034 dma_wmb(); 5035 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5036 5037 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 5038 } 5039 5040 if (rx_desc) { 5041 rx_q->dirty_rx = entry; 5042 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 5043 (rx_q->dirty_rx * sizeof(struct dma_desc)); 5044 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 5045 } 5046 5047 return ret; 5048 } 5049 5050 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5051 { 5052 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5053 * to represent incoming packet, whereas cb field in the same structure 5054 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5055 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5056 */ 5057 return (struct stmmac_xdp_buff *)xdp; 5058 } 5059 5060 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5061 { 5062 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5063 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5064 unsigned int count = 0, error = 0, len = 0; 5065 int dirty = stmmac_rx_dirty(priv, queue); 5066 unsigned int next_entry = rx_q->cur_rx; 5067 u32 rx_errors = 0, rx_dropped = 0; 5068 unsigned int desc_size; 5069 struct bpf_prog *prog; 5070 bool failure = false; 5071 unsigned long flags; 5072 int xdp_status = 0; 5073 int status = 0; 5074 5075 if (netif_msg_rx_status(priv)) { 5076 void *rx_head; 5077 5078 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5079 if (priv->extend_desc) { 5080 rx_head = (void *)rx_q->dma_erx; 5081 desc_size = sizeof(struct dma_extended_desc); 5082 } else { 5083 rx_head = (void *)rx_q->dma_rx; 5084 desc_size = sizeof(struct dma_desc); 5085 } 5086 5087 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5088 rx_q->dma_rx_phy, desc_size); 5089 } 5090 while (count < limit) { 5091 struct stmmac_rx_buffer *buf; 5092 struct stmmac_xdp_buff *ctx; 5093 unsigned int buf1_len = 0; 5094 struct dma_desc *np, *p; 5095 int entry; 5096 int res; 5097 5098 if (!count && rx_q->state_saved) { 5099 error = rx_q->state.error; 5100 len = rx_q->state.len; 5101 } else { 5102 rx_q->state_saved = false; 5103 error = 0; 5104 len = 0; 5105 } 5106 5107 if (count >= limit) 5108 break; 5109 5110 read_again: 5111 buf1_len = 0; 5112 entry = next_entry; 5113 buf = &rx_q->buf_pool[entry]; 5114 5115 if (dirty >= STMMAC_RX_FILL_BATCH) { 5116 failure = failure || 5117 !stmmac_rx_refill_zc(priv, queue, dirty); 5118 dirty = 0; 5119 } 5120 5121 if (priv->extend_desc) 5122 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5123 else 5124 p = rx_q->dma_rx + entry; 5125 5126 /* read the status of the incoming frame */ 5127 status = stmmac_rx_status(priv, &priv->xstats, p); 5128 /* check if managed by the DMA otherwise go ahead */ 5129 if (unlikely(status & dma_own)) 5130 break; 5131 5132 /* Prefetch the next RX descriptor */ 5133 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5134 priv->dma_conf.dma_rx_size); 5135 next_entry = rx_q->cur_rx; 5136 5137 if (priv->extend_desc) 5138 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5139 else 5140 np = rx_q->dma_rx + next_entry; 5141 5142 prefetch(np); 5143 5144 /* Ensure a valid XSK buffer before proceed */ 5145 if (!buf->xdp) 5146 break; 5147 5148 if (priv->extend_desc) 5149 stmmac_rx_extended_status(priv, &priv->xstats, 5150 rx_q->dma_erx + entry); 5151 if (unlikely(status == discard_frame)) { 5152 xsk_buff_free(buf->xdp); 5153 buf->xdp = NULL; 5154 dirty++; 5155 error = 1; 5156 if (!priv->hwts_rx_en) 5157 rx_errors++; 5158 } 5159 5160 if (unlikely(error && (status & rx_not_ls))) 5161 goto read_again; 5162 if (unlikely(error)) { 5163 count++; 5164 continue; 5165 } 5166 5167 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5168 if (likely(status & rx_not_ls)) { 5169 xsk_buff_free(buf->xdp); 5170 buf->xdp = NULL; 5171 dirty++; 5172 count++; 5173 goto read_again; 5174 } 5175 5176 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5177 ctx->priv = priv; 5178 ctx->desc = p; 5179 ctx->ndesc = np; 5180 5181 /* XDP ZC Frame only support primary buffers for now */ 5182 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5183 len += buf1_len; 5184 5185 /* ACS is disabled; strip manually. */ 5186 if (likely(!(status & rx_not_ls))) { 5187 buf1_len -= ETH_FCS_LEN; 5188 len -= ETH_FCS_LEN; 5189 } 5190 5191 /* RX buffer is good and fit into a XSK pool buffer */ 5192 buf->xdp->data_end = buf->xdp->data + buf1_len; 5193 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); 5194 5195 prog = READ_ONCE(priv->xdp_prog); 5196 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5197 5198 switch (res) { 5199 case STMMAC_XDP_PASS: 5200 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5201 xsk_buff_free(buf->xdp); 5202 break; 5203 case STMMAC_XDP_CONSUMED: 5204 xsk_buff_free(buf->xdp); 5205 rx_dropped++; 5206 break; 5207 case STMMAC_XDP_TX: 5208 case STMMAC_XDP_REDIRECT: 5209 xdp_status |= res; 5210 break; 5211 } 5212 5213 buf->xdp = NULL; 5214 dirty++; 5215 count++; 5216 } 5217 5218 if (status & rx_not_ls) { 5219 rx_q->state_saved = true; 5220 rx_q->state.error = error; 5221 rx_q->state.len = len; 5222 } 5223 5224 stmmac_finalize_xdp_rx(priv, xdp_status); 5225 5226 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5227 rxq_stats->rx_pkt_n += count; 5228 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5229 5230 priv->xstats.rx_dropped += rx_dropped; 5231 priv->xstats.rx_errors += rx_errors; 5232 5233 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5234 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5235 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5236 else 5237 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5238 5239 return (int)count; 5240 } 5241 5242 return failure ? limit : (int)count; 5243 } 5244 5245 /** 5246 * stmmac_rx - manage the receive process 5247 * @priv: driver private structure 5248 * @limit: napi bugget 5249 * @queue: RX queue index. 5250 * Description : this the function called by the napi poll method. 5251 * It gets all the frames inside the ring. 5252 */ 5253 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5254 { 5255 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5256 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5257 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5258 struct stmmac_channel *ch = &priv->channel[queue]; 5259 unsigned int count = 0, error = 0, len = 0; 5260 int status = 0, coe = priv->hw->rx_csum; 5261 unsigned int next_entry = rx_q->cur_rx; 5262 enum dma_data_direction dma_dir; 5263 unsigned int desc_size; 5264 struct sk_buff *skb = NULL; 5265 struct stmmac_xdp_buff ctx; 5266 unsigned long flags; 5267 int xdp_status = 0; 5268 int buf_sz; 5269 5270 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5271 buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5272 5273 if (netif_msg_rx_status(priv)) { 5274 void *rx_head; 5275 5276 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5277 if (priv->extend_desc) { 5278 rx_head = (void *)rx_q->dma_erx; 5279 desc_size = sizeof(struct dma_extended_desc); 5280 } else { 5281 rx_head = (void *)rx_q->dma_rx; 5282 desc_size = sizeof(struct dma_desc); 5283 } 5284 5285 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5286 rx_q->dma_rx_phy, desc_size); 5287 } 5288 while (count < limit) { 5289 unsigned int buf1_len = 0, buf2_len = 0; 5290 enum pkt_hash_types hash_type; 5291 struct stmmac_rx_buffer *buf; 5292 struct dma_desc *np, *p; 5293 int entry; 5294 u32 hash; 5295 5296 if (!count && rx_q->state_saved) { 5297 skb = rx_q->state.skb; 5298 error = rx_q->state.error; 5299 len = rx_q->state.len; 5300 } else { 5301 rx_q->state_saved = false; 5302 skb = NULL; 5303 error = 0; 5304 len = 0; 5305 } 5306 5307 if (count >= limit) 5308 break; 5309 5310 read_again: 5311 buf1_len = 0; 5312 buf2_len = 0; 5313 entry = next_entry; 5314 buf = &rx_q->buf_pool[entry]; 5315 5316 if (priv->extend_desc) 5317 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5318 else 5319 p = rx_q->dma_rx + entry; 5320 5321 /* read the status of the incoming frame */ 5322 status = stmmac_rx_status(priv, &priv->xstats, p); 5323 /* check if managed by the DMA otherwise go ahead */ 5324 if (unlikely(status & dma_own)) 5325 break; 5326 5327 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5328 priv->dma_conf.dma_rx_size); 5329 next_entry = rx_q->cur_rx; 5330 5331 if (priv->extend_desc) 5332 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5333 else 5334 np = rx_q->dma_rx + next_entry; 5335 5336 prefetch(np); 5337 5338 if (priv->extend_desc) 5339 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5340 if (unlikely(status == discard_frame)) { 5341 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5342 buf->page = NULL; 5343 error = 1; 5344 if (!priv->hwts_rx_en) 5345 rx_errors++; 5346 } 5347 5348 if (unlikely(error && (status & rx_not_ls))) 5349 goto read_again; 5350 if (unlikely(error)) { 5351 dev_kfree_skb(skb); 5352 skb = NULL; 5353 count++; 5354 continue; 5355 } 5356 5357 /* Buffer is good. Go on. */ 5358 5359 prefetch(page_address(buf->page) + buf->page_offset); 5360 if (buf->sec_page) 5361 prefetch(page_address(buf->sec_page)); 5362 5363 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5364 len += buf1_len; 5365 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5366 len += buf2_len; 5367 5368 /* ACS is disabled; strip manually. */ 5369 if (likely(!(status & rx_not_ls))) { 5370 if (buf2_len) { 5371 buf2_len -= ETH_FCS_LEN; 5372 len -= ETH_FCS_LEN; 5373 } else if (buf1_len) { 5374 buf1_len -= ETH_FCS_LEN; 5375 len -= ETH_FCS_LEN; 5376 } 5377 } 5378 5379 if (!skb) { 5380 unsigned int pre_len, sync_len; 5381 5382 dma_sync_single_for_cpu(priv->device, buf->addr, 5383 buf1_len, dma_dir); 5384 5385 xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq); 5386 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5387 buf->page_offset, buf1_len, true); 5388 5389 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5390 buf->page_offset; 5391 5392 ctx.priv = priv; 5393 ctx.desc = p; 5394 ctx.ndesc = np; 5395 5396 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5397 /* Due xdp_adjust_tail: DMA sync for_device 5398 * cover max len CPU touch 5399 */ 5400 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5401 buf->page_offset; 5402 sync_len = max(sync_len, pre_len); 5403 5404 /* For Not XDP_PASS verdict */ 5405 if (IS_ERR(skb)) { 5406 unsigned int xdp_res = -PTR_ERR(skb); 5407 5408 if (xdp_res & STMMAC_XDP_CONSUMED) { 5409 page_pool_put_page(rx_q->page_pool, 5410 virt_to_head_page(ctx.xdp.data), 5411 sync_len, true); 5412 buf->page = NULL; 5413 rx_dropped++; 5414 5415 /* Clear skb as it was set as 5416 * status by XDP program. 5417 */ 5418 skb = NULL; 5419 5420 if (unlikely((status & rx_not_ls))) 5421 goto read_again; 5422 5423 count++; 5424 continue; 5425 } else if (xdp_res & (STMMAC_XDP_TX | 5426 STMMAC_XDP_REDIRECT)) { 5427 xdp_status |= xdp_res; 5428 buf->page = NULL; 5429 skb = NULL; 5430 count++; 5431 continue; 5432 } 5433 } 5434 } 5435 5436 if (!skb) { 5437 /* XDP program may expand or reduce tail */ 5438 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5439 5440 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); 5441 if (!skb) { 5442 rx_dropped++; 5443 count++; 5444 goto drain_data; 5445 } 5446 5447 /* XDP program may adjust header */ 5448 skb_copy_to_linear_data(skb, ctx.xdp.data, buf1_len); 5449 skb_put(skb, buf1_len); 5450 5451 /* Data payload copied into SKB, page ready for recycle */ 5452 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5453 buf->page = NULL; 5454 } else if (buf1_len) { 5455 dma_sync_single_for_cpu(priv->device, buf->addr, 5456 buf1_len, dma_dir); 5457 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5458 buf->page, buf->page_offset, buf1_len, 5459 priv->dma_conf.dma_buf_sz); 5460 5461 /* Data payload appended into SKB */ 5462 skb_mark_for_recycle(skb); 5463 buf->page = NULL; 5464 } 5465 5466 if (buf2_len) { 5467 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5468 buf2_len, dma_dir); 5469 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5470 buf->sec_page, 0, buf2_len, 5471 priv->dma_conf.dma_buf_sz); 5472 5473 /* Data payload appended into SKB */ 5474 skb_mark_for_recycle(skb); 5475 buf->sec_page = NULL; 5476 } 5477 5478 drain_data: 5479 if (likely(status & rx_not_ls)) 5480 goto read_again; 5481 if (!skb) 5482 continue; 5483 5484 /* Got entire packet into SKB. Finish it. */ 5485 5486 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5487 stmmac_rx_vlan(priv->dev, skb); 5488 skb->protocol = eth_type_trans(skb, priv->dev); 5489 5490 if (unlikely(!coe)) 5491 skb_checksum_none_assert(skb); 5492 else 5493 skb->ip_summed = CHECKSUM_UNNECESSARY; 5494 5495 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5496 skb_set_hash(skb, hash, hash_type); 5497 5498 skb_record_rx_queue(skb, queue); 5499 napi_gro_receive(&ch->rx_napi, skb); 5500 skb = NULL; 5501 5502 rx_packets++; 5503 rx_bytes += len; 5504 count++; 5505 } 5506 5507 if (status & rx_not_ls || skb) { 5508 rx_q->state_saved = true; 5509 rx_q->state.skb = skb; 5510 rx_q->state.error = error; 5511 rx_q->state.len = len; 5512 } 5513 5514 stmmac_finalize_xdp_rx(priv, xdp_status); 5515 5516 stmmac_rx_refill(priv, queue); 5517 5518 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5519 rxq_stats->rx_packets += rx_packets; 5520 rxq_stats->rx_bytes += rx_bytes; 5521 rxq_stats->rx_pkt_n += count; 5522 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5523 5524 priv->xstats.rx_dropped += rx_dropped; 5525 priv->xstats.rx_errors += rx_errors; 5526 5527 return count; 5528 } 5529 5530 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5531 { 5532 struct stmmac_channel *ch = 5533 container_of(napi, struct stmmac_channel, rx_napi); 5534 struct stmmac_priv *priv = ch->priv_data; 5535 struct stmmac_rxq_stats *rxq_stats; 5536 u32 chan = ch->index; 5537 unsigned long flags; 5538 int work_done; 5539 5540 rxq_stats = &priv->xstats.rxq_stats[chan]; 5541 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5542 rxq_stats->napi_poll++; 5543 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5544 5545 work_done = stmmac_rx(priv, budget, chan); 5546 if (work_done < budget && napi_complete_done(napi, work_done)) { 5547 unsigned long flags; 5548 5549 spin_lock_irqsave(&ch->lock, flags); 5550 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5551 spin_unlock_irqrestore(&ch->lock, flags); 5552 } 5553 5554 return work_done; 5555 } 5556 5557 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5558 { 5559 struct stmmac_channel *ch = 5560 container_of(napi, struct stmmac_channel, tx_napi); 5561 struct stmmac_priv *priv = ch->priv_data; 5562 struct stmmac_txq_stats *txq_stats; 5563 u32 chan = ch->index; 5564 unsigned long flags; 5565 int work_done; 5566 5567 txq_stats = &priv->xstats.txq_stats[chan]; 5568 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 5569 txq_stats->napi_poll++; 5570 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 5571 5572 work_done = stmmac_tx_clean(priv, budget, chan); 5573 work_done = min(work_done, budget); 5574 5575 if (work_done < budget && napi_complete_done(napi, work_done)) { 5576 unsigned long flags; 5577 5578 spin_lock_irqsave(&ch->lock, flags); 5579 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5580 spin_unlock_irqrestore(&ch->lock, flags); 5581 } 5582 5583 return work_done; 5584 } 5585 5586 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5587 { 5588 struct stmmac_channel *ch = 5589 container_of(napi, struct stmmac_channel, rxtx_napi); 5590 struct stmmac_priv *priv = ch->priv_data; 5591 int rx_done, tx_done, rxtx_done; 5592 struct stmmac_rxq_stats *rxq_stats; 5593 struct stmmac_txq_stats *txq_stats; 5594 u32 chan = ch->index; 5595 unsigned long flags; 5596 5597 rxq_stats = &priv->xstats.rxq_stats[chan]; 5598 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5599 rxq_stats->napi_poll++; 5600 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5601 5602 txq_stats = &priv->xstats.txq_stats[chan]; 5603 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 5604 txq_stats->napi_poll++; 5605 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 5606 5607 tx_done = stmmac_tx_clean(priv, budget, chan); 5608 tx_done = min(tx_done, budget); 5609 5610 rx_done = stmmac_rx_zc(priv, budget, chan); 5611 5612 rxtx_done = max(tx_done, rx_done); 5613 5614 /* If either TX or RX work is not complete, return budget 5615 * and keep pooling 5616 */ 5617 if (rxtx_done >= budget) 5618 return budget; 5619 5620 /* all work done, exit the polling mode */ 5621 if (napi_complete_done(napi, rxtx_done)) { 5622 unsigned long flags; 5623 5624 spin_lock_irqsave(&ch->lock, flags); 5625 /* Both RX and TX work done are compelte, 5626 * so enable both RX & TX IRQs. 5627 */ 5628 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5629 spin_unlock_irqrestore(&ch->lock, flags); 5630 } 5631 5632 return min(rxtx_done, budget - 1); 5633 } 5634 5635 /** 5636 * stmmac_tx_timeout 5637 * @dev : Pointer to net device structure 5638 * @txqueue: the index of the hanging transmit queue 5639 * Description: this function is called when a packet transmission fails to 5640 * complete within a reasonable time. The driver will mark the error in the 5641 * netdev structure and arrange for the device to be reset to a sane state 5642 * in order to transmit a new packet. 5643 */ 5644 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5645 { 5646 struct stmmac_priv *priv = netdev_priv(dev); 5647 5648 stmmac_global_err(priv); 5649 } 5650 5651 /** 5652 * stmmac_set_rx_mode - entry point for multicast addressing 5653 * @dev : pointer to the device structure 5654 * Description: 5655 * This function is a driver entry point which gets called by the kernel 5656 * whenever multicast addresses must be enabled/disabled. 5657 * Return value: 5658 * void. 5659 */ 5660 static void stmmac_set_rx_mode(struct net_device *dev) 5661 { 5662 struct stmmac_priv *priv = netdev_priv(dev); 5663 5664 stmmac_set_filter(priv, priv->hw, dev); 5665 } 5666 5667 /** 5668 * stmmac_change_mtu - entry point to change MTU size for the device. 5669 * @dev : device pointer. 5670 * @new_mtu : the new MTU size for the device. 5671 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5672 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5673 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5674 * Return value: 5675 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5676 * file on failure. 5677 */ 5678 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5679 { 5680 struct stmmac_priv *priv = netdev_priv(dev); 5681 int txfifosz = priv->plat->tx_fifo_size; 5682 struct stmmac_dma_conf *dma_conf; 5683 const int mtu = new_mtu; 5684 int ret; 5685 5686 if (txfifosz == 0) 5687 txfifosz = priv->dma_cap.tx_fifo_size; 5688 5689 txfifosz /= priv->plat->tx_queues_to_use; 5690 5691 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5692 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 5693 return -EINVAL; 5694 } 5695 5696 new_mtu = STMMAC_ALIGN(new_mtu); 5697 5698 /* If condition true, FIFO is too small or MTU too large */ 5699 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 5700 return -EINVAL; 5701 5702 if (netif_running(dev)) { 5703 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 5704 /* Try to allocate the new DMA conf with the new mtu */ 5705 dma_conf = stmmac_setup_dma_desc(priv, mtu); 5706 if (IS_ERR(dma_conf)) { 5707 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 5708 mtu); 5709 return PTR_ERR(dma_conf); 5710 } 5711 5712 stmmac_release(dev); 5713 5714 ret = __stmmac_open(dev, dma_conf); 5715 if (ret) { 5716 free_dma_desc_resources(priv, dma_conf); 5717 kfree(dma_conf); 5718 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 5719 return ret; 5720 } 5721 5722 kfree(dma_conf); 5723 5724 stmmac_set_rx_mode(dev); 5725 } 5726 5727 dev->mtu = mtu; 5728 netdev_update_features(dev); 5729 5730 return 0; 5731 } 5732 5733 static netdev_features_t stmmac_fix_features(struct net_device *dev, 5734 netdev_features_t features) 5735 { 5736 struct stmmac_priv *priv = netdev_priv(dev); 5737 5738 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 5739 features &= ~NETIF_F_RXCSUM; 5740 5741 if (!priv->plat->tx_coe) 5742 features &= ~NETIF_F_CSUM_MASK; 5743 5744 /* Some GMAC devices have a bugged Jumbo frame support that 5745 * needs to have the Tx COE disabled for oversized frames 5746 * (due to limited buffer sizes). In this case we disable 5747 * the TX csum insertion in the TDES and not use SF. 5748 */ 5749 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 5750 features &= ~NETIF_F_CSUM_MASK; 5751 5752 /* Disable tso if asked by ethtool */ 5753 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 5754 if (features & NETIF_F_TSO) 5755 priv->tso = true; 5756 else 5757 priv->tso = false; 5758 } 5759 5760 return features; 5761 } 5762 5763 static int stmmac_set_features(struct net_device *netdev, 5764 netdev_features_t features) 5765 { 5766 struct stmmac_priv *priv = netdev_priv(netdev); 5767 5768 /* Keep the COE Type in case of csum is supporting */ 5769 if (features & NETIF_F_RXCSUM) 5770 priv->hw->rx_csum = priv->plat->rx_coe; 5771 else 5772 priv->hw->rx_csum = 0; 5773 /* No check needed because rx_coe has been set before and it will be 5774 * fixed in case of issue. 5775 */ 5776 stmmac_rx_ipc(priv, priv->hw); 5777 5778 if (priv->sph_cap) { 5779 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; 5780 u32 chan; 5781 5782 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 5783 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 5784 } 5785 5786 return 0; 5787 } 5788 5789 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) 5790 { 5791 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 5792 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 5793 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 5794 bool *hs_enable = &fpe_cfg->hs_enable; 5795 5796 if (status == FPE_EVENT_UNKNOWN || !*hs_enable) 5797 return; 5798 5799 /* If LP has sent verify mPacket, LP is FPE capable */ 5800 if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) { 5801 if (*lp_state < FPE_STATE_CAPABLE) 5802 *lp_state = FPE_STATE_CAPABLE; 5803 5804 /* If user has requested FPE enable, quickly response */ 5805 if (*hs_enable) 5806 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 5807 MPACKET_RESPONSE); 5808 } 5809 5810 /* If Local has sent verify mPacket, Local is FPE capable */ 5811 if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) { 5812 if (*lo_state < FPE_STATE_CAPABLE) 5813 *lo_state = FPE_STATE_CAPABLE; 5814 } 5815 5816 /* If LP has sent response mPacket, LP is entering FPE ON */ 5817 if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP) 5818 *lp_state = FPE_STATE_ENTERING_ON; 5819 5820 /* If Local has sent response mPacket, Local is entering FPE ON */ 5821 if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP) 5822 *lo_state = FPE_STATE_ENTERING_ON; 5823 5824 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && 5825 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && 5826 priv->fpe_wq) { 5827 queue_work(priv->fpe_wq, &priv->fpe_task); 5828 } 5829 } 5830 5831 static void stmmac_common_interrupt(struct stmmac_priv *priv) 5832 { 5833 u32 rx_cnt = priv->plat->rx_queues_to_use; 5834 u32 tx_cnt = priv->plat->tx_queues_to_use; 5835 u32 queues_count; 5836 u32 queue; 5837 bool xmac; 5838 5839 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 5840 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 5841 5842 if (priv->irq_wake) 5843 pm_wakeup_event(priv->device, 0); 5844 5845 if (priv->dma_cap.estsel) 5846 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, 5847 &priv->xstats, tx_cnt); 5848 5849 if (priv->dma_cap.fpesel) { 5850 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, 5851 priv->dev); 5852 5853 stmmac_fpe_event_status(priv, status); 5854 } 5855 5856 /* To handle GMAC own interrupts */ 5857 if ((priv->plat->has_gmac) || xmac) { 5858 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 5859 5860 if (unlikely(status)) { 5861 /* For LPI we need to save the tx status */ 5862 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 5863 priv->tx_path_in_lpi_mode = true; 5864 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 5865 priv->tx_path_in_lpi_mode = false; 5866 } 5867 5868 for (queue = 0; queue < queues_count; queue++) { 5869 status = stmmac_host_mtl_irq_status(priv, priv->hw, 5870 queue); 5871 } 5872 5873 /* PCS link status */ 5874 if (priv->hw->pcs && 5875 !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { 5876 if (priv->xstats.pcs_link) 5877 netif_carrier_on(priv->dev); 5878 else 5879 netif_carrier_off(priv->dev); 5880 } 5881 5882 stmmac_timestamp_interrupt(priv, priv); 5883 } 5884 } 5885 5886 /** 5887 * stmmac_interrupt - main ISR 5888 * @irq: interrupt number. 5889 * @dev_id: to pass the net device pointer. 5890 * Description: this is the main driver interrupt service routine. 5891 * It can call: 5892 * o DMA service routine (to manage incoming frame reception and transmission 5893 * status) 5894 * o Core interrupts to manage: remote wake-up, management counter, LPI 5895 * interrupts. 5896 */ 5897 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 5898 { 5899 struct net_device *dev = (struct net_device *)dev_id; 5900 struct stmmac_priv *priv = netdev_priv(dev); 5901 5902 /* Check if adapter is up */ 5903 if (test_bit(STMMAC_DOWN, &priv->state)) 5904 return IRQ_HANDLED; 5905 5906 /* Check if a fatal error happened */ 5907 if (stmmac_safety_feat_interrupt(priv)) 5908 return IRQ_HANDLED; 5909 5910 /* To handle Common interrupts */ 5911 stmmac_common_interrupt(priv); 5912 5913 /* To handle DMA interrupts */ 5914 stmmac_dma_interrupt(priv); 5915 5916 return IRQ_HANDLED; 5917 } 5918 5919 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 5920 { 5921 struct net_device *dev = (struct net_device *)dev_id; 5922 struct stmmac_priv *priv = netdev_priv(dev); 5923 5924 if (unlikely(!dev)) { 5925 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5926 return IRQ_NONE; 5927 } 5928 5929 /* Check if adapter is up */ 5930 if (test_bit(STMMAC_DOWN, &priv->state)) 5931 return IRQ_HANDLED; 5932 5933 /* To handle Common interrupts */ 5934 stmmac_common_interrupt(priv); 5935 5936 return IRQ_HANDLED; 5937 } 5938 5939 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 5940 { 5941 struct net_device *dev = (struct net_device *)dev_id; 5942 struct stmmac_priv *priv = netdev_priv(dev); 5943 5944 if (unlikely(!dev)) { 5945 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5946 return IRQ_NONE; 5947 } 5948 5949 /* Check if adapter is up */ 5950 if (test_bit(STMMAC_DOWN, &priv->state)) 5951 return IRQ_HANDLED; 5952 5953 /* Check if a fatal error happened */ 5954 stmmac_safety_feat_interrupt(priv); 5955 5956 return IRQ_HANDLED; 5957 } 5958 5959 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 5960 { 5961 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 5962 struct stmmac_dma_conf *dma_conf; 5963 int chan = tx_q->queue_index; 5964 struct stmmac_priv *priv; 5965 int status; 5966 5967 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 5968 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 5969 5970 if (unlikely(!data)) { 5971 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5972 return IRQ_NONE; 5973 } 5974 5975 /* Check if adapter is up */ 5976 if (test_bit(STMMAC_DOWN, &priv->state)) 5977 return IRQ_HANDLED; 5978 5979 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 5980 5981 if (unlikely(status & tx_hard_error_bump_tc)) { 5982 /* Try to bump up the dma threshold on this failure */ 5983 stmmac_bump_dma_threshold(priv, chan); 5984 } else if (unlikely(status == tx_hard_error)) { 5985 stmmac_tx_err(priv, chan); 5986 } 5987 5988 return IRQ_HANDLED; 5989 } 5990 5991 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 5992 { 5993 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 5994 struct stmmac_dma_conf *dma_conf; 5995 int chan = rx_q->queue_index; 5996 struct stmmac_priv *priv; 5997 5998 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 5999 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6000 6001 if (unlikely(!data)) { 6002 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 6003 return IRQ_NONE; 6004 } 6005 6006 /* Check if adapter is up */ 6007 if (test_bit(STMMAC_DOWN, &priv->state)) 6008 return IRQ_HANDLED; 6009 6010 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6011 6012 return IRQ_HANDLED; 6013 } 6014 6015 /** 6016 * stmmac_ioctl - Entry point for the Ioctl 6017 * @dev: Device pointer. 6018 * @rq: An IOCTL specefic structure, that can contain a pointer to 6019 * a proprietary structure used to pass information to the driver. 6020 * @cmd: IOCTL command 6021 * Description: 6022 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6023 */ 6024 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6025 { 6026 struct stmmac_priv *priv = netdev_priv (dev); 6027 int ret = -EOPNOTSUPP; 6028 6029 if (!netif_running(dev)) 6030 return -EINVAL; 6031 6032 switch (cmd) { 6033 case SIOCGMIIPHY: 6034 case SIOCGMIIREG: 6035 case SIOCSMIIREG: 6036 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6037 break; 6038 case SIOCSHWTSTAMP: 6039 ret = stmmac_hwtstamp_set(dev, rq); 6040 break; 6041 case SIOCGHWTSTAMP: 6042 ret = stmmac_hwtstamp_get(dev, rq); 6043 break; 6044 default: 6045 break; 6046 } 6047 6048 return ret; 6049 } 6050 6051 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6052 void *cb_priv) 6053 { 6054 struct stmmac_priv *priv = cb_priv; 6055 int ret = -EOPNOTSUPP; 6056 6057 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6058 return ret; 6059 6060 __stmmac_disable_all_queues(priv); 6061 6062 switch (type) { 6063 case TC_SETUP_CLSU32: 6064 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6065 break; 6066 case TC_SETUP_CLSFLOWER: 6067 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6068 break; 6069 default: 6070 break; 6071 } 6072 6073 stmmac_enable_all_queues(priv); 6074 return ret; 6075 } 6076 6077 static LIST_HEAD(stmmac_block_cb_list); 6078 6079 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6080 void *type_data) 6081 { 6082 struct stmmac_priv *priv = netdev_priv(ndev); 6083 6084 switch (type) { 6085 case TC_QUERY_CAPS: 6086 return stmmac_tc_query_caps(priv, priv, type_data); 6087 case TC_SETUP_BLOCK: 6088 return flow_block_cb_setup_simple(type_data, 6089 &stmmac_block_cb_list, 6090 stmmac_setup_tc_block_cb, 6091 priv, priv, true); 6092 case TC_SETUP_QDISC_CBS: 6093 return stmmac_tc_setup_cbs(priv, priv, type_data); 6094 case TC_SETUP_QDISC_TAPRIO: 6095 return stmmac_tc_setup_taprio(priv, priv, type_data); 6096 case TC_SETUP_QDISC_ETF: 6097 return stmmac_tc_setup_etf(priv, priv, type_data); 6098 default: 6099 return -EOPNOTSUPP; 6100 } 6101 } 6102 6103 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6104 struct net_device *sb_dev) 6105 { 6106 int gso = skb_shinfo(skb)->gso_type; 6107 6108 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6109 /* 6110 * There is no way to determine the number of TSO/USO 6111 * capable Queues. Let's use always the Queue 0 6112 * because if TSO/USO is supported then at least this 6113 * one will be capable. 6114 */ 6115 return 0; 6116 } 6117 6118 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6119 } 6120 6121 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6122 { 6123 struct stmmac_priv *priv = netdev_priv(ndev); 6124 int ret = 0; 6125 6126 ret = pm_runtime_resume_and_get(priv->device); 6127 if (ret < 0) 6128 return ret; 6129 6130 ret = eth_mac_addr(ndev, addr); 6131 if (ret) 6132 goto set_mac_error; 6133 6134 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6135 6136 set_mac_error: 6137 pm_runtime_put(priv->device); 6138 6139 return ret; 6140 } 6141 6142 #ifdef CONFIG_DEBUG_FS 6143 static struct dentry *stmmac_fs_dir; 6144 6145 static void sysfs_display_ring(void *head, int size, int extend_desc, 6146 struct seq_file *seq, dma_addr_t dma_phy_addr) 6147 { 6148 int i; 6149 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6150 struct dma_desc *p = (struct dma_desc *)head; 6151 dma_addr_t dma_addr; 6152 6153 for (i = 0; i < size; i++) { 6154 if (extend_desc) { 6155 dma_addr = dma_phy_addr + i * sizeof(*ep); 6156 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6157 i, &dma_addr, 6158 le32_to_cpu(ep->basic.des0), 6159 le32_to_cpu(ep->basic.des1), 6160 le32_to_cpu(ep->basic.des2), 6161 le32_to_cpu(ep->basic.des3)); 6162 ep++; 6163 } else { 6164 dma_addr = dma_phy_addr + i * sizeof(*p); 6165 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6166 i, &dma_addr, 6167 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6168 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6169 p++; 6170 } 6171 seq_printf(seq, "\n"); 6172 } 6173 } 6174 6175 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6176 { 6177 struct net_device *dev = seq->private; 6178 struct stmmac_priv *priv = netdev_priv(dev); 6179 u32 rx_count = priv->plat->rx_queues_to_use; 6180 u32 tx_count = priv->plat->tx_queues_to_use; 6181 u32 queue; 6182 6183 if ((dev->flags & IFF_UP) == 0) 6184 return 0; 6185 6186 for (queue = 0; queue < rx_count; queue++) { 6187 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6188 6189 seq_printf(seq, "RX Queue %d:\n", queue); 6190 6191 if (priv->extend_desc) { 6192 seq_printf(seq, "Extended descriptor ring:\n"); 6193 sysfs_display_ring((void *)rx_q->dma_erx, 6194 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6195 } else { 6196 seq_printf(seq, "Descriptor ring:\n"); 6197 sysfs_display_ring((void *)rx_q->dma_rx, 6198 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6199 } 6200 } 6201 6202 for (queue = 0; queue < tx_count; queue++) { 6203 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6204 6205 seq_printf(seq, "TX Queue %d:\n", queue); 6206 6207 if (priv->extend_desc) { 6208 seq_printf(seq, "Extended descriptor ring:\n"); 6209 sysfs_display_ring((void *)tx_q->dma_etx, 6210 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6211 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6212 seq_printf(seq, "Descriptor ring:\n"); 6213 sysfs_display_ring((void *)tx_q->dma_tx, 6214 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6215 } 6216 } 6217 6218 return 0; 6219 } 6220 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6221 6222 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6223 { 6224 static const char * const dwxgmac_timestamp_source[] = { 6225 "None", 6226 "Internal", 6227 "External", 6228 "Both", 6229 }; 6230 static const char * const dwxgmac_safety_feature_desc[] = { 6231 "No", 6232 "All Safety Features with ECC and Parity", 6233 "All Safety Features without ECC or Parity", 6234 "All Safety Features with Parity Only", 6235 "ECC Only", 6236 "UNDEFINED", 6237 "UNDEFINED", 6238 "UNDEFINED", 6239 }; 6240 struct net_device *dev = seq->private; 6241 struct stmmac_priv *priv = netdev_priv(dev); 6242 6243 if (!priv->hw_cap_support) { 6244 seq_printf(seq, "DMA HW features not supported\n"); 6245 return 0; 6246 } 6247 6248 seq_printf(seq, "==============================\n"); 6249 seq_printf(seq, "\tDMA HW features\n"); 6250 seq_printf(seq, "==============================\n"); 6251 6252 seq_printf(seq, "\t10/100 Mbps: %s\n", 6253 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6254 seq_printf(seq, "\t1000 Mbps: %s\n", 6255 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6256 seq_printf(seq, "\tHalf duplex: %s\n", 6257 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6258 if (priv->plat->has_xgmac) { 6259 seq_printf(seq, 6260 "\tNumber of Additional MAC address registers: %d\n", 6261 priv->dma_cap.multi_addr); 6262 } else { 6263 seq_printf(seq, "\tHash Filter: %s\n", 6264 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6265 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6266 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6267 } 6268 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6269 (priv->dma_cap.pcs) ? "Y" : "N"); 6270 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6271 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6272 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6273 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6274 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6275 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6276 seq_printf(seq, "\tRMON module: %s\n", 6277 (priv->dma_cap.rmon) ? "Y" : "N"); 6278 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6279 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6280 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6281 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6282 if (priv->plat->has_xgmac) 6283 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6284 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6285 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6286 (priv->dma_cap.eee) ? "Y" : "N"); 6287 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6288 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6289 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6290 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6291 priv->plat->has_xgmac) { 6292 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6293 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6294 } else { 6295 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6296 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6297 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6298 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6299 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6300 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6301 } 6302 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6303 priv->dma_cap.number_rx_channel); 6304 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6305 priv->dma_cap.number_tx_channel); 6306 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6307 priv->dma_cap.number_rx_queues); 6308 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6309 priv->dma_cap.number_tx_queues); 6310 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6311 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6312 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6313 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6314 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6315 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6316 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6317 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6318 priv->dma_cap.pps_out_num); 6319 seq_printf(seq, "\tSafety Features: %s\n", 6320 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6321 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6322 priv->dma_cap.frpsel ? "Y" : "N"); 6323 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6324 priv->dma_cap.host_dma_width); 6325 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6326 priv->dma_cap.rssen ? "Y" : "N"); 6327 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6328 priv->dma_cap.vlhash ? "Y" : "N"); 6329 seq_printf(seq, "\tSplit Header: %s\n", 6330 priv->dma_cap.sphen ? "Y" : "N"); 6331 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6332 priv->dma_cap.vlins ? "Y" : "N"); 6333 seq_printf(seq, "\tDouble VLAN: %s\n", 6334 priv->dma_cap.dvlan ? "Y" : "N"); 6335 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6336 priv->dma_cap.l3l4fnum); 6337 seq_printf(seq, "\tARP Offloading: %s\n", 6338 priv->dma_cap.arpoffsel ? "Y" : "N"); 6339 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6340 priv->dma_cap.estsel ? "Y" : "N"); 6341 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6342 priv->dma_cap.fpesel ? "Y" : "N"); 6343 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6344 priv->dma_cap.tbssel ? "Y" : "N"); 6345 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6346 priv->dma_cap.tbs_ch_num); 6347 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6348 priv->dma_cap.sgfsel ? "Y" : "N"); 6349 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6350 BIT(priv->dma_cap.ttsfd) >> 1); 6351 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6352 priv->dma_cap.numtc); 6353 seq_printf(seq, "\tDCB Feature: %s\n", 6354 priv->dma_cap.dcben ? "Y" : "N"); 6355 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6356 priv->dma_cap.advthword ? "Y" : "N"); 6357 seq_printf(seq, "\tPTP Offload: %s\n", 6358 priv->dma_cap.ptoen ? "Y" : "N"); 6359 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6360 priv->dma_cap.osten ? "Y" : "N"); 6361 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6362 priv->dma_cap.pfcen ? "Y" : "N"); 6363 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6364 BIT(priv->dma_cap.frpes) << 6); 6365 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6366 BIT(priv->dma_cap.frpbs) << 6); 6367 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6368 priv->dma_cap.frppipe_num); 6369 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6370 priv->dma_cap.nrvf_num ? 6371 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6372 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6373 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6374 seq_printf(seq, "\tDepth of GCL: %lu\n", 6375 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6376 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6377 priv->dma_cap.cbtisel ? "Y" : "N"); 6378 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6379 priv->dma_cap.aux_snapshot_n); 6380 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6381 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6382 seq_printf(seq, "\tEnhanced DMA: %s\n", 6383 priv->dma_cap.edma ? "Y" : "N"); 6384 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6385 priv->dma_cap.ediffc ? "Y" : "N"); 6386 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6387 priv->dma_cap.vxn ? "Y" : "N"); 6388 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6389 priv->dma_cap.dbgmem ? "Y" : "N"); 6390 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6391 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6392 return 0; 6393 } 6394 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6395 6396 /* Use network device events to rename debugfs file entries. 6397 */ 6398 static int stmmac_device_event(struct notifier_block *unused, 6399 unsigned long event, void *ptr) 6400 { 6401 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6402 struct stmmac_priv *priv = netdev_priv(dev); 6403 6404 if (dev->netdev_ops != &stmmac_netdev_ops) 6405 goto done; 6406 6407 switch (event) { 6408 case NETDEV_CHANGENAME: 6409 if (priv->dbgfs_dir) 6410 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, 6411 priv->dbgfs_dir, 6412 stmmac_fs_dir, 6413 dev->name); 6414 break; 6415 } 6416 done: 6417 return NOTIFY_DONE; 6418 } 6419 6420 static struct notifier_block stmmac_notifier = { 6421 .notifier_call = stmmac_device_event, 6422 }; 6423 6424 static void stmmac_init_fs(struct net_device *dev) 6425 { 6426 struct stmmac_priv *priv = netdev_priv(dev); 6427 6428 rtnl_lock(); 6429 6430 /* Create per netdev entries */ 6431 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6432 6433 /* Entry to report DMA RX/TX rings */ 6434 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6435 &stmmac_rings_status_fops); 6436 6437 /* Entry to report the DMA HW features */ 6438 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6439 &stmmac_dma_cap_fops); 6440 6441 rtnl_unlock(); 6442 } 6443 6444 static void stmmac_exit_fs(struct net_device *dev) 6445 { 6446 struct stmmac_priv *priv = netdev_priv(dev); 6447 6448 debugfs_remove_recursive(priv->dbgfs_dir); 6449 } 6450 #endif /* CONFIG_DEBUG_FS */ 6451 6452 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6453 { 6454 unsigned char *data = (unsigned char *)&vid_le; 6455 unsigned char data_byte = 0; 6456 u32 crc = ~0x0; 6457 u32 temp = 0; 6458 int i, bits; 6459 6460 bits = get_bitmask_order(VLAN_VID_MASK); 6461 for (i = 0; i < bits; i++) { 6462 if ((i % 8) == 0) 6463 data_byte = data[i / 8]; 6464 6465 temp = ((crc & 1) ^ data_byte) & 1; 6466 crc >>= 1; 6467 data_byte >>= 1; 6468 6469 if (temp) 6470 crc ^= 0xedb88320; 6471 } 6472 6473 return crc; 6474 } 6475 6476 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6477 { 6478 u32 crc, hash = 0; 6479 __le16 pmatch = 0; 6480 int count = 0; 6481 u16 vid = 0; 6482 6483 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6484 __le16 vid_le = cpu_to_le16(vid); 6485 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6486 hash |= (1 << crc); 6487 count++; 6488 } 6489 6490 if (!priv->dma_cap.vlhash) { 6491 if (count > 2) /* VID = 0 always passes filter */ 6492 return -EOPNOTSUPP; 6493 6494 pmatch = cpu_to_le16(vid); 6495 hash = 0; 6496 } 6497 6498 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6499 } 6500 6501 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6502 { 6503 struct stmmac_priv *priv = netdev_priv(ndev); 6504 bool is_double = false; 6505 int ret; 6506 6507 ret = pm_runtime_resume_and_get(priv->device); 6508 if (ret < 0) 6509 return ret; 6510 6511 if (be16_to_cpu(proto) == ETH_P_8021AD) 6512 is_double = true; 6513 6514 set_bit(vid, priv->active_vlans); 6515 ret = stmmac_vlan_update(priv, is_double); 6516 if (ret) { 6517 clear_bit(vid, priv->active_vlans); 6518 goto err_pm_put; 6519 } 6520 6521 if (priv->hw->num_vlan) { 6522 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6523 if (ret) 6524 goto err_pm_put; 6525 } 6526 err_pm_put: 6527 pm_runtime_put(priv->device); 6528 6529 return ret; 6530 } 6531 6532 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6533 { 6534 struct stmmac_priv *priv = netdev_priv(ndev); 6535 bool is_double = false; 6536 int ret; 6537 6538 ret = pm_runtime_resume_and_get(priv->device); 6539 if (ret < 0) 6540 return ret; 6541 6542 if (be16_to_cpu(proto) == ETH_P_8021AD) 6543 is_double = true; 6544 6545 clear_bit(vid, priv->active_vlans); 6546 6547 if (priv->hw->num_vlan) { 6548 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6549 if (ret) 6550 goto del_vlan_error; 6551 } 6552 6553 ret = stmmac_vlan_update(priv, is_double); 6554 6555 del_vlan_error: 6556 pm_runtime_put(priv->device); 6557 6558 return ret; 6559 } 6560 6561 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6562 { 6563 struct stmmac_priv *priv = netdev_priv(dev); 6564 6565 switch (bpf->command) { 6566 case XDP_SETUP_PROG: 6567 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6568 case XDP_SETUP_XSK_POOL: 6569 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6570 bpf->xsk.queue_id); 6571 default: 6572 return -EOPNOTSUPP; 6573 } 6574 } 6575 6576 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6577 struct xdp_frame **frames, u32 flags) 6578 { 6579 struct stmmac_priv *priv = netdev_priv(dev); 6580 int cpu = smp_processor_id(); 6581 struct netdev_queue *nq; 6582 int i, nxmit = 0; 6583 int queue; 6584 6585 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6586 return -ENETDOWN; 6587 6588 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6589 return -EINVAL; 6590 6591 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6592 nq = netdev_get_tx_queue(priv->dev, queue); 6593 6594 __netif_tx_lock(nq, cpu); 6595 /* Avoids TX time-out as we are sharing with slow path */ 6596 txq_trans_cond_update(nq); 6597 6598 for (i = 0; i < num_frames; i++) { 6599 int res; 6600 6601 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6602 if (res == STMMAC_XDP_CONSUMED) 6603 break; 6604 6605 nxmit++; 6606 } 6607 6608 if (flags & XDP_XMIT_FLUSH) { 6609 stmmac_flush_tx_descriptors(priv, queue); 6610 stmmac_tx_timer_arm(priv, queue); 6611 } 6612 6613 __netif_tx_unlock(nq); 6614 6615 return nxmit; 6616 } 6617 6618 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6619 { 6620 struct stmmac_channel *ch = &priv->channel[queue]; 6621 unsigned long flags; 6622 6623 spin_lock_irqsave(&ch->lock, flags); 6624 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6625 spin_unlock_irqrestore(&ch->lock, flags); 6626 6627 stmmac_stop_rx_dma(priv, queue); 6628 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6629 } 6630 6631 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6632 { 6633 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6634 struct stmmac_channel *ch = &priv->channel[queue]; 6635 unsigned long flags; 6636 u32 buf_size; 6637 int ret; 6638 6639 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6640 if (ret) { 6641 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6642 return; 6643 } 6644 6645 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 6646 if (ret) { 6647 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6648 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6649 return; 6650 } 6651 6652 stmmac_reset_rx_queue(priv, queue); 6653 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 6654 6655 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6656 rx_q->dma_rx_phy, rx_q->queue_index); 6657 6658 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6659 sizeof(struct dma_desc)); 6660 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6661 rx_q->rx_tail_addr, rx_q->queue_index); 6662 6663 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6664 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6665 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6666 buf_size, 6667 rx_q->queue_index); 6668 } else { 6669 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6670 priv->dma_conf.dma_buf_sz, 6671 rx_q->queue_index); 6672 } 6673 6674 stmmac_start_rx_dma(priv, queue); 6675 6676 spin_lock_irqsave(&ch->lock, flags); 6677 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6678 spin_unlock_irqrestore(&ch->lock, flags); 6679 } 6680 6681 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6682 { 6683 struct stmmac_channel *ch = &priv->channel[queue]; 6684 unsigned long flags; 6685 6686 spin_lock_irqsave(&ch->lock, flags); 6687 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6688 spin_unlock_irqrestore(&ch->lock, flags); 6689 6690 stmmac_stop_tx_dma(priv, queue); 6691 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6692 } 6693 6694 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6695 { 6696 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6697 struct stmmac_channel *ch = &priv->channel[queue]; 6698 unsigned long flags; 6699 int ret; 6700 6701 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6702 if (ret) { 6703 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6704 return; 6705 } 6706 6707 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 6708 if (ret) { 6709 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6710 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6711 return; 6712 } 6713 6714 stmmac_reset_tx_queue(priv, queue); 6715 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 6716 6717 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6718 tx_q->dma_tx_phy, tx_q->queue_index); 6719 6720 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6721 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6722 6723 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6724 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6725 tx_q->tx_tail_addr, tx_q->queue_index); 6726 6727 stmmac_start_tx_dma(priv, queue); 6728 6729 spin_lock_irqsave(&ch->lock, flags); 6730 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6731 spin_unlock_irqrestore(&ch->lock, flags); 6732 } 6733 6734 void stmmac_xdp_release(struct net_device *dev) 6735 { 6736 struct stmmac_priv *priv = netdev_priv(dev); 6737 u32 chan; 6738 6739 /* Ensure tx function is not running */ 6740 netif_tx_disable(dev); 6741 6742 /* Disable NAPI process */ 6743 stmmac_disable_all_queues(priv); 6744 6745 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6746 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6747 6748 /* Free the IRQ lines */ 6749 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 6750 6751 /* Stop TX/RX DMA channels */ 6752 stmmac_stop_all_dma(priv); 6753 6754 /* Release and free the Rx/Tx resources */ 6755 free_dma_desc_resources(priv, &priv->dma_conf); 6756 6757 /* Disable the MAC Rx/Tx */ 6758 stmmac_mac_set(priv, priv->ioaddr, false); 6759 6760 /* set trans_start so we don't get spurious 6761 * watchdogs during reset 6762 */ 6763 netif_trans_update(dev); 6764 netif_carrier_off(dev); 6765 } 6766 6767 int stmmac_xdp_open(struct net_device *dev) 6768 { 6769 struct stmmac_priv *priv = netdev_priv(dev); 6770 u32 rx_cnt = priv->plat->rx_queues_to_use; 6771 u32 tx_cnt = priv->plat->tx_queues_to_use; 6772 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 6773 struct stmmac_rx_queue *rx_q; 6774 struct stmmac_tx_queue *tx_q; 6775 u32 buf_size; 6776 bool sph_en; 6777 u32 chan; 6778 int ret; 6779 6780 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 6781 if (ret < 0) { 6782 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 6783 __func__); 6784 goto dma_desc_error; 6785 } 6786 6787 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 6788 if (ret < 0) { 6789 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 6790 __func__); 6791 goto init_error; 6792 } 6793 6794 stmmac_reset_queues_param(priv); 6795 6796 /* DMA CSR Channel configuration */ 6797 for (chan = 0; chan < dma_csr_ch; chan++) { 6798 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 6799 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6800 } 6801 6802 /* Adjust Split header */ 6803 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 6804 6805 /* DMA RX Channel Configuration */ 6806 for (chan = 0; chan < rx_cnt; chan++) { 6807 rx_q = &priv->dma_conf.rx_queue[chan]; 6808 6809 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6810 rx_q->dma_rx_phy, chan); 6811 6812 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 6813 (rx_q->buf_alloc_num * 6814 sizeof(struct dma_desc)); 6815 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6816 rx_q->rx_tail_addr, chan); 6817 6818 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6819 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6820 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6821 buf_size, 6822 rx_q->queue_index); 6823 } else { 6824 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6825 priv->dma_conf.dma_buf_sz, 6826 rx_q->queue_index); 6827 } 6828 6829 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6830 } 6831 6832 /* DMA TX Channel Configuration */ 6833 for (chan = 0; chan < tx_cnt; chan++) { 6834 tx_q = &priv->dma_conf.tx_queue[chan]; 6835 6836 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6837 tx_q->dma_tx_phy, chan); 6838 6839 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6840 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6841 tx_q->tx_tail_addr, chan); 6842 6843 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6844 tx_q->txtimer.function = stmmac_tx_timer; 6845 } 6846 6847 /* Enable the MAC Rx/Tx */ 6848 stmmac_mac_set(priv, priv->ioaddr, true); 6849 6850 /* Start Rx & Tx DMA Channels */ 6851 stmmac_start_all_dma(priv); 6852 6853 ret = stmmac_request_irq(dev); 6854 if (ret) 6855 goto irq_error; 6856 6857 /* Enable NAPI process*/ 6858 stmmac_enable_all_queues(priv); 6859 netif_carrier_on(dev); 6860 netif_tx_start_all_queues(dev); 6861 stmmac_enable_all_dma_irq(priv); 6862 6863 return 0; 6864 6865 irq_error: 6866 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6867 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6868 6869 stmmac_hw_teardown(dev); 6870 init_error: 6871 free_dma_desc_resources(priv, &priv->dma_conf); 6872 dma_desc_error: 6873 return ret; 6874 } 6875 6876 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 6877 { 6878 struct stmmac_priv *priv = netdev_priv(dev); 6879 struct stmmac_rx_queue *rx_q; 6880 struct stmmac_tx_queue *tx_q; 6881 struct stmmac_channel *ch; 6882 6883 if (test_bit(STMMAC_DOWN, &priv->state) || 6884 !netif_carrier_ok(priv->dev)) 6885 return -ENETDOWN; 6886 6887 if (!stmmac_xdp_is_enabled(priv)) 6888 return -EINVAL; 6889 6890 if (queue >= priv->plat->rx_queues_to_use || 6891 queue >= priv->plat->tx_queues_to_use) 6892 return -EINVAL; 6893 6894 rx_q = &priv->dma_conf.rx_queue[queue]; 6895 tx_q = &priv->dma_conf.tx_queue[queue]; 6896 ch = &priv->channel[queue]; 6897 6898 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 6899 return -EINVAL; 6900 6901 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 6902 /* EQoS does not have per-DMA channel SW interrupt, 6903 * so we schedule RX Napi straight-away. 6904 */ 6905 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 6906 __napi_schedule(&ch->rxtx_napi); 6907 } 6908 6909 return 0; 6910 } 6911 6912 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 6913 { 6914 struct stmmac_priv *priv = netdev_priv(dev); 6915 u32 tx_cnt = priv->plat->tx_queues_to_use; 6916 u32 rx_cnt = priv->plat->rx_queues_to_use; 6917 unsigned int start; 6918 int q; 6919 6920 for (q = 0; q < tx_cnt; q++) { 6921 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 6922 u64 tx_packets; 6923 u64 tx_bytes; 6924 6925 do { 6926 start = u64_stats_fetch_begin(&txq_stats->syncp); 6927 tx_packets = txq_stats->tx_packets; 6928 tx_bytes = txq_stats->tx_bytes; 6929 } while (u64_stats_fetch_retry(&txq_stats->syncp, start)); 6930 6931 stats->tx_packets += tx_packets; 6932 stats->tx_bytes += tx_bytes; 6933 } 6934 6935 for (q = 0; q < rx_cnt; q++) { 6936 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 6937 u64 rx_packets; 6938 u64 rx_bytes; 6939 6940 do { 6941 start = u64_stats_fetch_begin(&rxq_stats->syncp); 6942 rx_packets = rxq_stats->rx_packets; 6943 rx_bytes = rxq_stats->rx_bytes; 6944 } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 6945 6946 stats->rx_packets += rx_packets; 6947 stats->rx_bytes += rx_bytes; 6948 } 6949 6950 stats->rx_dropped = priv->xstats.rx_dropped; 6951 stats->rx_errors = priv->xstats.rx_errors; 6952 stats->tx_dropped = priv->xstats.tx_dropped; 6953 stats->tx_errors = priv->xstats.tx_errors; 6954 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 6955 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 6956 stats->rx_length_errors = priv->xstats.rx_length; 6957 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 6958 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 6959 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 6960 } 6961 6962 static const struct net_device_ops stmmac_netdev_ops = { 6963 .ndo_open = stmmac_open, 6964 .ndo_start_xmit = stmmac_xmit, 6965 .ndo_stop = stmmac_release, 6966 .ndo_change_mtu = stmmac_change_mtu, 6967 .ndo_fix_features = stmmac_fix_features, 6968 .ndo_set_features = stmmac_set_features, 6969 .ndo_set_rx_mode = stmmac_set_rx_mode, 6970 .ndo_tx_timeout = stmmac_tx_timeout, 6971 .ndo_eth_ioctl = stmmac_ioctl, 6972 .ndo_get_stats64 = stmmac_get_stats64, 6973 .ndo_setup_tc = stmmac_setup_tc, 6974 .ndo_select_queue = stmmac_select_queue, 6975 .ndo_set_mac_address = stmmac_set_mac_address, 6976 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 6977 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 6978 .ndo_bpf = stmmac_bpf, 6979 .ndo_xdp_xmit = stmmac_xdp_xmit, 6980 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 6981 }; 6982 6983 static void stmmac_reset_subtask(struct stmmac_priv *priv) 6984 { 6985 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 6986 return; 6987 if (test_bit(STMMAC_DOWN, &priv->state)) 6988 return; 6989 6990 netdev_err(priv->dev, "Reset adapter.\n"); 6991 6992 rtnl_lock(); 6993 netif_trans_update(priv->dev); 6994 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 6995 usleep_range(1000, 2000); 6996 6997 set_bit(STMMAC_DOWN, &priv->state); 6998 dev_close(priv->dev); 6999 dev_open(priv->dev, NULL); 7000 clear_bit(STMMAC_DOWN, &priv->state); 7001 clear_bit(STMMAC_RESETING, &priv->state); 7002 rtnl_unlock(); 7003 } 7004 7005 static void stmmac_service_task(struct work_struct *work) 7006 { 7007 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7008 service_task); 7009 7010 stmmac_reset_subtask(priv); 7011 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7012 } 7013 7014 /** 7015 * stmmac_hw_init - Init the MAC device 7016 * @priv: driver private structure 7017 * Description: this function is to configure the MAC device according to 7018 * some platform parameters or the HW capability register. It prepares the 7019 * driver to use either ring or chain modes and to setup either enhanced or 7020 * normal descriptors. 7021 */ 7022 static int stmmac_hw_init(struct stmmac_priv *priv) 7023 { 7024 int ret; 7025 7026 /* dwmac-sun8i only work in chain mode */ 7027 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7028 chain_mode = 1; 7029 priv->chain_mode = chain_mode; 7030 7031 /* Initialize HW Interface */ 7032 ret = stmmac_hwif_init(priv); 7033 if (ret) 7034 return ret; 7035 7036 /* Get the HW capability (new GMAC newer than 3.50a) */ 7037 priv->hw_cap_support = stmmac_get_hw_features(priv); 7038 if (priv->hw_cap_support) { 7039 dev_info(priv->device, "DMA HW capability register supported\n"); 7040 7041 /* We can override some gmac/dma configuration fields: e.g. 7042 * enh_desc, tx_coe (e.g. that are passed through the 7043 * platform) with the values from the HW capability 7044 * register (if supported). 7045 */ 7046 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7047 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7048 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7049 priv->hw->pmt = priv->plat->pmt; 7050 if (priv->dma_cap.hash_tb_sz) { 7051 priv->hw->multicast_filter_bins = 7052 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7053 priv->hw->mcast_bits_log2 = 7054 ilog2(priv->hw->multicast_filter_bins); 7055 } 7056 7057 /* TXCOE doesn't work in thresh DMA mode */ 7058 if (priv->plat->force_thresh_dma_mode) 7059 priv->plat->tx_coe = 0; 7060 else 7061 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7062 7063 /* In case of GMAC4 rx_coe is from HW cap register. */ 7064 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7065 7066 if (priv->dma_cap.rx_coe_type2) 7067 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7068 else if (priv->dma_cap.rx_coe_type1) 7069 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7070 7071 } else { 7072 dev_info(priv->device, "No HW DMA feature register supported\n"); 7073 } 7074 7075 if (priv->plat->rx_coe) { 7076 priv->hw->rx_csum = priv->plat->rx_coe; 7077 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7078 if (priv->synopsys_id < DWMAC_CORE_4_00) 7079 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7080 } 7081 if (priv->plat->tx_coe) 7082 dev_info(priv->device, "TX Checksum insertion supported\n"); 7083 7084 if (priv->plat->pmt) { 7085 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7086 device_set_wakeup_capable(priv->device, 1); 7087 } 7088 7089 if (priv->dma_cap.tsoen) 7090 dev_info(priv->device, "TSO supported\n"); 7091 7092 priv->hw->vlan_fail_q_en = 7093 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7094 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7095 7096 /* Run HW quirks, if any */ 7097 if (priv->hwif_quirks) { 7098 ret = priv->hwif_quirks(priv); 7099 if (ret) 7100 return ret; 7101 } 7102 7103 /* Rx Watchdog is available in the COREs newer than the 3.40. 7104 * In some case, for example on bugged HW this feature 7105 * has to be disable and this can be done by passing the 7106 * riwt_off field from the platform. 7107 */ 7108 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 7109 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 7110 priv->use_riwt = 1; 7111 dev_info(priv->device, 7112 "Enable RX Mitigation via HW Watchdog Timer\n"); 7113 } 7114 7115 return 0; 7116 } 7117 7118 static void stmmac_napi_add(struct net_device *dev) 7119 { 7120 struct stmmac_priv *priv = netdev_priv(dev); 7121 u32 queue, maxq; 7122 7123 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7124 7125 for (queue = 0; queue < maxq; queue++) { 7126 struct stmmac_channel *ch = &priv->channel[queue]; 7127 7128 ch->priv_data = priv; 7129 ch->index = queue; 7130 spin_lock_init(&ch->lock); 7131 7132 if (queue < priv->plat->rx_queues_to_use) { 7133 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7134 } 7135 if (queue < priv->plat->tx_queues_to_use) { 7136 netif_napi_add_tx(dev, &ch->tx_napi, 7137 stmmac_napi_poll_tx); 7138 } 7139 if (queue < priv->plat->rx_queues_to_use && 7140 queue < priv->plat->tx_queues_to_use) { 7141 netif_napi_add(dev, &ch->rxtx_napi, 7142 stmmac_napi_poll_rxtx); 7143 } 7144 } 7145 } 7146 7147 static void stmmac_napi_del(struct net_device *dev) 7148 { 7149 struct stmmac_priv *priv = netdev_priv(dev); 7150 u32 queue, maxq; 7151 7152 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7153 7154 for (queue = 0; queue < maxq; queue++) { 7155 struct stmmac_channel *ch = &priv->channel[queue]; 7156 7157 if (queue < priv->plat->rx_queues_to_use) 7158 netif_napi_del(&ch->rx_napi); 7159 if (queue < priv->plat->tx_queues_to_use) 7160 netif_napi_del(&ch->tx_napi); 7161 if (queue < priv->plat->rx_queues_to_use && 7162 queue < priv->plat->tx_queues_to_use) { 7163 netif_napi_del(&ch->rxtx_napi); 7164 } 7165 } 7166 } 7167 7168 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 7169 { 7170 struct stmmac_priv *priv = netdev_priv(dev); 7171 int ret = 0, i; 7172 7173 if (netif_running(dev)) 7174 stmmac_release(dev); 7175 7176 stmmac_napi_del(dev); 7177 7178 priv->plat->rx_queues_to_use = rx_cnt; 7179 priv->plat->tx_queues_to_use = tx_cnt; 7180 if (!netif_is_rxfh_configured(dev)) 7181 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7182 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7183 rx_cnt); 7184 7185 stmmac_napi_add(dev); 7186 7187 if (netif_running(dev)) 7188 ret = stmmac_open(dev); 7189 7190 return ret; 7191 } 7192 7193 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7194 { 7195 struct stmmac_priv *priv = netdev_priv(dev); 7196 int ret = 0; 7197 7198 if (netif_running(dev)) 7199 stmmac_release(dev); 7200 7201 priv->dma_conf.dma_rx_size = rx_size; 7202 priv->dma_conf.dma_tx_size = tx_size; 7203 7204 if (netif_running(dev)) 7205 ret = stmmac_open(dev); 7206 7207 return ret; 7208 } 7209 7210 #define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n" 7211 static void stmmac_fpe_lp_task(struct work_struct *work) 7212 { 7213 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7214 fpe_task); 7215 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 7216 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 7217 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 7218 bool *hs_enable = &fpe_cfg->hs_enable; 7219 bool *enable = &fpe_cfg->enable; 7220 int retries = 20; 7221 7222 while (retries-- > 0) { 7223 /* Bail out immediately if FPE handshake is OFF */ 7224 if (*lo_state == FPE_STATE_OFF || !*hs_enable) 7225 break; 7226 7227 if (*lo_state == FPE_STATE_ENTERING_ON && 7228 *lp_state == FPE_STATE_ENTERING_ON) { 7229 stmmac_fpe_configure(priv, priv->ioaddr, 7230 priv->plat->tx_queues_to_use, 7231 priv->plat->rx_queues_to_use, 7232 *enable); 7233 7234 netdev_info(priv->dev, "configured FPE\n"); 7235 7236 *lo_state = FPE_STATE_ON; 7237 *lp_state = FPE_STATE_ON; 7238 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); 7239 break; 7240 } 7241 7242 if ((*lo_state == FPE_STATE_CAPABLE || 7243 *lo_state == FPE_STATE_ENTERING_ON) && 7244 *lp_state != FPE_STATE_ON) { 7245 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, 7246 *lo_state, *lp_state); 7247 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7248 MPACKET_VERIFY); 7249 } 7250 /* Sleep then retry */ 7251 msleep(500); 7252 } 7253 7254 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 7255 } 7256 7257 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) 7258 { 7259 if (priv->plat->fpe_cfg->hs_enable != enable) { 7260 if (enable) { 7261 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7262 MPACKET_VERIFY); 7263 } else { 7264 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; 7265 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; 7266 } 7267 7268 priv->plat->fpe_cfg->hs_enable = enable; 7269 } 7270 } 7271 7272 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7273 { 7274 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7275 struct dma_desc *desc_contains_ts = ctx->desc; 7276 struct stmmac_priv *priv = ctx->priv; 7277 struct dma_desc *ndesc = ctx->ndesc; 7278 struct dma_desc *desc = ctx->desc; 7279 u64 ns = 0; 7280 7281 if (!priv->hwts_rx_en) 7282 return -ENODATA; 7283 7284 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7285 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 7286 desc_contains_ts = ndesc; 7287 7288 /* Check if timestamp is available */ 7289 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7290 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7291 ns -= priv->plat->cdc_error_adj; 7292 *timestamp = ns_to_ktime(ns); 7293 return 0; 7294 } 7295 7296 return -ENODATA; 7297 } 7298 7299 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7300 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7301 }; 7302 7303 /** 7304 * stmmac_dvr_probe 7305 * @device: device pointer 7306 * @plat_dat: platform data pointer 7307 * @res: stmmac resource pointer 7308 * Description: this is the main probe function used to 7309 * call the alloc_etherdev, allocate the priv structure. 7310 * Return: 7311 * returns 0 on success, otherwise errno. 7312 */ 7313 int stmmac_dvr_probe(struct device *device, 7314 struct plat_stmmacenet_data *plat_dat, 7315 struct stmmac_resources *res) 7316 { 7317 struct net_device *ndev = NULL; 7318 struct stmmac_priv *priv; 7319 u32 rxq; 7320 int i, ret = 0; 7321 7322 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7323 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7324 if (!ndev) 7325 return -ENOMEM; 7326 7327 SET_NETDEV_DEV(ndev, device); 7328 7329 priv = netdev_priv(ndev); 7330 priv->device = device; 7331 priv->dev = ndev; 7332 7333 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7334 u64_stats_init(&priv->xstats.rxq_stats[i].syncp); 7335 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7336 u64_stats_init(&priv->xstats.txq_stats[i].syncp); 7337 7338 stmmac_set_ethtool_ops(ndev); 7339 priv->pause = pause; 7340 priv->plat = plat_dat; 7341 priv->ioaddr = res->addr; 7342 priv->dev->base_addr = (unsigned long)res->addr; 7343 priv->plat->dma_cfg->multi_msi_en = 7344 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7345 7346 priv->dev->irq = res->irq; 7347 priv->wol_irq = res->wol_irq; 7348 priv->lpi_irq = res->lpi_irq; 7349 priv->sfty_ce_irq = res->sfty_ce_irq; 7350 priv->sfty_ue_irq = res->sfty_ue_irq; 7351 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7352 priv->rx_irq[i] = res->rx_irq[i]; 7353 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7354 priv->tx_irq[i] = res->tx_irq[i]; 7355 7356 if (!is_zero_ether_addr(res->mac)) 7357 eth_hw_addr_set(priv->dev, res->mac); 7358 7359 dev_set_drvdata(device, priv->dev); 7360 7361 /* Verify driver arguments */ 7362 stmmac_verify_args(); 7363 7364 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7365 if (!priv->af_xdp_zc_qps) 7366 return -ENOMEM; 7367 7368 /* Allocate workqueue */ 7369 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7370 if (!priv->wq) { 7371 dev_err(priv->device, "failed to create workqueue\n"); 7372 ret = -ENOMEM; 7373 goto error_wq_init; 7374 } 7375 7376 INIT_WORK(&priv->service_task, stmmac_service_task); 7377 7378 /* Initialize Link Partner FPE workqueue */ 7379 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); 7380 7381 /* Override with kernel parameters if supplied XXX CRS XXX 7382 * this needs to have multiple instances 7383 */ 7384 if ((phyaddr >= 0) && (phyaddr <= 31)) 7385 priv->plat->phy_addr = phyaddr; 7386 7387 if (priv->plat->stmmac_rst) { 7388 ret = reset_control_assert(priv->plat->stmmac_rst); 7389 reset_control_deassert(priv->plat->stmmac_rst); 7390 /* Some reset controllers have only reset callback instead of 7391 * assert + deassert callbacks pair. 7392 */ 7393 if (ret == -ENOTSUPP) 7394 reset_control_reset(priv->plat->stmmac_rst); 7395 } 7396 7397 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7398 if (ret == -ENOTSUPP) 7399 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7400 ERR_PTR(ret)); 7401 7402 /* Init MAC and get the capabilities */ 7403 ret = stmmac_hw_init(priv); 7404 if (ret) 7405 goto error_hw_init; 7406 7407 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7408 */ 7409 if (priv->synopsys_id < DWMAC_CORE_5_20) 7410 priv->plat->dma_cfg->dche = false; 7411 7412 stmmac_check_ether_addr(priv); 7413 7414 ndev->netdev_ops = &stmmac_netdev_ops; 7415 7416 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7417 7418 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7419 NETIF_F_RXCSUM; 7420 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7421 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7422 7423 ret = stmmac_tc_init(priv, priv); 7424 if (!ret) { 7425 ndev->hw_features |= NETIF_F_HW_TC; 7426 } 7427 7428 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 7429 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7430 if (priv->plat->has_gmac4) 7431 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7432 priv->tso = true; 7433 dev_info(priv->device, "TSO feature enabled\n"); 7434 } 7435 7436 if (priv->dma_cap.sphen && 7437 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7438 ndev->hw_features |= NETIF_F_GRO; 7439 priv->sph_cap = true; 7440 priv->sph = priv->sph_cap; 7441 dev_info(priv->device, "SPH feature enabled\n"); 7442 } 7443 7444 /* Ideally our host DMA address width is the same as for the 7445 * device. However, it may differ and then we have to use our 7446 * host DMA width for allocation and the device DMA width for 7447 * register handling. 7448 */ 7449 if (priv->plat->host_dma_width) 7450 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7451 else 7452 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7453 7454 if (priv->dma_cap.host_dma_width) { 7455 ret = dma_set_mask_and_coherent(device, 7456 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7457 if (!ret) { 7458 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7459 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7460 7461 /* 7462 * If more than 32 bits can be addressed, make sure to 7463 * enable enhanced addressing mode. 7464 */ 7465 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7466 priv->plat->dma_cfg->eame = true; 7467 } else { 7468 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7469 if (ret) { 7470 dev_err(priv->device, "Failed to set DMA Mask\n"); 7471 goto error_hw_init; 7472 } 7473 7474 priv->dma_cap.host_dma_width = 32; 7475 } 7476 } 7477 7478 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7479 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7480 #ifdef STMMAC_VLAN_TAG_USED 7481 /* Both mac100 and gmac support receive VLAN tag detection */ 7482 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7483 if (priv->dma_cap.vlhash) { 7484 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7485 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7486 } 7487 if (priv->dma_cap.vlins) { 7488 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7489 if (priv->dma_cap.dvlan) 7490 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; 7491 } 7492 #endif 7493 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7494 7495 priv->xstats.threshold = tc; 7496 7497 /* Initialize RSS */ 7498 rxq = priv->plat->rx_queues_to_use; 7499 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7500 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7501 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7502 7503 if (priv->dma_cap.rssen && priv->plat->rss_en) 7504 ndev->features |= NETIF_F_RXHASH; 7505 7506 ndev->vlan_features |= ndev->features; 7507 /* TSO doesn't work on VLANs yet */ 7508 ndev->vlan_features &= ~NETIF_F_TSO; 7509 7510 /* MTU range: 46 - hw-specific max */ 7511 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7512 if (priv->plat->has_xgmac) 7513 ndev->max_mtu = XGMAC_JUMBO_LEN; 7514 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 7515 ndev->max_mtu = JUMBO_LEN; 7516 else 7517 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7518 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 7519 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 7520 */ 7521 if ((priv->plat->maxmtu < ndev->max_mtu) && 7522 (priv->plat->maxmtu >= ndev->min_mtu)) 7523 ndev->max_mtu = priv->plat->maxmtu; 7524 else if (priv->plat->maxmtu < ndev->min_mtu) 7525 dev_warn(priv->device, 7526 "%s: warning: maxmtu having invalid value (%d)\n", 7527 __func__, priv->plat->maxmtu); 7528 7529 if (flow_ctrl) 7530 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 7531 7532 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 7533 7534 /* Setup channels NAPI */ 7535 stmmac_napi_add(ndev); 7536 7537 mutex_init(&priv->lock); 7538 7539 /* If a specific clk_csr value is passed from the platform 7540 * this means that the CSR Clock Range selection cannot be 7541 * changed at run-time and it is fixed. Viceversa the driver'll try to 7542 * set the MDC clock dynamically according to the csr actual 7543 * clock input. 7544 */ 7545 if (priv->plat->clk_csr >= 0) 7546 priv->clk_csr = priv->plat->clk_csr; 7547 else 7548 stmmac_clk_csr_set(priv); 7549 7550 stmmac_check_pcs_mode(priv); 7551 7552 pm_runtime_get_noresume(device); 7553 pm_runtime_set_active(device); 7554 if (!pm_runtime_enabled(device)) 7555 pm_runtime_enable(device); 7556 7557 if (priv->hw->pcs != STMMAC_PCS_TBI && 7558 priv->hw->pcs != STMMAC_PCS_RTBI) { 7559 /* MDIO bus Registration */ 7560 ret = stmmac_mdio_register(ndev); 7561 if (ret < 0) { 7562 dev_err_probe(priv->device, ret, 7563 "%s: MDIO bus (id: %d) registration failed\n", 7564 __func__, priv->plat->bus_id); 7565 goto error_mdio_register; 7566 } 7567 } 7568 7569 if (priv->plat->speed_mode_2500) 7570 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); 7571 7572 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { 7573 ret = stmmac_xpcs_setup(priv->mii); 7574 if (ret) 7575 goto error_xpcs_setup; 7576 } 7577 7578 ret = stmmac_phy_setup(priv); 7579 if (ret) { 7580 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7581 goto error_phy_setup; 7582 } 7583 7584 ret = register_netdev(ndev); 7585 if (ret) { 7586 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7587 __func__, ret); 7588 goto error_netdev_register; 7589 } 7590 7591 #ifdef CONFIG_DEBUG_FS 7592 stmmac_init_fs(ndev); 7593 #endif 7594 7595 if (priv->plat->dump_debug_regs) 7596 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7597 7598 /* Let pm_runtime_put() disable the clocks. 7599 * If CONFIG_PM is not enabled, the clocks will stay powered. 7600 */ 7601 pm_runtime_put(device); 7602 7603 return ret; 7604 7605 error_netdev_register: 7606 phylink_destroy(priv->phylink); 7607 error_xpcs_setup: 7608 error_phy_setup: 7609 if (priv->hw->pcs != STMMAC_PCS_TBI && 7610 priv->hw->pcs != STMMAC_PCS_RTBI) 7611 stmmac_mdio_unregister(ndev); 7612 error_mdio_register: 7613 stmmac_napi_del(ndev); 7614 error_hw_init: 7615 destroy_workqueue(priv->wq); 7616 error_wq_init: 7617 bitmap_free(priv->af_xdp_zc_qps); 7618 7619 return ret; 7620 } 7621 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 7622 7623 /** 7624 * stmmac_dvr_remove 7625 * @dev: device pointer 7626 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 7627 * changes the link status, releases the DMA descriptor rings. 7628 */ 7629 void stmmac_dvr_remove(struct device *dev) 7630 { 7631 struct net_device *ndev = dev_get_drvdata(dev); 7632 struct stmmac_priv *priv = netdev_priv(ndev); 7633 7634 netdev_info(priv->dev, "%s: removing driver", __func__); 7635 7636 pm_runtime_get_sync(dev); 7637 7638 stmmac_stop_all_dma(priv); 7639 stmmac_mac_set(priv, priv->ioaddr, false); 7640 netif_carrier_off(ndev); 7641 unregister_netdev(ndev); 7642 7643 #ifdef CONFIG_DEBUG_FS 7644 stmmac_exit_fs(ndev); 7645 #endif 7646 phylink_destroy(priv->phylink); 7647 if (priv->plat->stmmac_rst) 7648 reset_control_assert(priv->plat->stmmac_rst); 7649 reset_control_assert(priv->plat->stmmac_ahb_rst); 7650 if (priv->hw->pcs != STMMAC_PCS_TBI && 7651 priv->hw->pcs != STMMAC_PCS_RTBI) 7652 stmmac_mdio_unregister(ndev); 7653 destroy_workqueue(priv->wq); 7654 mutex_destroy(&priv->lock); 7655 bitmap_free(priv->af_xdp_zc_qps); 7656 7657 pm_runtime_disable(dev); 7658 pm_runtime_put_noidle(dev); 7659 } 7660 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 7661 7662 /** 7663 * stmmac_suspend - suspend callback 7664 * @dev: device pointer 7665 * Description: this is the function to suspend the device and it is called 7666 * by the platform driver to stop the network queue, release the resources, 7667 * program the PMT register (for WoL), clean and release driver resources. 7668 */ 7669 int stmmac_suspend(struct device *dev) 7670 { 7671 struct net_device *ndev = dev_get_drvdata(dev); 7672 struct stmmac_priv *priv = netdev_priv(ndev); 7673 u32 chan; 7674 7675 if (!ndev || !netif_running(ndev)) 7676 return 0; 7677 7678 mutex_lock(&priv->lock); 7679 7680 netif_device_detach(ndev); 7681 7682 stmmac_disable_all_queues(priv); 7683 7684 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7685 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7686 7687 if (priv->eee_enabled) { 7688 priv->tx_path_in_lpi_mode = false; 7689 del_timer_sync(&priv->eee_ctrl_timer); 7690 } 7691 7692 /* Stop TX/RX DMA */ 7693 stmmac_stop_all_dma(priv); 7694 7695 if (priv->plat->serdes_powerdown) 7696 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7697 7698 /* Enable Power down mode by programming the PMT regs */ 7699 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7700 stmmac_pmt(priv, priv->hw, priv->wolopts); 7701 priv->irq_wake = 1; 7702 } else { 7703 stmmac_mac_set(priv, priv->ioaddr, false); 7704 pinctrl_pm_select_sleep_state(priv->device); 7705 } 7706 7707 mutex_unlock(&priv->lock); 7708 7709 rtnl_lock(); 7710 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7711 phylink_suspend(priv->phylink, true); 7712 } else { 7713 if (device_may_wakeup(priv->device)) 7714 phylink_speed_down(priv->phylink, false); 7715 phylink_suspend(priv->phylink, false); 7716 } 7717 rtnl_unlock(); 7718 7719 if (priv->dma_cap.fpesel) { 7720 /* Disable FPE */ 7721 stmmac_fpe_configure(priv, priv->ioaddr, 7722 priv->plat->tx_queues_to_use, 7723 priv->plat->rx_queues_to_use, false); 7724 7725 stmmac_fpe_handshake(priv, false); 7726 stmmac_fpe_stop_wq(priv); 7727 } 7728 7729 priv->speed = SPEED_UNKNOWN; 7730 return 0; 7731 } 7732 EXPORT_SYMBOL_GPL(stmmac_suspend); 7733 7734 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 7735 { 7736 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7737 7738 rx_q->cur_rx = 0; 7739 rx_q->dirty_rx = 0; 7740 } 7741 7742 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 7743 { 7744 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7745 7746 tx_q->cur_tx = 0; 7747 tx_q->dirty_tx = 0; 7748 tx_q->mss = 0; 7749 7750 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 7751 } 7752 7753 /** 7754 * stmmac_reset_queues_param - reset queue parameters 7755 * @priv: device pointer 7756 */ 7757 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 7758 { 7759 u32 rx_cnt = priv->plat->rx_queues_to_use; 7760 u32 tx_cnt = priv->plat->tx_queues_to_use; 7761 u32 queue; 7762 7763 for (queue = 0; queue < rx_cnt; queue++) 7764 stmmac_reset_rx_queue(priv, queue); 7765 7766 for (queue = 0; queue < tx_cnt; queue++) 7767 stmmac_reset_tx_queue(priv, queue); 7768 } 7769 7770 /** 7771 * stmmac_resume - resume callback 7772 * @dev: device pointer 7773 * Description: when resume this function is invoked to setup the DMA and CORE 7774 * in a usable state. 7775 */ 7776 int stmmac_resume(struct device *dev) 7777 { 7778 struct net_device *ndev = dev_get_drvdata(dev); 7779 struct stmmac_priv *priv = netdev_priv(ndev); 7780 int ret; 7781 7782 if (!netif_running(ndev)) 7783 return 0; 7784 7785 /* Power Down bit, into the PM register, is cleared 7786 * automatically as soon as a magic packet or a Wake-up frame 7787 * is received. Anyway, it's better to manually clear 7788 * this bit because it can generate problems while resuming 7789 * from another devices (e.g. serial console). 7790 */ 7791 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7792 mutex_lock(&priv->lock); 7793 stmmac_pmt(priv, priv->hw, 0); 7794 mutex_unlock(&priv->lock); 7795 priv->irq_wake = 0; 7796 } else { 7797 pinctrl_pm_select_default_state(priv->device); 7798 /* reset the phy so that it's ready */ 7799 if (priv->mii) 7800 stmmac_mdio_reset(priv->mii); 7801 } 7802 7803 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 7804 priv->plat->serdes_powerup) { 7805 ret = priv->plat->serdes_powerup(ndev, 7806 priv->plat->bsp_priv); 7807 7808 if (ret < 0) 7809 return ret; 7810 } 7811 7812 rtnl_lock(); 7813 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7814 phylink_resume(priv->phylink); 7815 } else { 7816 phylink_resume(priv->phylink); 7817 if (device_may_wakeup(priv->device)) 7818 phylink_speed_up(priv->phylink); 7819 } 7820 rtnl_unlock(); 7821 7822 rtnl_lock(); 7823 mutex_lock(&priv->lock); 7824 7825 stmmac_reset_queues_param(priv); 7826 7827 stmmac_free_tx_skbufs(priv); 7828 stmmac_clear_descriptors(priv, &priv->dma_conf); 7829 7830 stmmac_hw_setup(ndev, false); 7831 stmmac_init_coalesce(priv); 7832 stmmac_set_rx_mode(ndev); 7833 7834 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); 7835 7836 stmmac_enable_all_queues(priv); 7837 stmmac_enable_all_dma_irq(priv); 7838 7839 mutex_unlock(&priv->lock); 7840 rtnl_unlock(); 7841 7842 netif_device_attach(ndev); 7843 7844 return 0; 7845 } 7846 EXPORT_SYMBOL_GPL(stmmac_resume); 7847 7848 #ifndef MODULE 7849 static int __init stmmac_cmdline_opt(char *str) 7850 { 7851 char *opt; 7852 7853 if (!str || !*str) 7854 return 1; 7855 while ((opt = strsep(&str, ",")) != NULL) { 7856 if (!strncmp(opt, "debug:", 6)) { 7857 if (kstrtoint(opt + 6, 0, &debug)) 7858 goto err; 7859 } else if (!strncmp(opt, "phyaddr:", 8)) { 7860 if (kstrtoint(opt + 8, 0, &phyaddr)) 7861 goto err; 7862 } else if (!strncmp(opt, "buf_sz:", 7)) { 7863 if (kstrtoint(opt + 7, 0, &buf_sz)) 7864 goto err; 7865 } else if (!strncmp(opt, "tc:", 3)) { 7866 if (kstrtoint(opt + 3, 0, &tc)) 7867 goto err; 7868 } else if (!strncmp(opt, "watchdog:", 9)) { 7869 if (kstrtoint(opt + 9, 0, &watchdog)) 7870 goto err; 7871 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 7872 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 7873 goto err; 7874 } else if (!strncmp(opt, "pause:", 6)) { 7875 if (kstrtoint(opt + 6, 0, &pause)) 7876 goto err; 7877 } else if (!strncmp(opt, "eee_timer:", 10)) { 7878 if (kstrtoint(opt + 10, 0, &eee_timer)) 7879 goto err; 7880 } else if (!strncmp(opt, "chain_mode:", 11)) { 7881 if (kstrtoint(opt + 11, 0, &chain_mode)) 7882 goto err; 7883 } 7884 } 7885 return 1; 7886 7887 err: 7888 pr_err("%s: ERROR broken module parameter conversion", __func__); 7889 return 1; 7890 } 7891 7892 __setup("stmmaceth=", stmmac_cmdline_opt); 7893 #endif /* MODULE */ 7894 7895 static int __init stmmac_init(void) 7896 { 7897 #ifdef CONFIG_DEBUG_FS 7898 /* Create debugfs main directory if it doesn't exist yet */ 7899 if (!stmmac_fs_dir) 7900 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 7901 register_netdevice_notifier(&stmmac_notifier); 7902 #endif 7903 7904 return 0; 7905 } 7906 7907 static void __exit stmmac_exit(void) 7908 { 7909 #ifdef CONFIG_DEBUG_FS 7910 unregister_netdevice_notifier(&stmmac_notifier); 7911 debugfs_remove_recursive(stmmac_fs_dir); 7912 #endif 7913 } 7914 7915 module_init(stmmac_init) 7916 module_exit(stmmac_exit) 7917 7918 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 7919 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 7920 MODULE_LICENSE("GPL"); 7921