1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/clk.h> 18 #include <linux/kernel.h> 19 #include <linux/interrupt.h> 20 #include <linux/ip.h> 21 #include <linux/tcp.h> 22 #include <linux/skbuff.h> 23 #include <linux/ethtool.h> 24 #include <linux/if_ether.h> 25 #include <linux/crc32.h> 26 #include <linux/mii.h> 27 #include <linux/if.h> 28 #include <linux/if_vlan.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/slab.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/prefetch.h> 33 #include <linux/pinctrl/consumer.h> 34 #ifdef CONFIG_DEBUG_FS 35 #include <linux/debugfs.h> 36 #include <linux/seq_file.h> 37 #endif /* CONFIG_DEBUG_FS */ 38 #include <linux/net_tstamp.h> 39 #include <linux/phylink.h> 40 #include <linux/udp.h> 41 #include <linux/bpf_trace.h> 42 #include <net/page_pool/helpers.h> 43 #include <net/pkt_cls.h> 44 #include <net/xdp_sock_drv.h> 45 #include "stmmac_ptp.h" 46 #include "stmmac_fpe.h" 47 #include "stmmac.h" 48 #include "stmmac_xdp.h" 49 #include <linux/reset.h> 50 #include <linux/of_mdio.h> 51 #include "dwmac1000.h" 52 #include "dwxgmac2.h" 53 #include "hwif.h" 54 55 /* As long as the interface is active, we keep the timestamping counter enabled 56 * with fine resolution and binary rollover. This avoid non-monotonic behavior 57 * (clock jumps) when changing timestamping settings at runtime. 58 */ 59 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ 60 PTP_TCR_TSCTRLSSR) 61 62 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 63 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 64 65 /* Module parameters */ 66 #define TX_TIMEO 5000 67 static int watchdog = TX_TIMEO; 68 module_param(watchdog, int, 0644); 69 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 70 71 static int debug = -1; 72 module_param(debug, int, 0644); 73 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 74 75 static int phyaddr = -1; 76 module_param(phyaddr, int, 0444); 77 MODULE_PARM_DESC(phyaddr, "Physical device address"); 78 79 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 80 #define STMMAC_RX_THRESH(x) ((x)->dma_conf.dma_rx_size / 4) 81 82 /* Limit to make sure XDP TX and slow path can coexist */ 83 #define STMMAC_XSK_TX_BUDGET_MAX 256 84 #define STMMAC_TX_XSK_AVAIL 16 85 #define STMMAC_RX_FILL_BATCH 16 86 87 #define STMMAC_XDP_PASS 0 88 #define STMMAC_XDP_CONSUMED BIT(0) 89 #define STMMAC_XDP_TX BIT(1) 90 #define STMMAC_XDP_REDIRECT BIT(2) 91 92 static int flow_ctrl = FLOW_AUTO; 93 module_param(flow_ctrl, int, 0644); 94 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 95 96 static int pause = PAUSE_TIME; 97 module_param(pause, int, 0644); 98 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 99 100 #define TC_DEFAULT 64 101 static int tc = TC_DEFAULT; 102 module_param(tc, int, 0644); 103 MODULE_PARM_DESC(tc, "DMA threshold control value"); 104 105 #define DEFAULT_BUFSIZE 1536 106 static int buf_sz = DEFAULT_BUFSIZE; 107 module_param(buf_sz, int, 0644); 108 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 109 110 #define STMMAC_RX_COPYBREAK 256 111 112 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 113 NETIF_MSG_LINK | NETIF_MSG_IFUP | 114 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 115 116 #define STMMAC_DEFAULT_LPI_TIMER 1000 117 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 118 module_param(eee_timer, int, 0644); 119 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 120 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 121 122 /* By default the driver will use the ring mode to manage tx and rx descriptors, 123 * but allow user to force to use the chain instead of the ring 124 */ 125 static unsigned int chain_mode; 126 module_param(chain_mode, int, 0444); 127 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 128 129 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 130 /* For MSI interrupts handling */ 131 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 132 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 133 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 134 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 135 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 136 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 137 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 138 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 139 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 140 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 141 u32 rxmode, u32 chan); 142 143 #ifdef CONFIG_DEBUG_FS 144 static const struct net_device_ops stmmac_netdev_ops; 145 static void stmmac_init_fs(struct net_device *dev); 146 static void stmmac_exit_fs(struct net_device *dev); 147 #endif 148 149 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 150 151 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) 152 { 153 int ret = 0; 154 155 if (enabled) { 156 ret = clk_prepare_enable(priv->plat->stmmac_clk); 157 if (ret) 158 return ret; 159 ret = clk_prepare_enable(priv->plat->pclk); 160 if (ret) { 161 clk_disable_unprepare(priv->plat->stmmac_clk); 162 return ret; 163 } 164 if (priv->plat->clks_config) { 165 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); 166 if (ret) { 167 clk_disable_unprepare(priv->plat->stmmac_clk); 168 clk_disable_unprepare(priv->plat->pclk); 169 return ret; 170 } 171 } 172 } else { 173 clk_disable_unprepare(priv->plat->stmmac_clk); 174 clk_disable_unprepare(priv->plat->pclk); 175 if (priv->plat->clks_config) 176 priv->plat->clks_config(priv->plat->bsp_priv, enabled); 177 } 178 179 return ret; 180 } 181 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); 182 183 /** 184 * stmmac_verify_args - verify the driver parameters. 185 * Description: it checks the driver parameters and set a default in case of 186 * errors. 187 */ 188 static void stmmac_verify_args(void) 189 { 190 if (unlikely(watchdog < 0)) 191 watchdog = TX_TIMEO; 192 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 193 buf_sz = DEFAULT_BUFSIZE; 194 if (unlikely(flow_ctrl > 1)) 195 flow_ctrl = FLOW_AUTO; 196 else if (likely(flow_ctrl < 0)) 197 flow_ctrl = FLOW_OFF; 198 if (unlikely((pause < 0) || (pause > 0xffff))) 199 pause = PAUSE_TIME; 200 if (eee_timer < 0) 201 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 202 } 203 204 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 205 { 206 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 207 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 208 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 209 u32 queue; 210 211 for (queue = 0; queue < maxq; queue++) { 212 struct stmmac_channel *ch = &priv->channel[queue]; 213 214 if (stmmac_xdp_is_enabled(priv) && 215 test_bit(queue, priv->af_xdp_zc_qps)) { 216 napi_disable(&ch->rxtx_napi); 217 continue; 218 } 219 220 if (queue < rx_queues_cnt) 221 napi_disable(&ch->rx_napi); 222 if (queue < tx_queues_cnt) 223 napi_disable(&ch->tx_napi); 224 } 225 } 226 227 /** 228 * stmmac_disable_all_queues - Disable all queues 229 * @priv: driver private structure 230 */ 231 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 232 { 233 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 234 struct stmmac_rx_queue *rx_q; 235 u32 queue; 236 237 /* synchronize_rcu() needed for pending XDP buffers to drain */ 238 for (queue = 0; queue < rx_queues_cnt; queue++) { 239 rx_q = &priv->dma_conf.rx_queue[queue]; 240 if (rx_q->xsk_pool) { 241 synchronize_rcu(); 242 break; 243 } 244 } 245 246 __stmmac_disable_all_queues(priv); 247 } 248 249 /** 250 * stmmac_enable_all_queues - Enable all queues 251 * @priv: driver private structure 252 */ 253 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 254 { 255 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 256 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 257 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 258 u32 queue; 259 260 for (queue = 0; queue < maxq; queue++) { 261 struct stmmac_channel *ch = &priv->channel[queue]; 262 263 if (stmmac_xdp_is_enabled(priv) && 264 test_bit(queue, priv->af_xdp_zc_qps)) { 265 napi_enable(&ch->rxtx_napi); 266 continue; 267 } 268 269 if (queue < rx_queues_cnt) 270 napi_enable(&ch->rx_napi); 271 if (queue < tx_queues_cnt) 272 napi_enable(&ch->tx_napi); 273 } 274 } 275 276 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 277 { 278 if (!test_bit(STMMAC_DOWN, &priv->state) && 279 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 280 queue_work(priv->wq, &priv->service_task); 281 } 282 283 static void stmmac_global_err(struct stmmac_priv *priv) 284 { 285 netif_carrier_off(priv->dev); 286 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 287 stmmac_service_event_schedule(priv); 288 } 289 290 /** 291 * stmmac_clk_csr_set - dynamically set the MDC clock 292 * @priv: driver private structure 293 * Description: this is to dynamically set the MDC clock according to the csr 294 * clock input. 295 * Note: 296 * If a specific clk_csr value is passed from the platform 297 * this means that the CSR Clock Range selection cannot be 298 * changed at run-time and it is fixed (as reported in the driver 299 * documentation). Viceversa the driver will try to set the MDC 300 * clock dynamically according to the actual clock input. 301 */ 302 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 303 { 304 u32 clk_rate; 305 306 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 307 308 /* Platform provided default clk_csr would be assumed valid 309 * for all other cases except for the below mentioned ones. 310 * For values higher than the IEEE 802.3 specified frequency 311 * we can not estimate the proper divider as it is not known 312 * the frequency of clk_csr_i. So we do not change the default 313 * divider. 314 */ 315 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 316 if (clk_rate < CSR_F_35M) 317 priv->clk_csr = STMMAC_CSR_20_35M; 318 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 319 priv->clk_csr = STMMAC_CSR_35_60M; 320 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 321 priv->clk_csr = STMMAC_CSR_60_100M; 322 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 323 priv->clk_csr = STMMAC_CSR_100_150M; 324 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 325 priv->clk_csr = STMMAC_CSR_150_250M; 326 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) 327 priv->clk_csr = STMMAC_CSR_250_300M; 328 } 329 330 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { 331 if (clk_rate > 160000000) 332 priv->clk_csr = 0x03; 333 else if (clk_rate > 80000000) 334 priv->clk_csr = 0x02; 335 else if (clk_rate > 40000000) 336 priv->clk_csr = 0x01; 337 else 338 priv->clk_csr = 0; 339 } 340 341 if (priv->plat->has_xgmac) { 342 if (clk_rate > 400000000) 343 priv->clk_csr = 0x5; 344 else if (clk_rate > 350000000) 345 priv->clk_csr = 0x4; 346 else if (clk_rate > 300000000) 347 priv->clk_csr = 0x3; 348 else if (clk_rate > 250000000) 349 priv->clk_csr = 0x2; 350 else if (clk_rate > 150000000) 351 priv->clk_csr = 0x1; 352 else 353 priv->clk_csr = 0x0; 354 } 355 } 356 357 static void print_pkt(unsigned char *buf, int len) 358 { 359 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 360 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 361 } 362 363 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 364 { 365 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 366 u32 avail; 367 368 if (tx_q->dirty_tx > tx_q->cur_tx) 369 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 370 else 371 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 372 373 return avail; 374 } 375 376 /** 377 * stmmac_rx_dirty - Get RX queue dirty 378 * @priv: driver private structure 379 * @queue: RX queue index 380 */ 381 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 382 { 383 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 384 u32 dirty; 385 386 if (rx_q->dirty_rx <= rx_q->cur_rx) 387 dirty = rx_q->cur_rx - rx_q->dirty_rx; 388 else 389 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 390 391 return dirty; 392 } 393 394 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) 395 { 396 int tx_lpi_timer; 397 398 /* Clear/set the SW EEE timer flag based on LPI ET enablement */ 399 priv->eee_sw_timer_en = en ? 0 : 1; 400 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; 401 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); 402 } 403 404 /** 405 * stmmac_enable_eee_mode - check and enter in LPI mode 406 * @priv: driver private structure 407 * Description: this function is to verify and enter in LPI mode in case of 408 * EEE. 409 */ 410 static int stmmac_enable_eee_mode(struct stmmac_priv *priv) 411 { 412 u32 tx_cnt = priv->plat->tx_queues_to_use; 413 u32 queue; 414 415 /* check if all TX queues have the work finished */ 416 for (queue = 0; queue < tx_cnt; queue++) { 417 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 418 419 if (tx_q->dirty_tx != tx_q->cur_tx) 420 return -EBUSY; /* still unfinished work */ 421 } 422 423 /* Check and enter in LPI mode */ 424 if (!priv->tx_path_in_lpi_mode) 425 stmmac_set_eee_mode(priv, priv->hw, 426 priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); 427 return 0; 428 } 429 430 /** 431 * stmmac_disable_eee_mode - disable and exit from LPI mode 432 * @priv: driver private structure 433 * Description: this function is to exit and disable EEE in case of 434 * LPI state is true. This is called by the xmit. 435 */ 436 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 437 { 438 if (!priv->eee_sw_timer_en) { 439 stmmac_lpi_entry_timer_config(priv, 0); 440 return; 441 } 442 443 stmmac_reset_eee_mode(priv, priv->hw); 444 del_timer_sync(&priv->eee_ctrl_timer); 445 priv->tx_path_in_lpi_mode = false; 446 } 447 448 /** 449 * stmmac_eee_ctrl_timer - EEE TX SW timer. 450 * @t: timer_list struct containing private info 451 * Description: 452 * if there is no data transfer and if we are not in LPI state, 453 * then MAC Transmitter can be moved to LPI state. 454 */ 455 static void stmmac_eee_ctrl_timer(struct timer_list *t) 456 { 457 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 458 459 if (stmmac_enable_eee_mode(priv)) 460 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 461 } 462 463 /** 464 * stmmac_eee_init - init EEE 465 * @priv: driver private structure 466 * Description: 467 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 468 * can also manage EEE, this function enable the LPI state and start related 469 * timer. 470 */ 471 bool stmmac_eee_init(struct stmmac_priv *priv) 472 { 473 int eee_tw_timer = priv->eee_tw_timer; 474 475 /* Check if MAC core supports the EEE feature. */ 476 if (!priv->dma_cap.eee) 477 return false; 478 479 mutex_lock(&priv->lock); 480 481 /* Check if it needs to be deactivated */ 482 if (!priv->eee_active) { 483 if (priv->eee_enabled) { 484 netdev_dbg(priv->dev, "disable EEE\n"); 485 stmmac_lpi_entry_timer_config(priv, 0); 486 del_timer_sync(&priv->eee_ctrl_timer); 487 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); 488 if (priv->hw->xpcs) 489 xpcs_config_eee(priv->hw->xpcs, 490 priv->plat->mult_fact_100ns, 491 false); 492 } 493 mutex_unlock(&priv->lock); 494 return false; 495 } 496 497 if (priv->eee_active && !priv->eee_enabled) { 498 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 499 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 500 eee_tw_timer); 501 if (priv->hw->xpcs) 502 xpcs_config_eee(priv->hw->xpcs, 503 priv->plat->mult_fact_100ns, 504 true); 505 } 506 507 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { 508 del_timer_sync(&priv->eee_ctrl_timer); 509 priv->tx_path_in_lpi_mode = false; 510 stmmac_lpi_entry_timer_config(priv, 1); 511 } else { 512 stmmac_lpi_entry_timer_config(priv, 0); 513 mod_timer(&priv->eee_ctrl_timer, 514 STMMAC_LPI_T(priv->tx_lpi_timer)); 515 } 516 517 mutex_unlock(&priv->lock); 518 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 519 return true; 520 } 521 522 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 523 * @priv: driver private structure 524 * @p : descriptor pointer 525 * @skb : the socket buffer 526 * Description : 527 * This function will read timestamp from the descriptor & pass it to stack. 528 * and also perform some sanity checks. 529 */ 530 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 531 struct dma_desc *p, struct sk_buff *skb) 532 { 533 struct skb_shared_hwtstamps shhwtstamp; 534 bool found = false; 535 u64 ns = 0; 536 537 if (!priv->hwts_tx_en) 538 return; 539 540 /* exit if skb doesn't support hw tstamp */ 541 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 542 return; 543 544 /* check tx tstamp status */ 545 if (stmmac_get_tx_timestamp_status(priv, p)) { 546 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 547 found = true; 548 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 549 found = true; 550 } 551 552 if (found) { 553 ns -= priv->plat->cdc_error_adj; 554 555 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 556 shhwtstamp.hwtstamp = ns_to_ktime(ns); 557 558 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 559 /* pass tstamp to stack */ 560 skb_tstamp_tx(skb, &shhwtstamp); 561 } 562 } 563 564 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 565 * @priv: driver private structure 566 * @p : descriptor pointer 567 * @np : next descriptor pointer 568 * @skb : the socket buffer 569 * Description : 570 * This function will read received packet's timestamp from the descriptor 571 * and pass it to stack. It also perform some sanity checks. 572 */ 573 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 574 struct dma_desc *np, struct sk_buff *skb) 575 { 576 struct skb_shared_hwtstamps *shhwtstamp = NULL; 577 struct dma_desc *desc = p; 578 u64 ns = 0; 579 580 if (!priv->hwts_rx_en) 581 return; 582 /* For GMAC4, the valid timestamp is from CTX next desc. */ 583 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 584 desc = np; 585 586 /* Check if timestamp is available */ 587 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 588 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 589 590 ns -= priv->plat->cdc_error_adj; 591 592 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 593 shhwtstamp = skb_hwtstamps(skb); 594 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 595 shhwtstamp->hwtstamp = ns_to_ktime(ns); 596 } else { 597 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 598 } 599 } 600 601 /** 602 * stmmac_hwtstamp_set - control hardware timestamping. 603 * @dev: device pointer. 604 * @ifr: An IOCTL specific structure, that can contain a pointer to 605 * a proprietary structure used to pass information to the driver. 606 * Description: 607 * This function configures the MAC to enable/disable both outgoing(TX) 608 * and incoming(RX) packets time stamping based on user input. 609 * Return Value: 610 * 0 on success and an appropriate -ve integer on failure. 611 */ 612 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 613 { 614 struct stmmac_priv *priv = netdev_priv(dev); 615 struct hwtstamp_config config; 616 u32 ptp_v2 = 0; 617 u32 tstamp_all = 0; 618 u32 ptp_over_ipv4_udp = 0; 619 u32 ptp_over_ipv6_udp = 0; 620 u32 ptp_over_ethernet = 0; 621 u32 snap_type_sel = 0; 622 u32 ts_master_en = 0; 623 u32 ts_event_en = 0; 624 625 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 626 netdev_alert(priv->dev, "No support for HW time stamping\n"); 627 priv->hwts_tx_en = 0; 628 priv->hwts_rx_en = 0; 629 630 return -EOPNOTSUPP; 631 } 632 633 if (copy_from_user(&config, ifr->ifr_data, 634 sizeof(config))) 635 return -EFAULT; 636 637 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 638 __func__, config.flags, config.tx_type, config.rx_filter); 639 640 if (config.tx_type != HWTSTAMP_TX_OFF && 641 config.tx_type != HWTSTAMP_TX_ON) 642 return -ERANGE; 643 644 if (priv->adv_ts) { 645 switch (config.rx_filter) { 646 case HWTSTAMP_FILTER_NONE: 647 /* time stamp no incoming packet at all */ 648 config.rx_filter = HWTSTAMP_FILTER_NONE; 649 break; 650 651 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 652 /* PTP v1, UDP, any kind of event packet */ 653 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 654 /* 'xmac' hardware can support Sync, Pdelay_Req and 655 * Pdelay_resp by setting bit14 and bits17/16 to 01 656 * This leaves Delay_Req timestamps out. 657 * Enable all events *and* general purpose message 658 * timestamping 659 */ 660 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 661 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 662 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 663 break; 664 665 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 666 /* PTP v1, UDP, Sync packet */ 667 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 668 /* take time stamp for SYNC messages only */ 669 ts_event_en = PTP_TCR_TSEVNTENA; 670 671 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 672 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 673 break; 674 675 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 676 /* PTP v1, UDP, Delay_req packet */ 677 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 678 /* take time stamp for Delay_Req messages only */ 679 ts_master_en = PTP_TCR_TSMSTRENA; 680 ts_event_en = PTP_TCR_TSEVNTENA; 681 682 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 683 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 684 break; 685 686 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 687 /* PTP v2, UDP, any kind of event packet */ 688 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 689 ptp_v2 = PTP_TCR_TSVER2ENA; 690 /* take time stamp for all event messages */ 691 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 692 693 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 694 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 695 break; 696 697 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 698 /* PTP v2, UDP, Sync packet */ 699 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 700 ptp_v2 = PTP_TCR_TSVER2ENA; 701 /* take time stamp for SYNC messages only */ 702 ts_event_en = PTP_TCR_TSEVNTENA; 703 704 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 705 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 706 break; 707 708 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 709 /* PTP v2, UDP, Delay_req packet */ 710 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 711 ptp_v2 = PTP_TCR_TSVER2ENA; 712 /* take time stamp for Delay_Req messages only */ 713 ts_master_en = PTP_TCR_TSMSTRENA; 714 ts_event_en = PTP_TCR_TSEVNTENA; 715 716 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 717 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 718 break; 719 720 case HWTSTAMP_FILTER_PTP_V2_EVENT: 721 /* PTP v2/802.AS1 any layer, any kind of event packet */ 722 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 723 ptp_v2 = PTP_TCR_TSVER2ENA; 724 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 725 if (priv->synopsys_id < DWMAC_CORE_4_10) 726 ts_event_en = PTP_TCR_TSEVNTENA; 727 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 728 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 729 ptp_over_ethernet = PTP_TCR_TSIPENA; 730 break; 731 732 case HWTSTAMP_FILTER_PTP_V2_SYNC: 733 /* PTP v2/802.AS1, any layer, Sync packet */ 734 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 735 ptp_v2 = PTP_TCR_TSVER2ENA; 736 /* take time stamp for SYNC messages only */ 737 ts_event_en = PTP_TCR_TSEVNTENA; 738 739 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 740 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 741 ptp_over_ethernet = PTP_TCR_TSIPENA; 742 break; 743 744 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 745 /* PTP v2/802.AS1, any layer, Delay_req packet */ 746 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 747 ptp_v2 = PTP_TCR_TSVER2ENA; 748 /* take time stamp for Delay_Req messages only */ 749 ts_master_en = PTP_TCR_TSMSTRENA; 750 ts_event_en = PTP_TCR_TSEVNTENA; 751 752 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 753 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 754 ptp_over_ethernet = PTP_TCR_TSIPENA; 755 break; 756 757 case HWTSTAMP_FILTER_NTP_ALL: 758 case HWTSTAMP_FILTER_ALL: 759 /* time stamp any incoming packet */ 760 config.rx_filter = HWTSTAMP_FILTER_ALL; 761 tstamp_all = PTP_TCR_TSENALL; 762 break; 763 764 default: 765 return -ERANGE; 766 } 767 } else { 768 switch (config.rx_filter) { 769 case HWTSTAMP_FILTER_NONE: 770 config.rx_filter = HWTSTAMP_FILTER_NONE; 771 break; 772 default: 773 /* PTP v1, UDP, any kind of event packet */ 774 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 775 break; 776 } 777 } 778 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 779 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 780 781 priv->systime_flags = STMMAC_HWTS_ACTIVE; 782 783 if (priv->hwts_tx_en || priv->hwts_rx_en) { 784 priv->systime_flags |= tstamp_all | ptp_v2 | 785 ptp_over_ethernet | ptp_over_ipv6_udp | 786 ptp_over_ipv4_udp | ts_event_en | 787 ts_master_en | snap_type_sel; 788 } 789 790 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 791 792 memcpy(&priv->tstamp_config, &config, sizeof(config)); 793 794 return copy_to_user(ifr->ifr_data, &config, 795 sizeof(config)) ? -EFAULT : 0; 796 } 797 798 /** 799 * stmmac_hwtstamp_get - read hardware timestamping. 800 * @dev: device pointer. 801 * @ifr: An IOCTL specific structure, that can contain a pointer to 802 * a proprietary structure used to pass information to the driver. 803 * Description: 804 * This function obtain the current hardware timestamping settings 805 * as requested. 806 */ 807 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 808 { 809 struct stmmac_priv *priv = netdev_priv(dev); 810 struct hwtstamp_config *config = &priv->tstamp_config; 811 812 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 813 return -EOPNOTSUPP; 814 815 return copy_to_user(ifr->ifr_data, config, 816 sizeof(*config)) ? -EFAULT : 0; 817 } 818 819 /** 820 * stmmac_init_tstamp_counter - init hardware timestamping counter 821 * @priv: driver private structure 822 * @systime_flags: timestamping flags 823 * Description: 824 * Initialize hardware counter for packet timestamping. 825 * This is valid as long as the interface is open and not suspended. 826 * Will be rerun after resuming from suspend, case in which the timestamping 827 * flags updated by stmmac_hwtstamp_set() also need to be restored. 828 */ 829 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) 830 { 831 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 832 struct timespec64 now; 833 u32 sec_inc = 0; 834 u64 temp = 0; 835 836 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 837 return -EOPNOTSUPP; 838 839 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 840 priv->systime_flags = systime_flags; 841 842 /* program Sub Second Increment reg */ 843 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 844 priv->plat->clk_ptp_rate, 845 xmac, &sec_inc); 846 temp = div_u64(1000000000ULL, sec_inc); 847 848 /* Store sub second increment for later use */ 849 priv->sub_second_inc = sec_inc; 850 851 /* calculate default added value: 852 * formula is : 853 * addend = (2^32)/freq_div_ratio; 854 * where, freq_div_ratio = 1e9ns/sec_inc 855 */ 856 temp = (u64)(temp << 32); 857 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 858 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 859 860 /* initialize system time */ 861 ktime_get_real_ts64(&now); 862 863 /* lower 32 bits of tv_sec are safe until y2106 */ 864 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 865 866 return 0; 867 } 868 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); 869 870 /** 871 * stmmac_init_ptp - init PTP 872 * @priv: driver private structure 873 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 874 * This is done by looking at the HW cap. register. 875 * This function also registers the ptp driver. 876 */ 877 static int stmmac_init_ptp(struct stmmac_priv *priv) 878 { 879 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 880 int ret; 881 882 if (priv->plat->ptp_clk_freq_config) 883 priv->plat->ptp_clk_freq_config(priv); 884 885 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); 886 if (ret) 887 return ret; 888 889 priv->adv_ts = 0; 890 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 891 if (xmac && priv->dma_cap.atime_stamp) 892 priv->adv_ts = 1; 893 /* Dwmac 3.x core with extend_desc can support adv_ts */ 894 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 895 priv->adv_ts = 1; 896 897 if (priv->dma_cap.time_stamp) 898 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 899 900 if (priv->adv_ts) 901 netdev_info(priv->dev, 902 "IEEE 1588-2008 Advanced Timestamp supported\n"); 903 904 priv->hwts_tx_en = 0; 905 priv->hwts_rx_en = 0; 906 907 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 908 stmmac_hwtstamp_correct_latency(priv, priv); 909 910 return 0; 911 } 912 913 static void stmmac_release_ptp(struct stmmac_priv *priv) 914 { 915 clk_disable_unprepare(priv->plat->clk_ptp_ref); 916 stmmac_ptp_unregister(priv); 917 } 918 919 /** 920 * stmmac_mac_flow_ctrl - Configure flow control in all queues 921 * @priv: driver private structure 922 * @duplex: duplex passed to the next function 923 * Description: It is used for configuring the flow control in all queues 924 */ 925 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 926 { 927 u32 tx_cnt = priv->plat->tx_queues_to_use; 928 929 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 930 priv->pause, tx_cnt); 931 } 932 933 static unsigned long stmmac_mac_get_caps(struct phylink_config *config, 934 phy_interface_t interface) 935 { 936 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 937 938 /* Refresh the MAC-specific capabilities */ 939 stmmac_mac_update_caps(priv); 940 941 config->mac_capabilities = priv->hw->link.caps; 942 943 if (priv->plat->max_speed) 944 phylink_limit_mac_speed(config, priv->plat->max_speed); 945 946 return config->mac_capabilities; 947 } 948 949 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 950 phy_interface_t interface) 951 { 952 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 953 struct phylink_pcs *pcs; 954 955 if (priv->plat->select_pcs) { 956 pcs = priv->plat->select_pcs(priv, interface); 957 if (!IS_ERR(pcs)) 958 return pcs; 959 } 960 961 return NULL; 962 } 963 964 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 965 const struct phylink_link_state *state) 966 { 967 /* Nothing to do, xpcs_config() handles everything */ 968 } 969 970 static void stmmac_mac_link_down(struct phylink_config *config, 971 unsigned int mode, phy_interface_t interface) 972 { 973 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 974 975 stmmac_mac_set(priv, priv->ioaddr, false); 976 priv->eee_active = false; 977 priv->tx_lpi_enabled = false; 978 priv->eee_enabled = stmmac_eee_init(priv); 979 stmmac_set_eee_pls(priv, priv->hw, false); 980 981 if (stmmac_fpe_supported(priv)) 982 stmmac_fpe_link_state_handle(priv, false); 983 } 984 985 static void stmmac_mac_link_up(struct phylink_config *config, 986 struct phy_device *phy, 987 unsigned int mode, phy_interface_t interface, 988 int speed, int duplex, 989 bool tx_pause, bool rx_pause) 990 { 991 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 992 u32 old_ctrl, ctrl; 993 994 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 995 priv->plat->serdes_powerup) 996 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 997 998 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 999 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1000 1001 if (interface == PHY_INTERFACE_MODE_USXGMII) { 1002 switch (speed) { 1003 case SPEED_10000: 1004 ctrl |= priv->hw->link.xgmii.speed10000; 1005 break; 1006 case SPEED_5000: 1007 ctrl |= priv->hw->link.xgmii.speed5000; 1008 break; 1009 case SPEED_2500: 1010 ctrl |= priv->hw->link.xgmii.speed2500; 1011 break; 1012 default: 1013 return; 1014 } 1015 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1016 switch (speed) { 1017 case SPEED_100000: 1018 ctrl |= priv->hw->link.xlgmii.speed100000; 1019 break; 1020 case SPEED_50000: 1021 ctrl |= priv->hw->link.xlgmii.speed50000; 1022 break; 1023 case SPEED_40000: 1024 ctrl |= priv->hw->link.xlgmii.speed40000; 1025 break; 1026 case SPEED_25000: 1027 ctrl |= priv->hw->link.xlgmii.speed25000; 1028 break; 1029 case SPEED_10000: 1030 ctrl |= priv->hw->link.xgmii.speed10000; 1031 break; 1032 case SPEED_2500: 1033 ctrl |= priv->hw->link.speed2500; 1034 break; 1035 case SPEED_1000: 1036 ctrl |= priv->hw->link.speed1000; 1037 break; 1038 default: 1039 return; 1040 } 1041 } else { 1042 switch (speed) { 1043 case SPEED_2500: 1044 ctrl |= priv->hw->link.speed2500; 1045 break; 1046 case SPEED_1000: 1047 ctrl |= priv->hw->link.speed1000; 1048 break; 1049 case SPEED_100: 1050 ctrl |= priv->hw->link.speed100; 1051 break; 1052 case SPEED_10: 1053 ctrl |= priv->hw->link.speed10; 1054 break; 1055 default: 1056 return; 1057 } 1058 } 1059 1060 priv->speed = speed; 1061 1062 if (priv->plat->fix_mac_speed) 1063 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); 1064 1065 if (!duplex) 1066 ctrl &= ~priv->hw->link.duplex; 1067 else 1068 ctrl |= priv->hw->link.duplex; 1069 1070 /* Flow Control operation */ 1071 if (rx_pause && tx_pause) 1072 priv->flow_ctrl = FLOW_AUTO; 1073 else if (rx_pause && !tx_pause) 1074 priv->flow_ctrl = FLOW_RX; 1075 else if (!rx_pause && tx_pause) 1076 priv->flow_ctrl = FLOW_TX; 1077 else 1078 priv->flow_ctrl = FLOW_OFF; 1079 1080 stmmac_mac_flow_ctrl(priv, duplex); 1081 1082 if (ctrl != old_ctrl) 1083 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1084 1085 stmmac_mac_set(priv, priv->ioaddr, true); 1086 if (phy && priv->dma_cap.eee) { 1087 priv->eee_active = 1088 phy_init_eee(phy, !(priv->plat->flags & 1089 STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) >= 0; 1090 priv->eee_enabled = stmmac_eee_init(priv); 1091 priv->tx_lpi_enabled = priv->eee_enabled; 1092 stmmac_set_eee_pls(priv, priv->hw, true); 1093 } 1094 1095 if (stmmac_fpe_supported(priv)) 1096 stmmac_fpe_link_state_handle(priv, true); 1097 1098 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1099 stmmac_hwtstamp_correct_latency(priv, priv); 1100 } 1101 1102 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1103 .mac_get_caps = stmmac_mac_get_caps, 1104 .mac_select_pcs = stmmac_mac_select_pcs, 1105 .mac_config = stmmac_mac_config, 1106 .mac_link_down = stmmac_mac_link_down, 1107 .mac_link_up = stmmac_mac_link_up, 1108 }; 1109 1110 /** 1111 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1112 * @priv: driver private structure 1113 * Description: this is to verify if the HW supports the PCS. 1114 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1115 * configured for the TBI, RTBI, or SGMII PHY interface. 1116 */ 1117 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1118 { 1119 int interface = priv->plat->mac_interface; 1120 1121 if (priv->dma_cap.pcs) { 1122 if ((interface == PHY_INTERFACE_MODE_RGMII) || 1123 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 1124 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 1125 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 1126 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 1127 priv->hw->pcs = STMMAC_PCS_RGMII; 1128 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 1129 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1130 priv->hw->pcs = STMMAC_PCS_SGMII; 1131 } 1132 } 1133 } 1134 1135 /** 1136 * stmmac_init_phy - PHY initialization 1137 * @dev: net device structure 1138 * Description: it initializes the driver's PHY state, and attaches the PHY 1139 * to the mac driver. 1140 * Return value: 1141 * 0 on success 1142 */ 1143 static int stmmac_init_phy(struct net_device *dev) 1144 { 1145 struct stmmac_priv *priv = netdev_priv(dev); 1146 struct fwnode_handle *phy_fwnode; 1147 struct fwnode_handle *fwnode; 1148 int ret; 1149 1150 if (!phylink_expects_phy(priv->phylink)) 1151 return 0; 1152 1153 fwnode = priv->plat->port_node; 1154 if (!fwnode) 1155 fwnode = dev_fwnode(priv->device); 1156 1157 if (fwnode) 1158 phy_fwnode = fwnode_get_phy_node(fwnode); 1159 else 1160 phy_fwnode = NULL; 1161 1162 /* Some DT bindings do not set-up the PHY handle. Let's try to 1163 * manually parse it 1164 */ 1165 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1166 int addr = priv->plat->phy_addr; 1167 struct phy_device *phydev; 1168 1169 if (addr < 0) { 1170 netdev_err(priv->dev, "no phy found\n"); 1171 return -ENODEV; 1172 } 1173 1174 phydev = mdiobus_get_phy(priv->mii, addr); 1175 if (!phydev) { 1176 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1177 return -ENODEV; 1178 } 1179 1180 if (priv->dma_cap.eee) 1181 phy_support_eee(phydev); 1182 1183 ret = phylink_connect_phy(priv->phylink, phydev); 1184 } else { 1185 fwnode_handle_put(phy_fwnode); 1186 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); 1187 } 1188 1189 if (!priv->plat->pmt) { 1190 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 1191 1192 phylink_ethtool_get_wol(priv->phylink, &wol); 1193 device_set_wakeup_capable(priv->device, !!wol.supported); 1194 device_set_wakeup_enable(priv->device, !!wol.wolopts); 1195 } 1196 1197 return ret; 1198 } 1199 1200 static int stmmac_phy_setup(struct stmmac_priv *priv) 1201 { 1202 struct stmmac_mdio_bus_data *mdio_bus_data; 1203 int mode = priv->plat->phy_interface; 1204 struct fwnode_handle *fwnode; 1205 struct phylink *phylink; 1206 1207 priv->phylink_config.dev = &priv->dev->dev; 1208 priv->phylink_config.type = PHYLINK_NETDEV; 1209 priv->phylink_config.mac_managed_pm = true; 1210 1211 /* Stmmac always requires an RX clock for hardware initialization */ 1212 priv->phylink_config.mac_requires_rxc = true; 1213 1214 mdio_bus_data = priv->plat->mdio_bus_data; 1215 if (mdio_bus_data) 1216 priv->phylink_config.default_an_inband = 1217 mdio_bus_data->default_an_inband; 1218 1219 /* Set the platform/firmware specified interface mode. Note, phylink 1220 * deals with the PHY interface mode, not the MAC interface mode. 1221 */ 1222 __set_bit(mode, priv->phylink_config.supported_interfaces); 1223 1224 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1225 if (priv->hw->xpcs) 1226 xpcs_get_interfaces(priv->hw->xpcs, 1227 priv->phylink_config.supported_interfaces); 1228 1229 fwnode = priv->plat->port_node; 1230 if (!fwnode) 1231 fwnode = dev_fwnode(priv->device); 1232 1233 phylink = phylink_create(&priv->phylink_config, fwnode, 1234 mode, &stmmac_phylink_mac_ops); 1235 if (IS_ERR(phylink)) 1236 return PTR_ERR(phylink); 1237 1238 priv->phylink = phylink; 1239 return 0; 1240 } 1241 1242 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1243 struct stmmac_dma_conf *dma_conf) 1244 { 1245 u32 rx_cnt = priv->plat->rx_queues_to_use; 1246 unsigned int desc_size; 1247 void *head_rx; 1248 u32 queue; 1249 1250 /* Display RX rings */ 1251 for (queue = 0; queue < rx_cnt; queue++) { 1252 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1253 1254 pr_info("\tRX Queue %u rings\n", queue); 1255 1256 if (priv->extend_desc) { 1257 head_rx = (void *)rx_q->dma_erx; 1258 desc_size = sizeof(struct dma_extended_desc); 1259 } else { 1260 head_rx = (void *)rx_q->dma_rx; 1261 desc_size = sizeof(struct dma_desc); 1262 } 1263 1264 /* Display RX ring */ 1265 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1266 rx_q->dma_rx_phy, desc_size); 1267 } 1268 } 1269 1270 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1271 struct stmmac_dma_conf *dma_conf) 1272 { 1273 u32 tx_cnt = priv->plat->tx_queues_to_use; 1274 unsigned int desc_size; 1275 void *head_tx; 1276 u32 queue; 1277 1278 /* Display TX rings */ 1279 for (queue = 0; queue < tx_cnt; queue++) { 1280 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1281 1282 pr_info("\tTX Queue %d rings\n", queue); 1283 1284 if (priv->extend_desc) { 1285 head_tx = (void *)tx_q->dma_etx; 1286 desc_size = sizeof(struct dma_extended_desc); 1287 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1288 head_tx = (void *)tx_q->dma_entx; 1289 desc_size = sizeof(struct dma_edesc); 1290 } else { 1291 head_tx = (void *)tx_q->dma_tx; 1292 desc_size = sizeof(struct dma_desc); 1293 } 1294 1295 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1296 tx_q->dma_tx_phy, desc_size); 1297 } 1298 } 1299 1300 static void stmmac_display_rings(struct stmmac_priv *priv, 1301 struct stmmac_dma_conf *dma_conf) 1302 { 1303 /* Display RX ring */ 1304 stmmac_display_rx_rings(priv, dma_conf); 1305 1306 /* Display TX ring */ 1307 stmmac_display_tx_rings(priv, dma_conf); 1308 } 1309 1310 static int stmmac_set_bfsize(int mtu, int bufsize) 1311 { 1312 int ret = bufsize; 1313 1314 if (mtu >= BUF_SIZE_8KiB) 1315 ret = BUF_SIZE_16KiB; 1316 else if (mtu >= BUF_SIZE_4KiB) 1317 ret = BUF_SIZE_8KiB; 1318 else if (mtu >= BUF_SIZE_2KiB) 1319 ret = BUF_SIZE_4KiB; 1320 else if (mtu > DEFAULT_BUFSIZE) 1321 ret = BUF_SIZE_2KiB; 1322 else 1323 ret = DEFAULT_BUFSIZE; 1324 1325 return ret; 1326 } 1327 1328 /** 1329 * stmmac_clear_rx_descriptors - clear RX descriptors 1330 * @priv: driver private structure 1331 * @dma_conf: structure to take the dma data 1332 * @queue: RX queue index 1333 * Description: this function is called to clear the RX descriptors 1334 * in case of both basic and extended descriptors are used. 1335 */ 1336 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1337 struct stmmac_dma_conf *dma_conf, 1338 u32 queue) 1339 { 1340 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1341 int i; 1342 1343 /* Clear the RX descriptors */ 1344 for (i = 0; i < dma_conf->dma_rx_size; i++) 1345 if (priv->extend_desc) 1346 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1347 priv->use_riwt, priv->mode, 1348 (i == dma_conf->dma_rx_size - 1), 1349 dma_conf->dma_buf_sz); 1350 else 1351 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1352 priv->use_riwt, priv->mode, 1353 (i == dma_conf->dma_rx_size - 1), 1354 dma_conf->dma_buf_sz); 1355 } 1356 1357 /** 1358 * stmmac_clear_tx_descriptors - clear tx descriptors 1359 * @priv: driver private structure 1360 * @dma_conf: structure to take the dma data 1361 * @queue: TX queue index. 1362 * Description: this function is called to clear the TX descriptors 1363 * in case of both basic and extended descriptors are used. 1364 */ 1365 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1366 struct stmmac_dma_conf *dma_conf, 1367 u32 queue) 1368 { 1369 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1370 int i; 1371 1372 /* Clear the TX descriptors */ 1373 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1374 int last = (i == (dma_conf->dma_tx_size - 1)); 1375 struct dma_desc *p; 1376 1377 if (priv->extend_desc) 1378 p = &tx_q->dma_etx[i].basic; 1379 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1380 p = &tx_q->dma_entx[i].basic; 1381 else 1382 p = &tx_q->dma_tx[i]; 1383 1384 stmmac_init_tx_desc(priv, p, priv->mode, last); 1385 } 1386 } 1387 1388 /** 1389 * stmmac_clear_descriptors - clear descriptors 1390 * @priv: driver private structure 1391 * @dma_conf: structure to take the dma data 1392 * Description: this function is called to clear the TX and RX descriptors 1393 * in case of both basic and extended descriptors are used. 1394 */ 1395 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1396 struct stmmac_dma_conf *dma_conf) 1397 { 1398 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1399 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1400 u32 queue; 1401 1402 /* Clear the RX descriptors */ 1403 for (queue = 0; queue < rx_queue_cnt; queue++) 1404 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1405 1406 /* Clear the TX descriptors */ 1407 for (queue = 0; queue < tx_queue_cnt; queue++) 1408 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1409 } 1410 1411 /** 1412 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1413 * @priv: driver private structure 1414 * @dma_conf: structure to take the dma data 1415 * @p: descriptor pointer 1416 * @i: descriptor index 1417 * @flags: gfp flag 1418 * @queue: RX queue index 1419 * Description: this function is called to allocate a receive buffer, perform 1420 * the DMA mapping and init the descriptor. 1421 */ 1422 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1423 struct stmmac_dma_conf *dma_conf, 1424 struct dma_desc *p, 1425 int i, gfp_t flags, u32 queue) 1426 { 1427 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1428 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1429 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1430 1431 if (priv->dma_cap.host_dma_width <= 32) 1432 gfp |= GFP_DMA32; 1433 1434 if (!buf->page) { 1435 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1436 if (!buf->page) 1437 return -ENOMEM; 1438 buf->page_offset = stmmac_rx_offset(priv); 1439 } 1440 1441 if (priv->sph && !buf->sec_page) { 1442 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1443 if (!buf->sec_page) 1444 return -ENOMEM; 1445 1446 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1447 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1448 } else { 1449 buf->sec_page = NULL; 1450 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1451 } 1452 1453 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1454 1455 stmmac_set_desc_addr(priv, p, buf->addr); 1456 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1457 stmmac_init_desc3(priv, p); 1458 1459 return 0; 1460 } 1461 1462 /** 1463 * stmmac_free_rx_buffer - free RX dma buffers 1464 * @priv: private structure 1465 * @rx_q: RX queue 1466 * @i: buffer index. 1467 */ 1468 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1469 struct stmmac_rx_queue *rx_q, 1470 int i) 1471 { 1472 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1473 1474 if (buf->page) 1475 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1476 buf->page = NULL; 1477 1478 if (buf->sec_page) 1479 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1480 buf->sec_page = NULL; 1481 } 1482 1483 /** 1484 * stmmac_free_tx_buffer - free RX dma buffers 1485 * @priv: private structure 1486 * @dma_conf: structure to take the dma data 1487 * @queue: RX queue index 1488 * @i: buffer index. 1489 */ 1490 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1491 struct stmmac_dma_conf *dma_conf, 1492 u32 queue, int i) 1493 { 1494 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1495 1496 if (tx_q->tx_skbuff_dma[i].buf && 1497 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1498 if (tx_q->tx_skbuff_dma[i].map_as_page) 1499 dma_unmap_page(priv->device, 1500 tx_q->tx_skbuff_dma[i].buf, 1501 tx_q->tx_skbuff_dma[i].len, 1502 DMA_TO_DEVICE); 1503 else 1504 dma_unmap_single(priv->device, 1505 tx_q->tx_skbuff_dma[i].buf, 1506 tx_q->tx_skbuff_dma[i].len, 1507 DMA_TO_DEVICE); 1508 } 1509 1510 if (tx_q->xdpf[i] && 1511 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1512 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1513 xdp_return_frame(tx_q->xdpf[i]); 1514 tx_q->xdpf[i] = NULL; 1515 } 1516 1517 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1518 tx_q->xsk_frames_done++; 1519 1520 if (tx_q->tx_skbuff[i] && 1521 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1522 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1523 tx_q->tx_skbuff[i] = NULL; 1524 } 1525 1526 tx_q->tx_skbuff_dma[i].buf = 0; 1527 tx_q->tx_skbuff_dma[i].map_as_page = false; 1528 } 1529 1530 /** 1531 * dma_free_rx_skbufs - free RX dma buffers 1532 * @priv: private structure 1533 * @dma_conf: structure to take the dma data 1534 * @queue: RX queue index 1535 */ 1536 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1537 struct stmmac_dma_conf *dma_conf, 1538 u32 queue) 1539 { 1540 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1541 int i; 1542 1543 for (i = 0; i < dma_conf->dma_rx_size; i++) 1544 stmmac_free_rx_buffer(priv, rx_q, i); 1545 } 1546 1547 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1548 struct stmmac_dma_conf *dma_conf, 1549 u32 queue, gfp_t flags) 1550 { 1551 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1552 int i; 1553 1554 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1555 struct dma_desc *p; 1556 int ret; 1557 1558 if (priv->extend_desc) 1559 p = &((rx_q->dma_erx + i)->basic); 1560 else 1561 p = rx_q->dma_rx + i; 1562 1563 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1564 queue); 1565 if (ret) 1566 return ret; 1567 1568 rx_q->buf_alloc_num++; 1569 } 1570 1571 return 0; 1572 } 1573 1574 /** 1575 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1576 * @priv: private structure 1577 * @dma_conf: structure to take the dma data 1578 * @queue: RX queue index 1579 */ 1580 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1581 struct stmmac_dma_conf *dma_conf, 1582 u32 queue) 1583 { 1584 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1585 int i; 1586 1587 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1588 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1589 1590 if (!buf->xdp) 1591 continue; 1592 1593 xsk_buff_free(buf->xdp); 1594 buf->xdp = NULL; 1595 } 1596 } 1597 1598 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1599 struct stmmac_dma_conf *dma_conf, 1600 u32 queue) 1601 { 1602 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1603 int i; 1604 1605 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1606 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1607 * use this macro to make sure no size violations. 1608 */ 1609 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1610 1611 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1612 struct stmmac_rx_buffer *buf; 1613 dma_addr_t dma_addr; 1614 struct dma_desc *p; 1615 1616 if (priv->extend_desc) 1617 p = (struct dma_desc *)(rx_q->dma_erx + i); 1618 else 1619 p = rx_q->dma_rx + i; 1620 1621 buf = &rx_q->buf_pool[i]; 1622 1623 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1624 if (!buf->xdp) 1625 return -ENOMEM; 1626 1627 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1628 stmmac_set_desc_addr(priv, p, dma_addr); 1629 rx_q->buf_alloc_num++; 1630 } 1631 1632 return 0; 1633 } 1634 1635 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1636 { 1637 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1638 return NULL; 1639 1640 return xsk_get_pool_from_qid(priv->dev, queue); 1641 } 1642 1643 /** 1644 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1645 * @priv: driver private structure 1646 * @dma_conf: structure to take the dma data 1647 * @queue: RX queue index 1648 * @flags: gfp flag. 1649 * Description: this function initializes the DMA RX descriptors 1650 * and allocates the socket buffers. It supports the chained and ring 1651 * modes. 1652 */ 1653 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1654 struct stmmac_dma_conf *dma_conf, 1655 u32 queue, gfp_t flags) 1656 { 1657 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1658 int ret; 1659 1660 netif_dbg(priv, probe, priv->dev, 1661 "(%s) dma_rx_phy=0x%08x\n", __func__, 1662 (u32)rx_q->dma_rx_phy); 1663 1664 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1665 1666 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1667 1668 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1669 1670 if (rx_q->xsk_pool) { 1671 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1672 MEM_TYPE_XSK_BUFF_POOL, 1673 NULL)); 1674 netdev_info(priv->dev, 1675 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1676 rx_q->queue_index); 1677 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1678 } else { 1679 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1680 MEM_TYPE_PAGE_POOL, 1681 rx_q->page_pool)); 1682 netdev_info(priv->dev, 1683 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1684 rx_q->queue_index); 1685 } 1686 1687 if (rx_q->xsk_pool) { 1688 /* RX XDP ZC buffer pool may not be populated, e.g. 1689 * xdpsock TX-only. 1690 */ 1691 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1692 } else { 1693 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1694 if (ret < 0) 1695 return -ENOMEM; 1696 } 1697 1698 /* Setup the chained descriptor addresses */ 1699 if (priv->mode == STMMAC_CHAIN_MODE) { 1700 if (priv->extend_desc) 1701 stmmac_mode_init(priv, rx_q->dma_erx, 1702 rx_q->dma_rx_phy, 1703 dma_conf->dma_rx_size, 1); 1704 else 1705 stmmac_mode_init(priv, rx_q->dma_rx, 1706 rx_q->dma_rx_phy, 1707 dma_conf->dma_rx_size, 0); 1708 } 1709 1710 return 0; 1711 } 1712 1713 static int init_dma_rx_desc_rings(struct net_device *dev, 1714 struct stmmac_dma_conf *dma_conf, 1715 gfp_t flags) 1716 { 1717 struct stmmac_priv *priv = netdev_priv(dev); 1718 u32 rx_count = priv->plat->rx_queues_to_use; 1719 int queue; 1720 int ret; 1721 1722 /* RX INITIALIZATION */ 1723 netif_dbg(priv, probe, priv->dev, 1724 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1725 1726 for (queue = 0; queue < rx_count; queue++) { 1727 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1728 if (ret) 1729 goto err_init_rx_buffers; 1730 } 1731 1732 return 0; 1733 1734 err_init_rx_buffers: 1735 while (queue >= 0) { 1736 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1737 1738 if (rx_q->xsk_pool) 1739 dma_free_rx_xskbufs(priv, dma_conf, queue); 1740 else 1741 dma_free_rx_skbufs(priv, dma_conf, queue); 1742 1743 rx_q->buf_alloc_num = 0; 1744 rx_q->xsk_pool = NULL; 1745 1746 queue--; 1747 } 1748 1749 return ret; 1750 } 1751 1752 /** 1753 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1754 * @priv: driver private structure 1755 * @dma_conf: structure to take the dma data 1756 * @queue: TX queue index 1757 * Description: this function initializes the DMA TX descriptors 1758 * and allocates the socket buffers. It supports the chained and ring 1759 * modes. 1760 */ 1761 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 1762 struct stmmac_dma_conf *dma_conf, 1763 u32 queue) 1764 { 1765 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1766 int i; 1767 1768 netif_dbg(priv, probe, priv->dev, 1769 "(%s) dma_tx_phy=0x%08x\n", __func__, 1770 (u32)tx_q->dma_tx_phy); 1771 1772 /* Setup the chained descriptor addresses */ 1773 if (priv->mode == STMMAC_CHAIN_MODE) { 1774 if (priv->extend_desc) 1775 stmmac_mode_init(priv, tx_q->dma_etx, 1776 tx_q->dma_tx_phy, 1777 dma_conf->dma_tx_size, 1); 1778 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1779 stmmac_mode_init(priv, tx_q->dma_tx, 1780 tx_q->dma_tx_phy, 1781 dma_conf->dma_tx_size, 0); 1782 } 1783 1784 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1785 1786 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1787 struct dma_desc *p; 1788 1789 if (priv->extend_desc) 1790 p = &((tx_q->dma_etx + i)->basic); 1791 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1792 p = &((tx_q->dma_entx + i)->basic); 1793 else 1794 p = tx_q->dma_tx + i; 1795 1796 stmmac_clear_desc(priv, p); 1797 1798 tx_q->tx_skbuff_dma[i].buf = 0; 1799 tx_q->tx_skbuff_dma[i].map_as_page = false; 1800 tx_q->tx_skbuff_dma[i].len = 0; 1801 tx_q->tx_skbuff_dma[i].last_segment = false; 1802 tx_q->tx_skbuff[i] = NULL; 1803 } 1804 1805 return 0; 1806 } 1807 1808 static int init_dma_tx_desc_rings(struct net_device *dev, 1809 struct stmmac_dma_conf *dma_conf) 1810 { 1811 struct stmmac_priv *priv = netdev_priv(dev); 1812 u32 tx_queue_cnt; 1813 u32 queue; 1814 1815 tx_queue_cnt = priv->plat->tx_queues_to_use; 1816 1817 for (queue = 0; queue < tx_queue_cnt; queue++) 1818 __init_dma_tx_desc_rings(priv, dma_conf, queue); 1819 1820 return 0; 1821 } 1822 1823 /** 1824 * init_dma_desc_rings - init the RX/TX descriptor rings 1825 * @dev: net device structure 1826 * @dma_conf: structure to take the dma data 1827 * @flags: gfp flag. 1828 * Description: this function initializes the DMA RX/TX descriptors 1829 * and allocates the socket buffers. It supports the chained and ring 1830 * modes. 1831 */ 1832 static int init_dma_desc_rings(struct net_device *dev, 1833 struct stmmac_dma_conf *dma_conf, 1834 gfp_t flags) 1835 { 1836 struct stmmac_priv *priv = netdev_priv(dev); 1837 int ret; 1838 1839 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 1840 if (ret) 1841 return ret; 1842 1843 ret = init_dma_tx_desc_rings(dev, dma_conf); 1844 1845 stmmac_clear_descriptors(priv, dma_conf); 1846 1847 if (netif_msg_hw(priv)) 1848 stmmac_display_rings(priv, dma_conf); 1849 1850 return ret; 1851 } 1852 1853 /** 1854 * dma_free_tx_skbufs - free TX dma buffers 1855 * @priv: private structure 1856 * @dma_conf: structure to take the dma data 1857 * @queue: TX queue index 1858 */ 1859 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 1860 struct stmmac_dma_conf *dma_conf, 1861 u32 queue) 1862 { 1863 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1864 int i; 1865 1866 tx_q->xsk_frames_done = 0; 1867 1868 for (i = 0; i < dma_conf->dma_tx_size; i++) 1869 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 1870 1871 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 1872 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 1873 tx_q->xsk_frames_done = 0; 1874 tx_q->xsk_pool = NULL; 1875 } 1876 } 1877 1878 /** 1879 * stmmac_free_tx_skbufs - free TX skb buffers 1880 * @priv: private structure 1881 */ 1882 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 1883 { 1884 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1885 u32 queue; 1886 1887 for (queue = 0; queue < tx_queue_cnt; queue++) 1888 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 1889 } 1890 1891 /** 1892 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 1893 * @priv: private structure 1894 * @dma_conf: structure to take the dma data 1895 * @queue: RX queue index 1896 */ 1897 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 1898 struct stmmac_dma_conf *dma_conf, 1899 u32 queue) 1900 { 1901 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1902 1903 /* Release the DMA RX socket buffers */ 1904 if (rx_q->xsk_pool) 1905 dma_free_rx_xskbufs(priv, dma_conf, queue); 1906 else 1907 dma_free_rx_skbufs(priv, dma_conf, queue); 1908 1909 rx_q->buf_alloc_num = 0; 1910 rx_q->xsk_pool = NULL; 1911 1912 /* Free DMA regions of consistent memory previously allocated */ 1913 if (!priv->extend_desc) 1914 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1915 sizeof(struct dma_desc), 1916 rx_q->dma_rx, rx_q->dma_rx_phy); 1917 else 1918 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1919 sizeof(struct dma_extended_desc), 1920 rx_q->dma_erx, rx_q->dma_rx_phy); 1921 1922 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 1923 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 1924 1925 kfree(rx_q->buf_pool); 1926 if (rx_q->page_pool) 1927 page_pool_destroy(rx_q->page_pool); 1928 } 1929 1930 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 1931 struct stmmac_dma_conf *dma_conf) 1932 { 1933 u32 rx_count = priv->plat->rx_queues_to_use; 1934 u32 queue; 1935 1936 /* Free RX queue resources */ 1937 for (queue = 0; queue < rx_count; queue++) 1938 __free_dma_rx_desc_resources(priv, dma_conf, queue); 1939 } 1940 1941 /** 1942 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 1943 * @priv: private structure 1944 * @dma_conf: structure to take the dma data 1945 * @queue: TX queue index 1946 */ 1947 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 1948 struct stmmac_dma_conf *dma_conf, 1949 u32 queue) 1950 { 1951 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1952 size_t size; 1953 void *addr; 1954 1955 /* Release the DMA TX socket buffers */ 1956 dma_free_tx_skbufs(priv, dma_conf, queue); 1957 1958 if (priv->extend_desc) { 1959 size = sizeof(struct dma_extended_desc); 1960 addr = tx_q->dma_etx; 1961 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1962 size = sizeof(struct dma_edesc); 1963 addr = tx_q->dma_entx; 1964 } else { 1965 size = sizeof(struct dma_desc); 1966 addr = tx_q->dma_tx; 1967 } 1968 1969 size *= dma_conf->dma_tx_size; 1970 1971 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 1972 1973 kfree(tx_q->tx_skbuff_dma); 1974 kfree(tx_q->tx_skbuff); 1975 } 1976 1977 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 1978 struct stmmac_dma_conf *dma_conf) 1979 { 1980 u32 tx_count = priv->plat->tx_queues_to_use; 1981 u32 queue; 1982 1983 /* Free TX queue resources */ 1984 for (queue = 0; queue < tx_count; queue++) 1985 __free_dma_tx_desc_resources(priv, dma_conf, queue); 1986 } 1987 1988 /** 1989 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 1990 * @priv: private structure 1991 * @dma_conf: structure to take the dma data 1992 * @queue: RX queue index 1993 * Description: according to which descriptor can be used (extend or basic) 1994 * this function allocates the resources for TX and RX paths. In case of 1995 * reception, for example, it pre-allocated the RX socket buffer in order to 1996 * allow zero-copy mechanism. 1997 */ 1998 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 1999 struct stmmac_dma_conf *dma_conf, 2000 u32 queue) 2001 { 2002 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2003 struct stmmac_channel *ch = &priv->channel[queue]; 2004 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2005 struct page_pool_params pp_params = { 0 }; 2006 unsigned int num_pages; 2007 unsigned int napi_id; 2008 int ret; 2009 2010 rx_q->queue_index = queue; 2011 rx_q->priv_data = priv; 2012 2013 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2014 pp_params.pool_size = dma_conf->dma_rx_size; 2015 num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); 2016 pp_params.order = ilog2(num_pages); 2017 pp_params.nid = dev_to_node(priv->device); 2018 pp_params.dev = priv->device; 2019 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2020 pp_params.offset = stmmac_rx_offset(priv); 2021 pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages); 2022 2023 rx_q->page_pool = page_pool_create(&pp_params); 2024 if (IS_ERR(rx_q->page_pool)) { 2025 ret = PTR_ERR(rx_q->page_pool); 2026 rx_q->page_pool = NULL; 2027 return ret; 2028 } 2029 2030 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, 2031 sizeof(*rx_q->buf_pool), 2032 GFP_KERNEL); 2033 if (!rx_q->buf_pool) 2034 return -ENOMEM; 2035 2036 if (priv->extend_desc) { 2037 rx_q->dma_erx = dma_alloc_coherent(priv->device, 2038 dma_conf->dma_rx_size * 2039 sizeof(struct dma_extended_desc), 2040 &rx_q->dma_rx_phy, 2041 GFP_KERNEL); 2042 if (!rx_q->dma_erx) 2043 return -ENOMEM; 2044 2045 } else { 2046 rx_q->dma_rx = dma_alloc_coherent(priv->device, 2047 dma_conf->dma_rx_size * 2048 sizeof(struct dma_desc), 2049 &rx_q->dma_rx_phy, 2050 GFP_KERNEL); 2051 if (!rx_q->dma_rx) 2052 return -ENOMEM; 2053 } 2054 2055 if (stmmac_xdp_is_enabled(priv) && 2056 test_bit(queue, priv->af_xdp_zc_qps)) 2057 napi_id = ch->rxtx_napi.napi_id; 2058 else 2059 napi_id = ch->rx_napi.napi_id; 2060 2061 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 2062 rx_q->queue_index, 2063 napi_id); 2064 if (ret) { 2065 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2066 return -EINVAL; 2067 } 2068 2069 return 0; 2070 } 2071 2072 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2073 struct stmmac_dma_conf *dma_conf) 2074 { 2075 u32 rx_count = priv->plat->rx_queues_to_use; 2076 u32 queue; 2077 int ret; 2078 2079 /* RX queues buffers and DMA */ 2080 for (queue = 0; queue < rx_count; queue++) { 2081 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2082 if (ret) 2083 goto err_dma; 2084 } 2085 2086 return 0; 2087 2088 err_dma: 2089 free_dma_rx_desc_resources(priv, dma_conf); 2090 2091 return ret; 2092 } 2093 2094 /** 2095 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2096 * @priv: private structure 2097 * @dma_conf: structure to take the dma data 2098 * @queue: TX queue index 2099 * Description: according to which descriptor can be used (extend or basic) 2100 * this function allocates the resources for TX and RX paths. In case of 2101 * reception, for example, it pre-allocated the RX socket buffer in order to 2102 * allow zero-copy mechanism. 2103 */ 2104 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2105 struct stmmac_dma_conf *dma_conf, 2106 u32 queue) 2107 { 2108 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2109 size_t size; 2110 void *addr; 2111 2112 tx_q->queue_index = queue; 2113 tx_q->priv_data = priv; 2114 2115 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, 2116 sizeof(*tx_q->tx_skbuff_dma), 2117 GFP_KERNEL); 2118 if (!tx_q->tx_skbuff_dma) 2119 return -ENOMEM; 2120 2121 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, 2122 sizeof(struct sk_buff *), 2123 GFP_KERNEL); 2124 if (!tx_q->tx_skbuff) 2125 return -ENOMEM; 2126 2127 if (priv->extend_desc) 2128 size = sizeof(struct dma_extended_desc); 2129 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2130 size = sizeof(struct dma_edesc); 2131 else 2132 size = sizeof(struct dma_desc); 2133 2134 size *= dma_conf->dma_tx_size; 2135 2136 addr = dma_alloc_coherent(priv->device, size, 2137 &tx_q->dma_tx_phy, GFP_KERNEL); 2138 if (!addr) 2139 return -ENOMEM; 2140 2141 if (priv->extend_desc) 2142 tx_q->dma_etx = addr; 2143 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2144 tx_q->dma_entx = addr; 2145 else 2146 tx_q->dma_tx = addr; 2147 2148 return 0; 2149 } 2150 2151 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2152 struct stmmac_dma_conf *dma_conf) 2153 { 2154 u32 tx_count = priv->plat->tx_queues_to_use; 2155 u32 queue; 2156 int ret; 2157 2158 /* TX queues buffers and DMA */ 2159 for (queue = 0; queue < tx_count; queue++) { 2160 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2161 if (ret) 2162 goto err_dma; 2163 } 2164 2165 return 0; 2166 2167 err_dma: 2168 free_dma_tx_desc_resources(priv, dma_conf); 2169 return ret; 2170 } 2171 2172 /** 2173 * alloc_dma_desc_resources - alloc TX/RX resources. 2174 * @priv: private structure 2175 * @dma_conf: structure to take the dma data 2176 * Description: according to which descriptor can be used (extend or basic) 2177 * this function allocates the resources for TX and RX paths. In case of 2178 * reception, for example, it pre-allocated the RX socket buffer in order to 2179 * allow zero-copy mechanism. 2180 */ 2181 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2182 struct stmmac_dma_conf *dma_conf) 2183 { 2184 /* RX Allocation */ 2185 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2186 2187 if (ret) 2188 return ret; 2189 2190 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2191 2192 return ret; 2193 } 2194 2195 /** 2196 * free_dma_desc_resources - free dma desc resources 2197 * @priv: private structure 2198 * @dma_conf: structure to take the dma data 2199 */ 2200 static void free_dma_desc_resources(struct stmmac_priv *priv, 2201 struct stmmac_dma_conf *dma_conf) 2202 { 2203 /* Release the DMA TX socket buffers */ 2204 free_dma_tx_desc_resources(priv, dma_conf); 2205 2206 /* Release the DMA RX socket buffers later 2207 * to ensure all pending XDP_TX buffers are returned. 2208 */ 2209 free_dma_rx_desc_resources(priv, dma_conf); 2210 } 2211 2212 /** 2213 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2214 * @priv: driver private structure 2215 * Description: It is used for enabling the rx queues in the MAC 2216 */ 2217 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2218 { 2219 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2220 int queue; 2221 u8 mode; 2222 2223 for (queue = 0; queue < rx_queues_count; queue++) { 2224 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2225 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2226 } 2227 } 2228 2229 /** 2230 * stmmac_start_rx_dma - start RX DMA channel 2231 * @priv: driver private structure 2232 * @chan: RX channel index 2233 * Description: 2234 * This starts a RX DMA channel 2235 */ 2236 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2237 { 2238 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2239 stmmac_start_rx(priv, priv->ioaddr, chan); 2240 } 2241 2242 /** 2243 * stmmac_start_tx_dma - start TX DMA channel 2244 * @priv: driver private structure 2245 * @chan: TX channel index 2246 * Description: 2247 * This starts a TX DMA channel 2248 */ 2249 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2250 { 2251 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2252 stmmac_start_tx(priv, priv->ioaddr, chan); 2253 } 2254 2255 /** 2256 * stmmac_stop_rx_dma - stop RX DMA channel 2257 * @priv: driver private structure 2258 * @chan: RX channel index 2259 * Description: 2260 * This stops a RX DMA channel 2261 */ 2262 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2263 { 2264 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2265 stmmac_stop_rx(priv, priv->ioaddr, chan); 2266 } 2267 2268 /** 2269 * stmmac_stop_tx_dma - stop TX DMA channel 2270 * @priv: driver private structure 2271 * @chan: TX channel index 2272 * Description: 2273 * This stops a TX DMA channel 2274 */ 2275 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2276 { 2277 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2278 stmmac_stop_tx(priv, priv->ioaddr, chan); 2279 } 2280 2281 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2282 { 2283 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2284 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2285 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2286 u32 chan; 2287 2288 for (chan = 0; chan < dma_csr_ch; chan++) { 2289 struct stmmac_channel *ch = &priv->channel[chan]; 2290 unsigned long flags; 2291 2292 spin_lock_irqsave(&ch->lock, flags); 2293 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2294 spin_unlock_irqrestore(&ch->lock, flags); 2295 } 2296 } 2297 2298 /** 2299 * stmmac_start_all_dma - start all RX and TX DMA channels 2300 * @priv: driver private structure 2301 * Description: 2302 * This starts all the RX and TX DMA channels 2303 */ 2304 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2305 { 2306 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2307 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2308 u32 chan = 0; 2309 2310 for (chan = 0; chan < rx_channels_count; chan++) 2311 stmmac_start_rx_dma(priv, chan); 2312 2313 for (chan = 0; chan < tx_channels_count; chan++) 2314 stmmac_start_tx_dma(priv, chan); 2315 } 2316 2317 /** 2318 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2319 * @priv: driver private structure 2320 * Description: 2321 * This stops the RX and TX DMA channels 2322 */ 2323 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2324 { 2325 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2326 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2327 u32 chan = 0; 2328 2329 for (chan = 0; chan < rx_channels_count; chan++) 2330 stmmac_stop_rx_dma(priv, chan); 2331 2332 for (chan = 0; chan < tx_channels_count; chan++) 2333 stmmac_stop_tx_dma(priv, chan); 2334 } 2335 2336 /** 2337 * stmmac_dma_operation_mode - HW DMA operation mode 2338 * @priv: driver private structure 2339 * Description: it is used for configuring the DMA operation mode register in 2340 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2341 */ 2342 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2343 { 2344 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2345 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2346 int rxfifosz = priv->plat->rx_fifo_size; 2347 int txfifosz = priv->plat->tx_fifo_size; 2348 u32 txmode = 0; 2349 u32 rxmode = 0; 2350 u32 chan = 0; 2351 u8 qmode = 0; 2352 2353 if (rxfifosz == 0) 2354 rxfifosz = priv->dma_cap.rx_fifo_size; 2355 if (txfifosz == 0) 2356 txfifosz = priv->dma_cap.tx_fifo_size; 2357 2358 /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ 2359 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { 2360 rxfifosz /= rx_channels_count; 2361 txfifosz /= tx_channels_count; 2362 } 2363 2364 if (priv->plat->force_thresh_dma_mode) { 2365 txmode = tc; 2366 rxmode = tc; 2367 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2368 /* 2369 * In case of GMAC, SF mode can be enabled 2370 * to perform the TX COE in HW. This depends on: 2371 * 1) TX COE if actually supported 2372 * 2) There is no bugged Jumbo frame support 2373 * that needs to not insert csum in the TDES. 2374 */ 2375 txmode = SF_DMA_MODE; 2376 rxmode = SF_DMA_MODE; 2377 priv->xstats.threshold = SF_DMA_MODE; 2378 } else { 2379 txmode = tc; 2380 rxmode = SF_DMA_MODE; 2381 } 2382 2383 /* configure all channels */ 2384 for (chan = 0; chan < rx_channels_count; chan++) { 2385 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2386 u32 buf_size; 2387 2388 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2389 2390 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2391 rxfifosz, qmode); 2392 2393 if (rx_q->xsk_pool) { 2394 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2395 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2396 buf_size, 2397 chan); 2398 } else { 2399 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2400 priv->dma_conf.dma_buf_sz, 2401 chan); 2402 } 2403 } 2404 2405 for (chan = 0; chan < tx_channels_count; chan++) { 2406 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2407 2408 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2409 txfifosz, qmode); 2410 } 2411 } 2412 2413 static void stmmac_xsk_request_timestamp(void *_priv) 2414 { 2415 struct stmmac_metadata_request *meta_req = _priv; 2416 2417 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2418 *meta_req->set_ic = true; 2419 } 2420 2421 static u64 stmmac_xsk_fill_timestamp(void *_priv) 2422 { 2423 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2424 struct stmmac_priv *priv = tx_compl->priv; 2425 struct dma_desc *desc = tx_compl->desc; 2426 bool found = false; 2427 u64 ns = 0; 2428 2429 if (!priv->hwts_tx_en) 2430 return 0; 2431 2432 /* check tx tstamp status */ 2433 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2434 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2435 found = true; 2436 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2437 found = true; 2438 } 2439 2440 if (found) { 2441 ns -= priv->plat->cdc_error_adj; 2442 return ns_to_ktime(ns); 2443 } 2444 2445 return 0; 2446 } 2447 2448 static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2449 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2450 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2451 }; 2452 2453 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2454 { 2455 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2456 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2457 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2458 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2459 unsigned int entry = tx_q->cur_tx; 2460 struct dma_desc *tx_desc = NULL; 2461 struct xdp_desc xdp_desc; 2462 bool work_done = true; 2463 u32 tx_set_ic_bit = 0; 2464 2465 /* Avoids TX time-out as we are sharing with slow path */ 2466 txq_trans_cond_update(nq); 2467 2468 budget = min(budget, stmmac_tx_avail(priv, queue)); 2469 2470 while (budget-- > 0) { 2471 struct stmmac_metadata_request meta_req; 2472 struct xsk_tx_metadata *meta = NULL; 2473 dma_addr_t dma_addr; 2474 bool set_ic; 2475 2476 /* We are sharing with slow path and stop XSK TX desc submission when 2477 * available TX ring is less than threshold. 2478 */ 2479 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2480 !netif_carrier_ok(priv->dev)) { 2481 work_done = false; 2482 break; 2483 } 2484 2485 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2486 break; 2487 2488 if (priv->est && priv->est->enable && 2489 priv->est->max_sdu[queue] && 2490 xdp_desc.len > priv->est->max_sdu[queue]) { 2491 priv->xstats.max_sdu_txq_drop[queue]++; 2492 continue; 2493 } 2494 2495 if (likely(priv->extend_desc)) 2496 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2497 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2498 tx_desc = &tx_q->dma_entx[entry].basic; 2499 else 2500 tx_desc = tx_q->dma_tx + entry; 2501 2502 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2503 meta = xsk_buff_get_metadata(pool, xdp_desc.addr); 2504 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2505 2506 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2507 2508 /* To return XDP buffer to XSK pool, we simple call 2509 * xsk_tx_completed(), so we don't need to fill up 2510 * 'buf' and 'xdpf'. 2511 */ 2512 tx_q->tx_skbuff_dma[entry].buf = 0; 2513 tx_q->xdpf[entry] = NULL; 2514 2515 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2516 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2517 tx_q->tx_skbuff_dma[entry].last_segment = true; 2518 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2519 2520 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2521 2522 tx_q->tx_count_frames++; 2523 2524 if (!priv->tx_coal_frames[queue]) 2525 set_ic = false; 2526 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2527 set_ic = true; 2528 else 2529 set_ic = false; 2530 2531 meta_req.priv = priv; 2532 meta_req.tx_desc = tx_desc; 2533 meta_req.set_ic = &set_ic; 2534 xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops, 2535 &meta_req); 2536 if (set_ic) { 2537 tx_q->tx_count_frames = 0; 2538 stmmac_set_tx_ic(priv, tx_desc); 2539 tx_set_ic_bit++; 2540 } 2541 2542 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2543 true, priv->mode, true, true, 2544 xdp_desc.len); 2545 2546 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 2547 2548 xsk_tx_metadata_to_compl(meta, 2549 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2550 2551 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2552 entry = tx_q->cur_tx; 2553 } 2554 u64_stats_update_begin(&txq_stats->napi_syncp); 2555 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2556 u64_stats_update_end(&txq_stats->napi_syncp); 2557 2558 if (tx_desc) { 2559 stmmac_flush_tx_descriptors(priv, queue); 2560 xsk_tx_release(pool); 2561 } 2562 2563 /* Return true if all of the 3 conditions are met 2564 * a) TX Budget is still available 2565 * b) work_done = true when XSK TX desc peek is empty (no more 2566 * pending XSK TX for transmission) 2567 */ 2568 return !!budget && work_done; 2569 } 2570 2571 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2572 { 2573 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2574 tc += 64; 2575 2576 if (priv->plat->force_thresh_dma_mode) 2577 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2578 else 2579 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2580 chan); 2581 2582 priv->xstats.threshold = tc; 2583 } 2584 } 2585 2586 /** 2587 * stmmac_tx_clean - to manage the transmission completion 2588 * @priv: driver private structure 2589 * @budget: napi budget limiting this functions packet handling 2590 * @queue: TX queue index 2591 * @pending_packets: signal to arm the TX coal timer 2592 * Description: it reclaims the transmit resources after transmission completes. 2593 * If some packets still needs to be handled, due to TX coalesce, set 2594 * pending_packets to true to make NAPI arm the TX coal timer. 2595 */ 2596 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2597 bool *pending_packets) 2598 { 2599 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2600 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2601 unsigned int bytes_compl = 0, pkts_compl = 0; 2602 unsigned int entry, xmits = 0, count = 0; 2603 u32 tx_packets = 0, tx_errors = 0; 2604 2605 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2606 2607 tx_q->xsk_frames_done = 0; 2608 2609 entry = tx_q->dirty_tx; 2610 2611 /* Try to clean all TX complete frame in 1 shot */ 2612 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2613 struct xdp_frame *xdpf; 2614 struct sk_buff *skb; 2615 struct dma_desc *p; 2616 int status; 2617 2618 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2619 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2620 xdpf = tx_q->xdpf[entry]; 2621 skb = NULL; 2622 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2623 xdpf = NULL; 2624 skb = tx_q->tx_skbuff[entry]; 2625 } else { 2626 xdpf = NULL; 2627 skb = NULL; 2628 } 2629 2630 if (priv->extend_desc) 2631 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2632 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2633 p = &tx_q->dma_entx[entry].basic; 2634 else 2635 p = tx_q->dma_tx + entry; 2636 2637 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2638 /* Check if the descriptor is owned by the DMA */ 2639 if (unlikely(status & tx_dma_own)) 2640 break; 2641 2642 count++; 2643 2644 /* Make sure descriptor fields are read after reading 2645 * the own bit. 2646 */ 2647 dma_rmb(); 2648 2649 /* Just consider the last segment and ...*/ 2650 if (likely(!(status & tx_not_ls))) { 2651 /* ... verify the status error condition */ 2652 if (unlikely(status & tx_err)) { 2653 tx_errors++; 2654 if (unlikely(status & tx_err_bump_tc)) 2655 stmmac_bump_dma_threshold(priv, queue); 2656 } else { 2657 tx_packets++; 2658 } 2659 if (skb) { 2660 stmmac_get_tx_hwtstamp(priv, p, skb); 2661 } else if (tx_q->xsk_pool && 2662 xp_tx_metadata_enabled(tx_q->xsk_pool)) { 2663 struct stmmac_xsk_tx_complete tx_compl = { 2664 .priv = priv, 2665 .desc = p, 2666 }; 2667 2668 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2669 &stmmac_xsk_tx_metadata_ops, 2670 &tx_compl); 2671 } 2672 } 2673 2674 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2675 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2676 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2677 dma_unmap_page(priv->device, 2678 tx_q->tx_skbuff_dma[entry].buf, 2679 tx_q->tx_skbuff_dma[entry].len, 2680 DMA_TO_DEVICE); 2681 else 2682 dma_unmap_single(priv->device, 2683 tx_q->tx_skbuff_dma[entry].buf, 2684 tx_q->tx_skbuff_dma[entry].len, 2685 DMA_TO_DEVICE); 2686 tx_q->tx_skbuff_dma[entry].buf = 0; 2687 tx_q->tx_skbuff_dma[entry].len = 0; 2688 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2689 } 2690 2691 stmmac_clean_desc3(priv, tx_q, p); 2692 2693 tx_q->tx_skbuff_dma[entry].last_segment = false; 2694 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2695 2696 if (xdpf && 2697 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2698 xdp_return_frame_rx_napi(xdpf); 2699 tx_q->xdpf[entry] = NULL; 2700 } 2701 2702 if (xdpf && 2703 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2704 xdp_return_frame(xdpf); 2705 tx_q->xdpf[entry] = NULL; 2706 } 2707 2708 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2709 tx_q->xsk_frames_done++; 2710 2711 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2712 if (likely(skb)) { 2713 pkts_compl++; 2714 bytes_compl += skb->len; 2715 dev_consume_skb_any(skb); 2716 tx_q->tx_skbuff[entry] = NULL; 2717 } 2718 } 2719 2720 stmmac_release_tx_desc(priv, p, priv->mode); 2721 2722 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 2723 } 2724 tx_q->dirty_tx = entry; 2725 2726 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2727 pkts_compl, bytes_compl); 2728 2729 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2730 queue))) && 2731 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2732 2733 netif_dbg(priv, tx_done, priv->dev, 2734 "%s: restart transmit\n", __func__); 2735 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2736 } 2737 2738 if (tx_q->xsk_pool) { 2739 bool work_done; 2740 2741 if (tx_q->xsk_frames_done) 2742 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2743 2744 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2745 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2746 2747 /* For XSK TX, we try to send as many as possible. 2748 * If XSK work done (XSK TX desc empty and budget still 2749 * available), return "budget - 1" to reenable TX IRQ. 2750 * Else, return "budget" to make NAPI continue polling. 2751 */ 2752 work_done = stmmac_xdp_xmit_zc(priv, queue, 2753 STMMAC_XSK_TX_BUDGET_MAX); 2754 if (work_done) 2755 xmits = budget - 1; 2756 else 2757 xmits = budget; 2758 } 2759 2760 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && 2761 priv->eee_sw_timer_en) { 2762 if (stmmac_enable_eee_mode(priv)) 2763 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 2764 } 2765 2766 /* We still have pending packets, let's call for a new scheduling */ 2767 if (tx_q->dirty_tx != tx_q->cur_tx) 2768 *pending_packets = true; 2769 2770 u64_stats_update_begin(&txq_stats->napi_syncp); 2771 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2772 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2773 u64_stats_inc(&txq_stats->napi.tx_clean); 2774 u64_stats_update_end(&txq_stats->napi_syncp); 2775 2776 priv->xstats.tx_errors += tx_errors; 2777 2778 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2779 2780 /* Combine decisions from TX clean and XSK TX */ 2781 return max(count, xmits); 2782 } 2783 2784 /** 2785 * stmmac_tx_err - to manage the tx error 2786 * @priv: driver private structure 2787 * @chan: channel index 2788 * Description: it cleans the descriptors and restarts the transmission 2789 * in case of transmission errors. 2790 */ 2791 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2792 { 2793 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2794 2795 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2796 2797 stmmac_stop_tx_dma(priv, chan); 2798 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 2799 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 2800 stmmac_reset_tx_queue(priv, chan); 2801 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2802 tx_q->dma_tx_phy, chan); 2803 stmmac_start_tx_dma(priv, chan); 2804 2805 priv->xstats.tx_errors++; 2806 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2807 } 2808 2809 /** 2810 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2811 * @priv: driver private structure 2812 * @txmode: TX operating mode 2813 * @rxmode: RX operating mode 2814 * @chan: channel index 2815 * Description: it is used for configuring of the DMA operation mode in 2816 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2817 * mode. 2818 */ 2819 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2820 u32 rxmode, u32 chan) 2821 { 2822 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2823 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2824 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2825 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2826 int rxfifosz = priv->plat->rx_fifo_size; 2827 int txfifosz = priv->plat->tx_fifo_size; 2828 2829 if (rxfifosz == 0) 2830 rxfifosz = priv->dma_cap.rx_fifo_size; 2831 if (txfifosz == 0) 2832 txfifosz = priv->dma_cap.tx_fifo_size; 2833 2834 /* Adjust for real per queue fifo size */ 2835 rxfifosz /= rx_channels_count; 2836 txfifosz /= tx_channels_count; 2837 2838 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2839 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2840 } 2841 2842 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2843 { 2844 int ret; 2845 2846 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2847 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2848 if (ret && (ret != -EINVAL)) { 2849 stmmac_global_err(priv); 2850 return true; 2851 } 2852 2853 return false; 2854 } 2855 2856 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 2857 { 2858 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2859 &priv->xstats, chan, dir); 2860 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2861 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2862 struct stmmac_channel *ch = &priv->channel[chan]; 2863 struct napi_struct *rx_napi; 2864 struct napi_struct *tx_napi; 2865 unsigned long flags; 2866 2867 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 2868 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2869 2870 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 2871 if (napi_schedule_prep(rx_napi)) { 2872 spin_lock_irqsave(&ch->lock, flags); 2873 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 2874 spin_unlock_irqrestore(&ch->lock, flags); 2875 __napi_schedule(rx_napi); 2876 } 2877 } 2878 2879 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 2880 if (napi_schedule_prep(tx_napi)) { 2881 spin_lock_irqsave(&ch->lock, flags); 2882 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 2883 spin_unlock_irqrestore(&ch->lock, flags); 2884 __napi_schedule(tx_napi); 2885 } 2886 } 2887 2888 return status; 2889 } 2890 2891 /** 2892 * stmmac_dma_interrupt - DMA ISR 2893 * @priv: driver private structure 2894 * Description: this is the DMA ISR. It is called by the main ISR. 2895 * It calls the dwmac dma routine and schedule poll method in case of some 2896 * work can be done. 2897 */ 2898 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2899 { 2900 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2901 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2902 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2903 tx_channel_count : rx_channel_count; 2904 u32 chan; 2905 int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2906 2907 /* Make sure we never check beyond our status buffer. */ 2908 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2909 channels_to_check = ARRAY_SIZE(status); 2910 2911 for (chan = 0; chan < channels_to_check; chan++) 2912 status[chan] = stmmac_napi_check(priv, chan, 2913 DMA_DIR_RXTX); 2914 2915 for (chan = 0; chan < tx_channel_count; chan++) { 2916 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2917 /* Try to bump up the dma threshold on this failure */ 2918 stmmac_bump_dma_threshold(priv, chan); 2919 } else if (unlikely(status[chan] == tx_hard_error)) { 2920 stmmac_tx_err(priv, chan); 2921 } 2922 } 2923 } 2924 2925 /** 2926 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2927 * @priv: driver private structure 2928 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2929 */ 2930 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2931 { 2932 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2933 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2934 2935 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 2936 2937 if (priv->dma_cap.rmon) { 2938 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 2939 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2940 } else 2941 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2942 } 2943 2944 /** 2945 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2946 * @priv: driver private structure 2947 * Description: 2948 * new GMAC chip generations have a new register to indicate the 2949 * presence of the optional feature/functions. 2950 * This can be also used to override the value passed through the 2951 * platform and necessary for old MAC10/100 and GMAC chips. 2952 */ 2953 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2954 { 2955 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2956 } 2957 2958 /** 2959 * stmmac_check_ether_addr - check if the MAC addr is valid 2960 * @priv: driver private structure 2961 * Description: 2962 * it is to verify if the MAC address is valid, in case of failures it 2963 * generates a random MAC address 2964 */ 2965 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2966 { 2967 u8 addr[ETH_ALEN]; 2968 2969 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2970 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 2971 if (is_valid_ether_addr(addr)) 2972 eth_hw_addr_set(priv->dev, addr); 2973 else 2974 eth_hw_addr_random(priv->dev); 2975 dev_info(priv->device, "device MAC address %pM\n", 2976 priv->dev->dev_addr); 2977 } 2978 } 2979 2980 /** 2981 * stmmac_init_dma_engine - DMA init. 2982 * @priv: driver private structure 2983 * Description: 2984 * It inits the DMA invoking the specific MAC/GMAC callback. 2985 * Some DMA parameters can be passed from the platform; 2986 * in case of these are not passed a default is kept for the MAC or GMAC. 2987 */ 2988 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2989 { 2990 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2991 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2992 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2993 struct stmmac_rx_queue *rx_q; 2994 struct stmmac_tx_queue *tx_q; 2995 u32 chan = 0; 2996 int ret = 0; 2997 2998 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2999 dev_err(priv->device, "Invalid DMA configuration\n"); 3000 return -EINVAL; 3001 } 3002 3003 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 3004 priv->plat->dma_cfg->atds = 1; 3005 3006 ret = stmmac_reset(priv, priv->ioaddr); 3007 if (ret) { 3008 dev_err(priv->device, "Failed to reset the dma\n"); 3009 return ret; 3010 } 3011 3012 /* DMA Configuration */ 3013 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); 3014 3015 if (priv->plat->axi) 3016 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3017 3018 /* DMA CSR Channel configuration */ 3019 for (chan = 0; chan < dma_csr_ch; chan++) { 3020 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3021 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3022 } 3023 3024 /* DMA RX Channel Configuration */ 3025 for (chan = 0; chan < rx_channels_count; chan++) { 3026 rx_q = &priv->dma_conf.rx_queue[chan]; 3027 3028 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3029 rx_q->dma_rx_phy, chan); 3030 3031 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 3032 (rx_q->buf_alloc_num * 3033 sizeof(struct dma_desc)); 3034 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 3035 rx_q->rx_tail_addr, chan); 3036 } 3037 3038 /* DMA TX Channel Configuration */ 3039 for (chan = 0; chan < tx_channels_count; chan++) { 3040 tx_q = &priv->dma_conf.tx_queue[chan]; 3041 3042 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3043 tx_q->dma_tx_phy, chan); 3044 3045 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 3046 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 3047 tx_q->tx_tail_addr, chan); 3048 } 3049 3050 return ret; 3051 } 3052 3053 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3054 { 3055 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3056 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3057 struct stmmac_channel *ch; 3058 struct napi_struct *napi; 3059 3060 if (!tx_coal_timer) 3061 return; 3062 3063 ch = &priv->channel[tx_q->queue_index]; 3064 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3065 3066 /* Arm timer only if napi is not already scheduled. 3067 * Try to cancel any timer if napi is scheduled, timer will be armed 3068 * again in the next scheduled napi. 3069 */ 3070 if (unlikely(!napi_is_scheduled(napi))) 3071 hrtimer_start(&tx_q->txtimer, 3072 STMMAC_COAL_TIMER(tx_coal_timer), 3073 HRTIMER_MODE_REL); 3074 else 3075 hrtimer_try_to_cancel(&tx_q->txtimer); 3076 } 3077 3078 /** 3079 * stmmac_tx_timer - mitigation sw timer for tx. 3080 * @t: data pointer 3081 * Description: 3082 * This is the timer handler to directly invoke the stmmac_tx_clean. 3083 */ 3084 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3085 { 3086 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3087 struct stmmac_priv *priv = tx_q->priv_data; 3088 struct stmmac_channel *ch; 3089 struct napi_struct *napi; 3090 3091 ch = &priv->channel[tx_q->queue_index]; 3092 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3093 3094 if (likely(napi_schedule_prep(napi))) { 3095 unsigned long flags; 3096 3097 spin_lock_irqsave(&ch->lock, flags); 3098 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3099 spin_unlock_irqrestore(&ch->lock, flags); 3100 __napi_schedule(napi); 3101 } 3102 3103 return HRTIMER_NORESTART; 3104 } 3105 3106 /** 3107 * stmmac_init_coalesce - init mitigation options. 3108 * @priv: driver private structure 3109 * Description: 3110 * This inits the coalesce parameters: i.e. timer rate, 3111 * timer handler and default threshold used for enabling the 3112 * interrupt on completion bit. 3113 */ 3114 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3115 { 3116 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3117 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3118 u32 chan; 3119 3120 for (chan = 0; chan < tx_channel_count; chan++) { 3121 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3122 3123 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3124 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3125 3126 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3127 tx_q->txtimer.function = stmmac_tx_timer; 3128 } 3129 3130 for (chan = 0; chan < rx_channel_count; chan++) 3131 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3132 } 3133 3134 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3135 { 3136 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3137 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3138 u32 chan; 3139 3140 /* set TX ring length */ 3141 for (chan = 0; chan < tx_channels_count; chan++) 3142 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3143 (priv->dma_conf.dma_tx_size - 1), chan); 3144 3145 /* set RX ring length */ 3146 for (chan = 0; chan < rx_channels_count; chan++) 3147 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3148 (priv->dma_conf.dma_rx_size - 1), chan); 3149 } 3150 3151 /** 3152 * stmmac_set_tx_queue_weight - Set TX queue weight 3153 * @priv: driver private structure 3154 * Description: It is used for setting TX queues weight 3155 */ 3156 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3157 { 3158 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3159 u32 weight; 3160 u32 queue; 3161 3162 for (queue = 0; queue < tx_queues_count; queue++) { 3163 weight = priv->plat->tx_queues_cfg[queue].weight; 3164 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3165 } 3166 } 3167 3168 /** 3169 * stmmac_configure_cbs - Configure CBS in TX queue 3170 * @priv: driver private structure 3171 * Description: It is used for configuring CBS in AVB TX queues 3172 */ 3173 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3174 { 3175 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3176 u32 mode_to_use; 3177 u32 queue; 3178 3179 /* queue 0 is reserved for legacy traffic */ 3180 for (queue = 1; queue < tx_queues_count; queue++) { 3181 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3182 if (mode_to_use == MTL_QUEUE_DCB) 3183 continue; 3184 3185 stmmac_config_cbs(priv, priv->hw, 3186 priv->plat->tx_queues_cfg[queue].send_slope, 3187 priv->plat->tx_queues_cfg[queue].idle_slope, 3188 priv->plat->tx_queues_cfg[queue].high_credit, 3189 priv->plat->tx_queues_cfg[queue].low_credit, 3190 queue); 3191 } 3192 } 3193 3194 /** 3195 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3196 * @priv: driver private structure 3197 * Description: It is used for mapping RX queues to RX dma channels 3198 */ 3199 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3200 { 3201 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3202 u32 queue; 3203 u32 chan; 3204 3205 for (queue = 0; queue < rx_queues_count; queue++) { 3206 chan = priv->plat->rx_queues_cfg[queue].chan; 3207 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3208 } 3209 } 3210 3211 /** 3212 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3213 * @priv: driver private structure 3214 * Description: It is used for configuring the RX Queue Priority 3215 */ 3216 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3217 { 3218 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3219 u32 queue; 3220 u32 prio; 3221 3222 for (queue = 0; queue < rx_queues_count; queue++) { 3223 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3224 continue; 3225 3226 prio = priv->plat->rx_queues_cfg[queue].prio; 3227 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3228 } 3229 } 3230 3231 /** 3232 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3233 * @priv: driver private structure 3234 * Description: It is used for configuring the TX Queue Priority 3235 */ 3236 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3237 { 3238 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3239 u32 queue; 3240 u32 prio; 3241 3242 for (queue = 0; queue < tx_queues_count; queue++) { 3243 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3244 continue; 3245 3246 prio = priv->plat->tx_queues_cfg[queue].prio; 3247 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3248 } 3249 } 3250 3251 /** 3252 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3253 * @priv: driver private structure 3254 * Description: It is used for configuring the RX queue routing 3255 */ 3256 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3257 { 3258 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3259 u32 queue; 3260 u8 packet; 3261 3262 for (queue = 0; queue < rx_queues_count; queue++) { 3263 /* no specific packet type routing specified for the queue */ 3264 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3265 continue; 3266 3267 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3268 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3269 } 3270 } 3271 3272 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3273 { 3274 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3275 priv->rss.enable = false; 3276 return; 3277 } 3278 3279 if (priv->dev->features & NETIF_F_RXHASH) 3280 priv->rss.enable = true; 3281 else 3282 priv->rss.enable = false; 3283 3284 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3285 priv->plat->rx_queues_to_use); 3286 } 3287 3288 /** 3289 * stmmac_mtl_configuration - Configure MTL 3290 * @priv: driver private structure 3291 * Description: It is used for configurring MTL 3292 */ 3293 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3294 { 3295 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3296 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3297 3298 if (tx_queues_count > 1) 3299 stmmac_set_tx_queue_weight(priv); 3300 3301 /* Configure MTL RX algorithms */ 3302 if (rx_queues_count > 1) 3303 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3304 priv->plat->rx_sched_algorithm); 3305 3306 /* Configure MTL TX algorithms */ 3307 if (tx_queues_count > 1) 3308 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3309 priv->plat->tx_sched_algorithm); 3310 3311 /* Configure CBS in AVB TX queues */ 3312 if (tx_queues_count > 1) 3313 stmmac_configure_cbs(priv); 3314 3315 /* Map RX MTL to DMA channels */ 3316 stmmac_rx_queue_dma_chan_map(priv); 3317 3318 /* Enable MAC RX Queues */ 3319 stmmac_mac_enable_rx_queues(priv); 3320 3321 /* Set RX priorities */ 3322 if (rx_queues_count > 1) 3323 stmmac_mac_config_rx_queues_prio(priv); 3324 3325 /* Set TX priorities */ 3326 if (tx_queues_count > 1) 3327 stmmac_mac_config_tx_queues_prio(priv); 3328 3329 /* Set RX routing */ 3330 if (rx_queues_count > 1) 3331 stmmac_mac_config_rx_queues_routing(priv); 3332 3333 /* Receive Side Scaling */ 3334 if (rx_queues_count > 1) 3335 stmmac_mac_config_rss(priv); 3336 } 3337 3338 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3339 { 3340 if (priv->dma_cap.asp) { 3341 netdev_info(priv->dev, "Enabling Safety Features\n"); 3342 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3343 priv->plat->safety_feat_cfg); 3344 } else { 3345 netdev_info(priv->dev, "No Safety Features support found\n"); 3346 } 3347 } 3348 3349 /** 3350 * stmmac_hw_setup - setup mac in a usable state. 3351 * @dev : pointer to the device structure. 3352 * @ptp_register: register PTP if set 3353 * Description: 3354 * this is the main function to setup the HW in a usable state because the 3355 * dma engine is reset, the core registers are configured (e.g. AXI, 3356 * Checksum features, timers). The DMA is ready to start receiving and 3357 * transmitting. 3358 * Return value: 3359 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3360 * file on failure. 3361 */ 3362 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) 3363 { 3364 struct stmmac_priv *priv = netdev_priv(dev); 3365 u32 rx_cnt = priv->plat->rx_queues_to_use; 3366 u32 tx_cnt = priv->plat->tx_queues_to_use; 3367 bool sph_en; 3368 u32 chan; 3369 int ret; 3370 3371 /* Make sure RX clock is enabled */ 3372 if (priv->hw->phylink_pcs) 3373 phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs); 3374 3375 /* DMA initialization and SW reset */ 3376 ret = stmmac_init_dma_engine(priv); 3377 if (ret < 0) { 3378 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3379 __func__); 3380 return ret; 3381 } 3382 3383 /* Copy the MAC addr into the HW */ 3384 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3385 3386 /* PS and related bits will be programmed according to the speed */ 3387 if (priv->hw->pcs) { 3388 int speed = priv->plat->mac_port_sel_speed; 3389 3390 if ((speed == SPEED_10) || (speed == SPEED_100) || 3391 (speed == SPEED_1000)) { 3392 priv->hw->ps = speed; 3393 } else { 3394 dev_warn(priv->device, "invalid port speed\n"); 3395 priv->hw->ps = 0; 3396 } 3397 } 3398 3399 /* Initialize the MAC Core */ 3400 stmmac_core_init(priv, priv->hw, dev); 3401 3402 /* Initialize MTL*/ 3403 stmmac_mtl_configuration(priv); 3404 3405 /* Initialize Safety Features */ 3406 stmmac_safety_feat_configuration(priv); 3407 3408 ret = stmmac_rx_ipc(priv, priv->hw); 3409 if (!ret) { 3410 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3411 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3412 priv->hw->rx_csum = 0; 3413 } 3414 3415 /* Enable the MAC Rx/Tx */ 3416 stmmac_mac_set(priv, priv->ioaddr, true); 3417 3418 /* Set the HW DMA mode and the COE */ 3419 stmmac_dma_operation_mode(priv); 3420 3421 stmmac_mmc_setup(priv); 3422 3423 if (ptp_register) { 3424 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 3425 if (ret < 0) 3426 netdev_warn(priv->dev, 3427 "failed to enable PTP reference clock: %pe\n", 3428 ERR_PTR(ret)); 3429 } 3430 3431 ret = stmmac_init_ptp(priv); 3432 if (ret == -EOPNOTSUPP) 3433 netdev_info(priv->dev, "PTP not supported by HW\n"); 3434 else if (ret) 3435 netdev_warn(priv->dev, "PTP init failed\n"); 3436 else if (ptp_register) 3437 stmmac_ptp_register(priv); 3438 3439 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; 3440 3441 /* Convert the timer from msec to usec */ 3442 if (!priv->tx_lpi_timer) 3443 priv->tx_lpi_timer = eee_timer * 1000; 3444 3445 if (priv->use_riwt) { 3446 u32 queue; 3447 3448 for (queue = 0; queue < rx_cnt; queue++) { 3449 if (!priv->rx_riwt[queue]) 3450 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3451 3452 stmmac_rx_watchdog(priv, priv->ioaddr, 3453 priv->rx_riwt[queue], queue); 3454 } 3455 } 3456 3457 if (priv->hw->pcs) 3458 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); 3459 3460 /* set TX and RX rings length */ 3461 stmmac_set_rings_length(priv); 3462 3463 /* Enable TSO */ 3464 if (priv->tso) { 3465 for (chan = 0; chan < tx_cnt; chan++) { 3466 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3467 3468 /* TSO and TBS cannot co-exist */ 3469 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3470 continue; 3471 3472 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3473 } 3474 } 3475 3476 /* Enable Split Header */ 3477 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 3478 for (chan = 0; chan < rx_cnt; chan++) 3479 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3480 3481 3482 /* VLAN Tag Insertion */ 3483 if (priv->dma_cap.vlins) 3484 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3485 3486 /* TBS */ 3487 for (chan = 0; chan < tx_cnt; chan++) { 3488 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3489 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3490 3491 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3492 } 3493 3494 /* Configure real RX and TX queues */ 3495 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3496 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3497 3498 /* Start the ball rolling... */ 3499 stmmac_start_all_dma(priv); 3500 3501 stmmac_set_hw_vlan_mode(priv, priv->hw); 3502 3503 return 0; 3504 } 3505 3506 static void stmmac_hw_teardown(struct net_device *dev) 3507 { 3508 struct stmmac_priv *priv = netdev_priv(dev); 3509 3510 clk_disable_unprepare(priv->plat->clk_ptp_ref); 3511 } 3512 3513 static void stmmac_free_irq(struct net_device *dev, 3514 enum request_irq_err irq_err, int irq_idx) 3515 { 3516 struct stmmac_priv *priv = netdev_priv(dev); 3517 int j; 3518 3519 switch (irq_err) { 3520 case REQ_IRQ_ERR_ALL: 3521 irq_idx = priv->plat->tx_queues_to_use; 3522 fallthrough; 3523 case REQ_IRQ_ERR_TX: 3524 for (j = irq_idx - 1; j >= 0; j--) { 3525 if (priv->tx_irq[j] > 0) { 3526 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3527 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); 3528 } 3529 } 3530 irq_idx = priv->plat->rx_queues_to_use; 3531 fallthrough; 3532 case REQ_IRQ_ERR_RX: 3533 for (j = irq_idx - 1; j >= 0; j--) { 3534 if (priv->rx_irq[j] > 0) { 3535 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3536 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); 3537 } 3538 } 3539 3540 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3541 free_irq(priv->sfty_ue_irq, dev); 3542 fallthrough; 3543 case REQ_IRQ_ERR_SFTY_UE: 3544 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3545 free_irq(priv->sfty_ce_irq, dev); 3546 fallthrough; 3547 case REQ_IRQ_ERR_SFTY_CE: 3548 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) 3549 free_irq(priv->lpi_irq, dev); 3550 fallthrough; 3551 case REQ_IRQ_ERR_LPI: 3552 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3553 free_irq(priv->wol_irq, dev); 3554 fallthrough; 3555 case REQ_IRQ_ERR_SFTY: 3556 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) 3557 free_irq(priv->sfty_irq, dev); 3558 fallthrough; 3559 case REQ_IRQ_ERR_WOL: 3560 free_irq(dev->irq, dev); 3561 fallthrough; 3562 case REQ_IRQ_ERR_MAC: 3563 case REQ_IRQ_ERR_NO: 3564 /* If MAC IRQ request error, no more IRQ to free */ 3565 break; 3566 } 3567 } 3568 3569 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3570 { 3571 struct stmmac_priv *priv = netdev_priv(dev); 3572 enum request_irq_err irq_err; 3573 cpumask_t cpu_mask; 3574 int irq_idx = 0; 3575 char *int_name; 3576 int ret; 3577 int i; 3578 3579 /* For common interrupt */ 3580 int_name = priv->int_name_mac; 3581 sprintf(int_name, "%s:%s", dev->name, "mac"); 3582 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3583 0, int_name, dev); 3584 if (unlikely(ret < 0)) { 3585 netdev_err(priv->dev, 3586 "%s: alloc mac MSI %d (error: %d)\n", 3587 __func__, dev->irq, ret); 3588 irq_err = REQ_IRQ_ERR_MAC; 3589 goto irq_error; 3590 } 3591 3592 /* Request the Wake IRQ in case of another line 3593 * is used for WoL 3594 */ 3595 priv->wol_irq_disabled = true; 3596 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3597 int_name = priv->int_name_wol; 3598 sprintf(int_name, "%s:%s", dev->name, "wol"); 3599 ret = request_irq(priv->wol_irq, 3600 stmmac_mac_interrupt, 3601 0, int_name, dev); 3602 if (unlikely(ret < 0)) { 3603 netdev_err(priv->dev, 3604 "%s: alloc wol MSI %d (error: %d)\n", 3605 __func__, priv->wol_irq, ret); 3606 irq_err = REQ_IRQ_ERR_WOL; 3607 goto irq_error; 3608 } 3609 } 3610 3611 /* Request the LPI IRQ in case of another line 3612 * is used for LPI 3613 */ 3614 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3615 int_name = priv->int_name_lpi; 3616 sprintf(int_name, "%s:%s", dev->name, "lpi"); 3617 ret = request_irq(priv->lpi_irq, 3618 stmmac_mac_interrupt, 3619 0, int_name, dev); 3620 if (unlikely(ret < 0)) { 3621 netdev_err(priv->dev, 3622 "%s: alloc lpi MSI %d (error: %d)\n", 3623 __func__, priv->lpi_irq, ret); 3624 irq_err = REQ_IRQ_ERR_LPI; 3625 goto irq_error; 3626 } 3627 } 3628 3629 /* Request the common Safety Feature Correctible/Uncorrectible 3630 * Error line in case of another line is used 3631 */ 3632 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3633 int_name = priv->int_name_sfty; 3634 sprintf(int_name, "%s:%s", dev->name, "safety"); 3635 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3636 0, int_name, dev); 3637 if (unlikely(ret < 0)) { 3638 netdev_err(priv->dev, 3639 "%s: alloc sfty MSI %d (error: %d)\n", 3640 __func__, priv->sfty_irq, ret); 3641 irq_err = REQ_IRQ_ERR_SFTY; 3642 goto irq_error; 3643 } 3644 } 3645 3646 /* Request the Safety Feature Correctible Error line in 3647 * case of another line is used 3648 */ 3649 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3650 int_name = priv->int_name_sfty_ce; 3651 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3652 ret = request_irq(priv->sfty_ce_irq, 3653 stmmac_safety_interrupt, 3654 0, int_name, dev); 3655 if (unlikely(ret < 0)) { 3656 netdev_err(priv->dev, 3657 "%s: alloc sfty ce MSI %d (error: %d)\n", 3658 __func__, priv->sfty_ce_irq, ret); 3659 irq_err = REQ_IRQ_ERR_SFTY_CE; 3660 goto irq_error; 3661 } 3662 } 3663 3664 /* Request the Safety Feature Uncorrectible Error line in 3665 * case of another line is used 3666 */ 3667 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3668 int_name = priv->int_name_sfty_ue; 3669 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3670 ret = request_irq(priv->sfty_ue_irq, 3671 stmmac_safety_interrupt, 3672 0, int_name, dev); 3673 if (unlikely(ret < 0)) { 3674 netdev_err(priv->dev, 3675 "%s: alloc sfty ue MSI %d (error: %d)\n", 3676 __func__, priv->sfty_ue_irq, ret); 3677 irq_err = REQ_IRQ_ERR_SFTY_UE; 3678 goto irq_error; 3679 } 3680 } 3681 3682 /* Request Rx MSI irq */ 3683 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3684 if (i >= MTL_MAX_RX_QUEUES) 3685 break; 3686 if (priv->rx_irq[i] == 0) 3687 continue; 3688 3689 int_name = priv->int_name_rx_irq[i]; 3690 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3691 ret = request_irq(priv->rx_irq[i], 3692 stmmac_msi_intr_rx, 3693 0, int_name, &priv->dma_conf.rx_queue[i]); 3694 if (unlikely(ret < 0)) { 3695 netdev_err(priv->dev, 3696 "%s: alloc rx-%d MSI %d (error: %d)\n", 3697 __func__, i, priv->rx_irq[i], ret); 3698 irq_err = REQ_IRQ_ERR_RX; 3699 irq_idx = i; 3700 goto irq_error; 3701 } 3702 cpumask_clear(&cpu_mask); 3703 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3704 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); 3705 } 3706 3707 /* Request Tx MSI irq */ 3708 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3709 if (i >= MTL_MAX_TX_QUEUES) 3710 break; 3711 if (priv->tx_irq[i] == 0) 3712 continue; 3713 3714 int_name = priv->int_name_tx_irq[i]; 3715 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3716 ret = request_irq(priv->tx_irq[i], 3717 stmmac_msi_intr_tx, 3718 0, int_name, &priv->dma_conf.tx_queue[i]); 3719 if (unlikely(ret < 0)) { 3720 netdev_err(priv->dev, 3721 "%s: alloc tx-%d MSI %d (error: %d)\n", 3722 __func__, i, priv->tx_irq[i], ret); 3723 irq_err = REQ_IRQ_ERR_TX; 3724 irq_idx = i; 3725 goto irq_error; 3726 } 3727 cpumask_clear(&cpu_mask); 3728 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3729 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); 3730 } 3731 3732 return 0; 3733 3734 irq_error: 3735 stmmac_free_irq(dev, irq_err, irq_idx); 3736 return ret; 3737 } 3738 3739 static int stmmac_request_irq_single(struct net_device *dev) 3740 { 3741 struct stmmac_priv *priv = netdev_priv(dev); 3742 enum request_irq_err irq_err; 3743 int ret; 3744 3745 ret = request_irq(dev->irq, stmmac_interrupt, 3746 IRQF_SHARED, dev->name, dev); 3747 if (unlikely(ret < 0)) { 3748 netdev_err(priv->dev, 3749 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3750 __func__, dev->irq, ret); 3751 irq_err = REQ_IRQ_ERR_MAC; 3752 goto irq_error; 3753 } 3754 3755 /* Request the Wake IRQ in case of another line 3756 * is used for WoL 3757 */ 3758 priv->wol_irq_disabled = true; 3759 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3760 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3761 IRQF_SHARED, dev->name, dev); 3762 if (unlikely(ret < 0)) { 3763 netdev_err(priv->dev, 3764 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3765 __func__, priv->wol_irq, ret); 3766 irq_err = REQ_IRQ_ERR_WOL; 3767 goto irq_error; 3768 } 3769 } 3770 3771 /* Request the IRQ lines */ 3772 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3773 ret = request_irq(priv->lpi_irq, stmmac_interrupt, 3774 IRQF_SHARED, dev->name, dev); 3775 if (unlikely(ret < 0)) { 3776 netdev_err(priv->dev, 3777 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 3778 __func__, priv->lpi_irq, ret); 3779 irq_err = REQ_IRQ_ERR_LPI; 3780 goto irq_error; 3781 } 3782 } 3783 3784 /* Request the common Safety Feature Correctible/Uncorrectible 3785 * Error line in case of another line is used 3786 */ 3787 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3788 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3789 IRQF_SHARED, dev->name, dev); 3790 if (unlikely(ret < 0)) { 3791 netdev_err(priv->dev, 3792 "%s: ERROR: allocating the sfty IRQ %d (%d)\n", 3793 __func__, priv->sfty_irq, ret); 3794 irq_err = REQ_IRQ_ERR_SFTY; 3795 goto irq_error; 3796 } 3797 } 3798 3799 return 0; 3800 3801 irq_error: 3802 stmmac_free_irq(dev, irq_err, 0); 3803 return ret; 3804 } 3805 3806 static int stmmac_request_irq(struct net_device *dev) 3807 { 3808 struct stmmac_priv *priv = netdev_priv(dev); 3809 int ret; 3810 3811 /* Request the IRQ lines */ 3812 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 3813 ret = stmmac_request_irq_multi_msi(dev); 3814 else 3815 ret = stmmac_request_irq_single(dev); 3816 3817 return ret; 3818 } 3819 3820 /** 3821 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 3822 * @priv: driver private structure 3823 * @mtu: MTU to setup the dma queue and buf with 3824 * Description: Allocate and generate a dma_conf based on the provided MTU. 3825 * Allocate the Tx/Rx DMA queue and init them. 3826 * Return value: 3827 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 3828 */ 3829 static struct stmmac_dma_conf * 3830 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 3831 { 3832 struct stmmac_dma_conf *dma_conf; 3833 int chan, bfsize, ret; 3834 3835 dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); 3836 if (!dma_conf) { 3837 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 3838 __func__); 3839 return ERR_PTR(-ENOMEM); 3840 } 3841 3842 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 3843 if (bfsize < 0) 3844 bfsize = 0; 3845 3846 if (bfsize < BUF_SIZE_16KiB) 3847 bfsize = stmmac_set_bfsize(mtu, 0); 3848 3849 dma_conf->dma_buf_sz = bfsize; 3850 /* Chose the tx/rx size from the already defined one in the 3851 * priv struct. (if defined) 3852 */ 3853 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 3854 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 3855 3856 if (!dma_conf->dma_tx_size) 3857 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3858 if (!dma_conf->dma_rx_size) 3859 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3860 3861 /* Earlier check for TBS */ 3862 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3863 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 3864 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3865 3866 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3867 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3868 } 3869 3870 ret = alloc_dma_desc_resources(priv, dma_conf); 3871 if (ret < 0) { 3872 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 3873 __func__); 3874 goto alloc_error; 3875 } 3876 3877 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 3878 if (ret < 0) { 3879 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 3880 __func__); 3881 goto init_error; 3882 } 3883 3884 return dma_conf; 3885 3886 init_error: 3887 free_dma_desc_resources(priv, dma_conf); 3888 alloc_error: 3889 kfree(dma_conf); 3890 return ERR_PTR(ret); 3891 } 3892 3893 /** 3894 * __stmmac_open - open entry point of the driver 3895 * @dev : pointer to the device structure. 3896 * @dma_conf : structure to take the dma data 3897 * Description: 3898 * This function is the open entry point of the driver. 3899 * Return value: 3900 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3901 * file on failure. 3902 */ 3903 static int __stmmac_open(struct net_device *dev, 3904 struct stmmac_dma_conf *dma_conf) 3905 { 3906 struct stmmac_priv *priv = netdev_priv(dev); 3907 int mode = priv->plat->phy_interface; 3908 u32 chan; 3909 int ret; 3910 3911 ret = pm_runtime_resume_and_get(priv->device); 3912 if (ret < 0) 3913 return ret; 3914 3915 if ((!priv->hw->xpcs || 3916 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) { 3917 ret = stmmac_init_phy(dev); 3918 if (ret) { 3919 netdev_err(priv->dev, 3920 "%s: Cannot attach to PHY (error: %d)\n", 3921 __func__, ret); 3922 goto init_phy_error; 3923 } 3924 } 3925 3926 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 3927 3928 buf_sz = dma_conf->dma_buf_sz; 3929 for (int i = 0; i < MTL_MAX_TX_QUEUES; i++) 3930 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 3931 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 3932 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 3933 3934 stmmac_reset_queues_param(priv); 3935 3936 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 3937 priv->plat->serdes_powerup) { 3938 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); 3939 if (ret < 0) { 3940 netdev_err(priv->dev, "%s: Serdes powerup failed\n", 3941 __func__); 3942 goto init_error; 3943 } 3944 } 3945 3946 ret = stmmac_hw_setup(dev, true); 3947 if (ret < 0) { 3948 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 3949 goto init_error; 3950 } 3951 3952 stmmac_init_coalesce(priv); 3953 3954 phylink_start(priv->phylink); 3955 /* We may have called phylink_speed_down before */ 3956 phylink_speed_up(priv->phylink); 3957 3958 ret = stmmac_request_irq(dev); 3959 if (ret) 3960 goto irq_error; 3961 3962 stmmac_enable_all_queues(priv); 3963 netif_tx_start_all_queues(priv->dev); 3964 stmmac_enable_all_dma_irq(priv); 3965 3966 return 0; 3967 3968 irq_error: 3969 phylink_stop(priv->phylink); 3970 3971 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3972 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3973 3974 stmmac_hw_teardown(dev); 3975 init_error: 3976 phylink_disconnect_phy(priv->phylink); 3977 init_phy_error: 3978 pm_runtime_put(priv->device); 3979 return ret; 3980 } 3981 3982 static int stmmac_open(struct net_device *dev) 3983 { 3984 struct stmmac_priv *priv = netdev_priv(dev); 3985 struct stmmac_dma_conf *dma_conf; 3986 int ret; 3987 3988 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 3989 if (IS_ERR(dma_conf)) 3990 return PTR_ERR(dma_conf); 3991 3992 ret = __stmmac_open(dev, dma_conf); 3993 if (ret) 3994 free_dma_desc_resources(priv, dma_conf); 3995 3996 kfree(dma_conf); 3997 return ret; 3998 } 3999 4000 /** 4001 * stmmac_release - close entry point of the driver 4002 * @dev : device pointer. 4003 * Description: 4004 * This is the stop entry point of the driver. 4005 */ 4006 static int stmmac_release(struct net_device *dev) 4007 { 4008 struct stmmac_priv *priv = netdev_priv(dev); 4009 u32 chan; 4010 4011 if (device_may_wakeup(priv->device)) 4012 phylink_speed_down(priv->phylink, false); 4013 /* Stop and disconnect the PHY */ 4014 phylink_stop(priv->phylink); 4015 phylink_disconnect_phy(priv->phylink); 4016 4017 stmmac_disable_all_queues(priv); 4018 4019 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4020 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4021 4022 netif_tx_disable(dev); 4023 4024 /* Free the IRQ lines */ 4025 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4026 4027 if (priv->eee_enabled) { 4028 priv->tx_path_in_lpi_mode = false; 4029 del_timer_sync(&priv->eee_ctrl_timer); 4030 } 4031 4032 /* Stop TX/RX DMA and clear the descriptors */ 4033 stmmac_stop_all_dma(priv); 4034 4035 /* Release and free the Rx/Tx resources */ 4036 free_dma_desc_resources(priv, &priv->dma_conf); 4037 4038 /* Disable the MAC Rx/Tx */ 4039 stmmac_mac_set(priv, priv->ioaddr, false); 4040 4041 /* Powerdown Serdes if there is */ 4042 if (priv->plat->serdes_powerdown) 4043 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); 4044 4045 stmmac_release_ptp(priv); 4046 4047 if (stmmac_fpe_supported(priv)) 4048 timer_shutdown_sync(&priv->fpe_cfg.verify_timer); 4049 4050 pm_runtime_put(priv->device); 4051 4052 return 0; 4053 } 4054 4055 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4056 struct stmmac_tx_queue *tx_q) 4057 { 4058 u16 tag = 0x0, inner_tag = 0x0; 4059 u32 inner_type = 0x0; 4060 struct dma_desc *p; 4061 4062 if (!priv->dma_cap.vlins) 4063 return false; 4064 if (!skb_vlan_tag_present(skb)) 4065 return false; 4066 if (skb->vlan_proto == htons(ETH_P_8021AD)) { 4067 inner_tag = skb_vlan_tag_get(skb); 4068 inner_type = STMMAC_VLAN_INSERT; 4069 } 4070 4071 tag = skb_vlan_tag_get(skb); 4072 4073 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4074 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4075 else 4076 p = &tx_q->dma_tx[tx_q->cur_tx]; 4077 4078 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) 4079 return false; 4080 4081 stmmac_set_tx_owner(priv, p); 4082 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4083 return true; 4084 } 4085 4086 /** 4087 * stmmac_tso_allocator - close entry point of the driver 4088 * @priv: driver private structure 4089 * @des: buffer start address 4090 * @total_len: total length to fill in descriptors 4091 * @last_segment: condition for the last descriptor 4092 * @queue: TX queue index 4093 * Description: 4094 * This function fills descriptor and request new descriptors according to 4095 * buffer length to fill 4096 */ 4097 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4098 int total_len, bool last_segment, u32 queue) 4099 { 4100 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4101 struct dma_desc *desc; 4102 u32 buff_size; 4103 int tmp_len; 4104 4105 tmp_len = total_len; 4106 4107 while (tmp_len > 0) { 4108 dma_addr_t curr_addr; 4109 4110 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4111 priv->dma_conf.dma_tx_size); 4112 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4113 4114 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4115 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4116 else 4117 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4118 4119 curr_addr = des + (total_len - tmp_len); 4120 if (priv->dma_cap.addr64 <= 32) 4121 desc->des0 = cpu_to_le32(curr_addr); 4122 else 4123 stmmac_set_desc_addr(priv, desc, curr_addr); 4124 4125 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4126 TSO_MAX_BUFF_SIZE : tmp_len; 4127 4128 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4129 0, 1, 4130 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4131 0, 0); 4132 4133 tmp_len -= TSO_MAX_BUFF_SIZE; 4134 } 4135 } 4136 4137 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4138 { 4139 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4140 int desc_size; 4141 4142 if (likely(priv->extend_desc)) 4143 desc_size = sizeof(struct dma_extended_desc); 4144 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4145 desc_size = sizeof(struct dma_edesc); 4146 else 4147 desc_size = sizeof(struct dma_desc); 4148 4149 /* The own bit must be the latest setting done when prepare the 4150 * descriptor and then barrier is needed to make sure that 4151 * all is coherent before granting the DMA engine. 4152 */ 4153 wmb(); 4154 4155 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 4156 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 4157 } 4158 4159 /** 4160 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4161 * @skb : the socket buffer 4162 * @dev : device pointer 4163 * Description: this is the transmit function that is called on TSO frames 4164 * (support available on GMAC4 and newer chips). 4165 * Diagram below show the ring programming in case of TSO frames: 4166 * 4167 * First Descriptor 4168 * -------- 4169 * | DES0 |---> buffer1 = L2/L3/L4 header 4170 * | DES1 |---> TCP Payload (can continue on next descr...) 4171 * | DES2 |---> buffer 1 and 2 len 4172 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4173 * -------- 4174 * | 4175 * ... 4176 * | 4177 * -------- 4178 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 4179 * | DES1 | --| 4180 * | DES2 | --> buffer 1 and 2 len 4181 * | DES3 | 4182 * -------- 4183 * 4184 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4185 */ 4186 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4187 { 4188 struct dma_desc *desc, *first, *mss_desc = NULL; 4189 struct stmmac_priv *priv = netdev_priv(dev); 4190 int tmp_pay_len = 0, first_tx, nfrags; 4191 unsigned int first_entry, tx_packets; 4192 struct stmmac_txq_stats *txq_stats; 4193 struct stmmac_tx_queue *tx_q; 4194 u32 pay_len, mss, queue; 4195 dma_addr_t tso_des, des; 4196 u8 proto_hdr_len, hdr; 4197 bool set_ic; 4198 int i; 4199 4200 /* Always insert VLAN tag to SKB payload for TSO frames. 4201 * 4202 * Never insert VLAN tag by HW, since segments splited by 4203 * TSO engine will be un-tagged by mistake. 4204 */ 4205 if (skb_vlan_tag_present(skb)) { 4206 skb = __vlan_hwaccel_push_inside(skb); 4207 if (unlikely(!skb)) { 4208 priv->xstats.tx_dropped++; 4209 return NETDEV_TX_OK; 4210 } 4211 } 4212 4213 nfrags = skb_shinfo(skb)->nr_frags; 4214 queue = skb_get_queue_mapping(skb); 4215 4216 tx_q = &priv->dma_conf.tx_queue[queue]; 4217 txq_stats = &priv->xstats.txq_stats[queue]; 4218 first_tx = tx_q->cur_tx; 4219 4220 /* Compute header lengths */ 4221 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 4222 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 4223 hdr = sizeof(struct udphdr); 4224 } else { 4225 proto_hdr_len = skb_tcp_all_headers(skb); 4226 hdr = tcp_hdrlen(skb); 4227 } 4228 4229 /* Desc availability based on threshold should be enough safe */ 4230 if (unlikely(stmmac_tx_avail(priv, queue) < 4231 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4232 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4233 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4234 queue)); 4235 /* This is a hard error, log it. */ 4236 netdev_err(priv->dev, 4237 "%s: Tx Ring full when queue awake\n", 4238 __func__); 4239 } 4240 return NETDEV_TX_BUSY; 4241 } 4242 4243 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4244 4245 mss = skb_shinfo(skb)->gso_size; 4246 4247 /* set new MSS value if needed */ 4248 if (mss != tx_q->mss) { 4249 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4250 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4251 else 4252 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4253 4254 stmmac_set_mss(priv, mss_desc, mss); 4255 tx_q->mss = mss; 4256 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4257 priv->dma_conf.dma_tx_size); 4258 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4259 } 4260 4261 if (netif_msg_tx_queued(priv)) { 4262 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4263 __func__, hdr, proto_hdr_len, pay_len, mss); 4264 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4265 skb->data_len); 4266 } 4267 4268 first_entry = tx_q->cur_tx; 4269 WARN_ON(tx_q->tx_skbuff[first_entry]); 4270 4271 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4272 desc = &tx_q->dma_entx[first_entry].basic; 4273 else 4274 desc = &tx_q->dma_tx[first_entry]; 4275 first = desc; 4276 4277 /* first descriptor: fill Headers on Buf1 */ 4278 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4279 DMA_TO_DEVICE); 4280 if (dma_mapping_error(priv->device, des)) 4281 goto dma_map_err; 4282 4283 if (priv->dma_cap.addr64 <= 32) { 4284 first->des0 = cpu_to_le32(des); 4285 4286 /* Fill start of payload in buff2 of first descriptor */ 4287 if (pay_len) 4288 first->des1 = cpu_to_le32(des + proto_hdr_len); 4289 4290 /* If needed take extra descriptors to fill the remaining payload */ 4291 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 4292 tso_des = des; 4293 } else { 4294 stmmac_set_desc_addr(priv, first, des); 4295 tmp_pay_len = pay_len; 4296 tso_des = des + proto_hdr_len; 4297 pay_len = 0; 4298 } 4299 4300 stmmac_tso_allocator(priv, tso_des, tmp_pay_len, (nfrags == 0), queue); 4301 4302 /* In case two or more DMA transmit descriptors are allocated for this 4303 * non-paged SKB data, the DMA buffer address should be saved to 4304 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4305 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4306 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4307 * since the tail areas of the DMA buffer can be accessed by DMA engine 4308 * sooner or later. 4309 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4310 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4311 * this DMA buffer right after the DMA engine completely finishes the 4312 * full buffer transmission. 4313 */ 4314 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4315 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb); 4316 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false; 4317 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4318 4319 /* Prepare fragments */ 4320 for (i = 0; i < nfrags; i++) { 4321 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4322 4323 des = skb_frag_dma_map(priv->device, frag, 0, 4324 skb_frag_size(frag), 4325 DMA_TO_DEVICE); 4326 if (dma_mapping_error(priv->device, des)) 4327 goto dma_map_err; 4328 4329 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4330 (i == nfrags - 1), queue); 4331 4332 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4333 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4334 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4335 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4336 } 4337 4338 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4339 4340 /* Only the last descriptor gets to point to the skb. */ 4341 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4342 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4343 4344 /* Manage tx mitigation */ 4345 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4346 tx_q->tx_count_frames += tx_packets; 4347 4348 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4349 set_ic = true; 4350 else if (!priv->tx_coal_frames[queue]) 4351 set_ic = false; 4352 else if (tx_packets > priv->tx_coal_frames[queue]) 4353 set_ic = true; 4354 else if ((tx_q->tx_count_frames % 4355 priv->tx_coal_frames[queue]) < tx_packets) 4356 set_ic = true; 4357 else 4358 set_ic = false; 4359 4360 if (set_ic) { 4361 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4362 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4363 else 4364 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4365 4366 tx_q->tx_count_frames = 0; 4367 stmmac_set_tx_ic(priv, desc); 4368 } 4369 4370 /* We've used all descriptors we need for this skb, however, 4371 * advance cur_tx so that it references a fresh descriptor. 4372 * ndo_start_xmit will fill this descriptor the next time it's 4373 * called and stmmac_tx_clean may clean up to this descriptor. 4374 */ 4375 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4376 4377 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4378 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4379 __func__); 4380 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4381 } 4382 4383 u64_stats_update_begin(&txq_stats->q_syncp); 4384 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4385 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4386 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4387 if (set_ic) 4388 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4389 u64_stats_update_end(&txq_stats->q_syncp); 4390 4391 if (priv->sarc_type) 4392 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4393 4394 skb_tx_timestamp(skb); 4395 4396 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4397 priv->hwts_tx_en)) { 4398 /* declare that device is doing timestamping */ 4399 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4400 stmmac_enable_tx_timestamp(priv, first); 4401 } 4402 4403 /* Complete the first descriptor before granting the DMA */ 4404 stmmac_prepare_tso_tx_desc(priv, first, 1, 4405 proto_hdr_len, 4406 pay_len, 4407 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 4408 hdr / 4, (skb->len - proto_hdr_len)); 4409 4410 /* If context desc is used to change MSS */ 4411 if (mss_desc) { 4412 /* Make sure that first descriptor has been completely 4413 * written, including its own bit. This is because MSS is 4414 * actually before first descriptor, so we need to make 4415 * sure that MSS's own bit is the last thing written. 4416 */ 4417 dma_wmb(); 4418 stmmac_set_tx_owner(priv, mss_desc); 4419 } 4420 4421 if (netif_msg_pktdata(priv)) { 4422 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4423 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4424 tx_q->cur_tx, first, nfrags); 4425 pr_info(">>> frame to be transmitted: "); 4426 print_pkt(skb->data, skb_headlen(skb)); 4427 } 4428 4429 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4430 4431 stmmac_flush_tx_descriptors(priv, queue); 4432 stmmac_tx_timer_arm(priv, queue); 4433 4434 return NETDEV_TX_OK; 4435 4436 dma_map_err: 4437 dev_err(priv->device, "Tx dma map failed\n"); 4438 dev_kfree_skb(skb); 4439 priv->xstats.tx_dropped++; 4440 return NETDEV_TX_OK; 4441 } 4442 4443 /** 4444 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4445 * @skb: socket buffer to check 4446 * 4447 * Check if a packet has an ethertype that will trigger the IP header checks 4448 * and IP/TCP checksum engine of the stmmac core. 4449 * 4450 * Return: true if the ethertype can trigger the checksum engine, false 4451 * otherwise 4452 */ 4453 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4454 { 4455 int depth = 0; 4456 __be16 proto; 4457 4458 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4459 &depth); 4460 4461 return (depth <= ETH_HLEN) && 4462 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4463 } 4464 4465 /** 4466 * stmmac_xmit - Tx entry point of the driver 4467 * @skb : the socket buffer 4468 * @dev : device pointer 4469 * Description : this is the tx entry point of the driver. 4470 * It programs the chain or the ring and supports oversized frames 4471 * and SG feature. 4472 */ 4473 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4474 { 4475 unsigned int first_entry, tx_packets, enh_desc; 4476 struct stmmac_priv *priv = netdev_priv(dev); 4477 unsigned int nopaged_len = skb_headlen(skb); 4478 int i, csum_insertion = 0, is_jumbo = 0; 4479 u32 queue = skb_get_queue_mapping(skb); 4480 int nfrags = skb_shinfo(skb)->nr_frags; 4481 int gso = skb_shinfo(skb)->gso_type; 4482 struct stmmac_txq_stats *txq_stats; 4483 struct dma_edesc *tbs_desc = NULL; 4484 struct dma_desc *desc, *first; 4485 struct stmmac_tx_queue *tx_q; 4486 bool has_vlan, set_ic; 4487 int entry, first_tx; 4488 dma_addr_t des; 4489 4490 tx_q = &priv->dma_conf.tx_queue[queue]; 4491 txq_stats = &priv->xstats.txq_stats[queue]; 4492 first_tx = tx_q->cur_tx; 4493 4494 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4495 stmmac_disable_eee_mode(priv); 4496 4497 /* Manage oversized TCP frames for GMAC4 device */ 4498 if (skb_is_gso(skb) && priv->tso) { 4499 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4500 return stmmac_tso_xmit(skb, dev); 4501 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) 4502 return stmmac_tso_xmit(skb, dev); 4503 } 4504 4505 if (priv->est && priv->est->enable && 4506 priv->est->max_sdu[queue] && 4507 skb->len > priv->est->max_sdu[queue]){ 4508 priv->xstats.max_sdu_txq_drop[queue]++; 4509 goto max_sdu_err; 4510 } 4511 4512 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4513 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4514 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4515 queue)); 4516 /* This is a hard error, log it. */ 4517 netdev_err(priv->dev, 4518 "%s: Tx Ring full when queue awake\n", 4519 __func__); 4520 } 4521 return NETDEV_TX_BUSY; 4522 } 4523 4524 /* Check if VLAN can be inserted by HW */ 4525 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4526 4527 entry = tx_q->cur_tx; 4528 first_entry = entry; 4529 WARN_ON(tx_q->tx_skbuff[first_entry]); 4530 4531 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4532 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 4533 * queues. In that case, checksum offloading for those queues that don't 4534 * support tx coe needs to fallback to software checksum calculation. 4535 * 4536 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 4537 * also have to be checksummed in software. 4538 */ 4539 if (csum_insertion && 4540 (priv->plat->tx_queues_cfg[queue].coe_unsupported || 4541 !stmmac_has_ip_ethertype(skb))) { 4542 if (unlikely(skb_checksum_help(skb))) 4543 goto dma_map_err; 4544 csum_insertion = !csum_insertion; 4545 } 4546 4547 if (likely(priv->extend_desc)) 4548 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4549 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4550 desc = &tx_q->dma_entx[entry].basic; 4551 else 4552 desc = tx_q->dma_tx + entry; 4553 4554 first = desc; 4555 4556 if (has_vlan) 4557 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4558 4559 enh_desc = priv->plat->enh_desc; 4560 /* To program the descriptors according to the size of the frame */ 4561 if (enh_desc) 4562 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4563 4564 if (unlikely(is_jumbo)) { 4565 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4566 if (unlikely(entry < 0) && (entry != -EINVAL)) 4567 goto dma_map_err; 4568 } 4569 4570 for (i = 0; i < nfrags; i++) { 4571 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4572 int len = skb_frag_size(frag); 4573 bool last_segment = (i == (nfrags - 1)); 4574 4575 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4576 WARN_ON(tx_q->tx_skbuff[entry]); 4577 4578 if (likely(priv->extend_desc)) 4579 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4580 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4581 desc = &tx_q->dma_entx[entry].basic; 4582 else 4583 desc = tx_q->dma_tx + entry; 4584 4585 des = skb_frag_dma_map(priv->device, frag, 0, len, 4586 DMA_TO_DEVICE); 4587 if (dma_mapping_error(priv->device, des)) 4588 goto dma_map_err; /* should reuse desc w/o issues */ 4589 4590 tx_q->tx_skbuff_dma[entry].buf = des; 4591 4592 stmmac_set_desc_addr(priv, desc, des); 4593 4594 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4595 tx_q->tx_skbuff_dma[entry].len = len; 4596 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4597 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4598 4599 /* Prepare the descriptor and set the own bit too */ 4600 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4601 priv->mode, 1, last_segment, skb->len); 4602 } 4603 4604 /* Only the last descriptor gets to point to the skb. */ 4605 tx_q->tx_skbuff[entry] = skb; 4606 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4607 4608 /* According to the coalesce parameter the IC bit for the latest 4609 * segment is reset and the timer re-started to clean the tx status. 4610 * This approach takes care about the fragments: desc is the first 4611 * element in case of no SG. 4612 */ 4613 tx_packets = (entry + 1) - first_tx; 4614 tx_q->tx_count_frames += tx_packets; 4615 4616 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4617 set_ic = true; 4618 else if (!priv->tx_coal_frames[queue]) 4619 set_ic = false; 4620 else if (tx_packets > priv->tx_coal_frames[queue]) 4621 set_ic = true; 4622 else if ((tx_q->tx_count_frames % 4623 priv->tx_coal_frames[queue]) < tx_packets) 4624 set_ic = true; 4625 else 4626 set_ic = false; 4627 4628 if (set_ic) { 4629 if (likely(priv->extend_desc)) 4630 desc = &tx_q->dma_etx[entry].basic; 4631 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4632 desc = &tx_q->dma_entx[entry].basic; 4633 else 4634 desc = &tx_q->dma_tx[entry]; 4635 4636 tx_q->tx_count_frames = 0; 4637 stmmac_set_tx_ic(priv, desc); 4638 } 4639 4640 /* We've used all descriptors we need for this skb, however, 4641 * advance cur_tx so that it references a fresh descriptor. 4642 * ndo_start_xmit will fill this descriptor the next time it's 4643 * called and stmmac_tx_clean may clean up to this descriptor. 4644 */ 4645 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4646 tx_q->cur_tx = entry; 4647 4648 if (netif_msg_pktdata(priv)) { 4649 netdev_dbg(priv->dev, 4650 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4651 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4652 entry, first, nfrags); 4653 4654 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4655 print_pkt(skb->data, skb->len); 4656 } 4657 4658 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4659 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4660 __func__); 4661 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4662 } 4663 4664 u64_stats_update_begin(&txq_stats->q_syncp); 4665 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4666 if (set_ic) 4667 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4668 u64_stats_update_end(&txq_stats->q_syncp); 4669 4670 if (priv->sarc_type) 4671 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4672 4673 skb_tx_timestamp(skb); 4674 4675 /* Ready to fill the first descriptor and set the OWN bit w/o any 4676 * problems because all the descriptors are actually ready to be 4677 * passed to the DMA engine. 4678 */ 4679 if (likely(!is_jumbo)) { 4680 bool last_segment = (nfrags == 0); 4681 4682 des = dma_map_single(priv->device, skb->data, 4683 nopaged_len, DMA_TO_DEVICE); 4684 if (dma_mapping_error(priv->device, des)) 4685 goto dma_map_err; 4686 4687 tx_q->tx_skbuff_dma[first_entry].buf = des; 4688 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4689 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4690 4691 stmmac_set_desc_addr(priv, first, des); 4692 4693 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4694 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4695 4696 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4697 priv->hwts_tx_en)) { 4698 /* declare that device is doing timestamping */ 4699 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4700 stmmac_enable_tx_timestamp(priv, first); 4701 } 4702 4703 /* Prepare the first descriptor setting the OWN bit too */ 4704 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4705 csum_insertion, priv->mode, 0, last_segment, 4706 skb->len); 4707 } 4708 4709 if (tx_q->tbs & STMMAC_TBS_EN) { 4710 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4711 4712 tbs_desc = &tx_q->dma_entx[first_entry]; 4713 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4714 } 4715 4716 stmmac_set_tx_owner(priv, first); 4717 4718 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4719 4720 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 4721 4722 stmmac_flush_tx_descriptors(priv, queue); 4723 stmmac_tx_timer_arm(priv, queue); 4724 4725 return NETDEV_TX_OK; 4726 4727 dma_map_err: 4728 netdev_err(priv->dev, "Tx DMA map failed\n"); 4729 max_sdu_err: 4730 dev_kfree_skb(skb); 4731 priv->xstats.tx_dropped++; 4732 return NETDEV_TX_OK; 4733 } 4734 4735 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4736 { 4737 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 4738 __be16 vlan_proto = veth->h_vlan_proto; 4739 u16 vlanid; 4740 4741 if ((vlan_proto == htons(ETH_P_8021Q) && 4742 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4743 (vlan_proto == htons(ETH_P_8021AD) && 4744 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4745 /* pop the vlan tag */ 4746 vlanid = ntohs(veth->h_vlan_TCI); 4747 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4748 skb_pull(skb, VLAN_HLEN); 4749 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4750 } 4751 } 4752 4753 /** 4754 * stmmac_rx_refill - refill used skb preallocated buffers 4755 * @priv: driver private structure 4756 * @queue: RX queue index 4757 * Description : this is to reallocate the skb for the reception process 4758 * that is based on zero-copy. 4759 */ 4760 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4761 { 4762 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4763 int dirty = stmmac_rx_dirty(priv, queue); 4764 unsigned int entry = rx_q->dirty_rx; 4765 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4766 4767 if (priv->dma_cap.host_dma_width <= 32) 4768 gfp |= GFP_DMA32; 4769 4770 while (dirty-- > 0) { 4771 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4772 struct dma_desc *p; 4773 bool use_rx_wd; 4774 4775 if (priv->extend_desc) 4776 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4777 else 4778 p = rx_q->dma_rx + entry; 4779 4780 if (!buf->page) { 4781 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4782 if (!buf->page) 4783 break; 4784 } 4785 4786 if (priv->sph && !buf->sec_page) { 4787 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4788 if (!buf->sec_page) 4789 break; 4790 4791 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4792 } 4793 4794 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4795 4796 stmmac_set_desc_addr(priv, p, buf->addr); 4797 if (priv->sph) 4798 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4799 else 4800 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4801 stmmac_refill_desc3(priv, rx_q, p); 4802 4803 rx_q->rx_count_frames++; 4804 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4805 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4806 rx_q->rx_count_frames = 0; 4807 4808 use_rx_wd = !priv->rx_coal_frames[queue]; 4809 use_rx_wd |= rx_q->rx_count_frames > 0; 4810 if (!priv->use_riwt) 4811 use_rx_wd = false; 4812 4813 dma_wmb(); 4814 stmmac_set_rx_owner(priv, p, use_rx_wd); 4815 4816 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 4817 } 4818 rx_q->dirty_rx = entry; 4819 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4820 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4821 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4822 } 4823 4824 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4825 struct dma_desc *p, 4826 int status, unsigned int len) 4827 { 4828 unsigned int plen = 0, hlen = 0; 4829 int coe = priv->hw->rx_csum; 4830 4831 /* Not first descriptor, buffer is always zero */ 4832 if (priv->sph && len) 4833 return 0; 4834 4835 /* First descriptor, get split header length */ 4836 stmmac_get_rx_header_len(priv, p, &hlen); 4837 if (priv->sph && hlen) { 4838 priv->xstats.rx_split_hdr_pkt_n++; 4839 return hlen; 4840 } 4841 4842 /* First descriptor, not last descriptor and not split header */ 4843 if (status & rx_not_ls) 4844 return priv->dma_conf.dma_buf_sz; 4845 4846 plen = stmmac_get_rx_frame_len(priv, p, coe); 4847 4848 /* First descriptor and last descriptor and not split header */ 4849 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 4850 } 4851 4852 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4853 struct dma_desc *p, 4854 int status, unsigned int len) 4855 { 4856 int coe = priv->hw->rx_csum; 4857 unsigned int plen = 0; 4858 4859 /* Not split header, buffer is not available */ 4860 if (!priv->sph) 4861 return 0; 4862 4863 /* Not last descriptor */ 4864 if (status & rx_not_ls) 4865 return priv->dma_conf.dma_buf_sz; 4866 4867 plen = stmmac_get_rx_frame_len(priv, p, coe); 4868 4869 /* Last descriptor */ 4870 return plen - len; 4871 } 4872 4873 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 4874 struct xdp_frame *xdpf, bool dma_map) 4875 { 4876 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 4877 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4878 unsigned int entry = tx_q->cur_tx; 4879 struct dma_desc *tx_desc; 4880 dma_addr_t dma_addr; 4881 bool set_ic; 4882 4883 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 4884 return STMMAC_XDP_CONSUMED; 4885 4886 if (priv->est && priv->est->enable && 4887 priv->est->max_sdu[queue] && 4888 xdpf->len > priv->est->max_sdu[queue]) { 4889 priv->xstats.max_sdu_txq_drop[queue]++; 4890 return STMMAC_XDP_CONSUMED; 4891 } 4892 4893 if (likely(priv->extend_desc)) 4894 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4895 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4896 tx_desc = &tx_q->dma_entx[entry].basic; 4897 else 4898 tx_desc = tx_q->dma_tx + entry; 4899 4900 if (dma_map) { 4901 dma_addr = dma_map_single(priv->device, xdpf->data, 4902 xdpf->len, DMA_TO_DEVICE); 4903 if (dma_mapping_error(priv->device, dma_addr)) 4904 return STMMAC_XDP_CONSUMED; 4905 4906 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 4907 } else { 4908 struct page *page = virt_to_page(xdpf->data); 4909 4910 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 4911 xdpf->headroom; 4912 dma_sync_single_for_device(priv->device, dma_addr, 4913 xdpf->len, DMA_BIDIRECTIONAL); 4914 4915 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 4916 } 4917 4918 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 4919 tx_q->tx_skbuff_dma[entry].map_as_page = false; 4920 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 4921 tx_q->tx_skbuff_dma[entry].last_segment = true; 4922 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 4923 4924 tx_q->xdpf[entry] = xdpf; 4925 4926 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 4927 4928 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 4929 true, priv->mode, true, true, 4930 xdpf->len); 4931 4932 tx_q->tx_count_frames++; 4933 4934 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 4935 set_ic = true; 4936 else 4937 set_ic = false; 4938 4939 if (set_ic) { 4940 tx_q->tx_count_frames = 0; 4941 stmmac_set_tx_ic(priv, tx_desc); 4942 u64_stats_update_begin(&txq_stats->q_syncp); 4943 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4944 u64_stats_update_end(&txq_stats->q_syncp); 4945 } 4946 4947 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 4948 4949 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4950 tx_q->cur_tx = entry; 4951 4952 return STMMAC_XDP_TX; 4953 } 4954 4955 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 4956 int cpu) 4957 { 4958 int index = cpu; 4959 4960 if (unlikely(index < 0)) 4961 index = 0; 4962 4963 while (index >= priv->plat->tx_queues_to_use) 4964 index -= priv->plat->tx_queues_to_use; 4965 4966 return index; 4967 } 4968 4969 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 4970 struct xdp_buff *xdp) 4971 { 4972 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 4973 int cpu = smp_processor_id(); 4974 struct netdev_queue *nq; 4975 int queue; 4976 int res; 4977 4978 if (unlikely(!xdpf)) 4979 return STMMAC_XDP_CONSUMED; 4980 4981 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4982 nq = netdev_get_tx_queue(priv->dev, queue); 4983 4984 __netif_tx_lock(nq, cpu); 4985 /* Avoids TX time-out as we are sharing with slow path */ 4986 txq_trans_cond_update(nq); 4987 4988 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); 4989 if (res == STMMAC_XDP_TX) 4990 stmmac_flush_tx_descriptors(priv, queue); 4991 4992 __netif_tx_unlock(nq); 4993 4994 return res; 4995 } 4996 4997 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 4998 struct bpf_prog *prog, 4999 struct xdp_buff *xdp) 5000 { 5001 u32 act; 5002 int res; 5003 5004 act = bpf_prog_run_xdp(prog, xdp); 5005 switch (act) { 5006 case XDP_PASS: 5007 res = STMMAC_XDP_PASS; 5008 break; 5009 case XDP_TX: 5010 res = stmmac_xdp_xmit_back(priv, xdp); 5011 break; 5012 case XDP_REDIRECT: 5013 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 5014 res = STMMAC_XDP_CONSUMED; 5015 else 5016 res = STMMAC_XDP_REDIRECT; 5017 break; 5018 default: 5019 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5020 fallthrough; 5021 case XDP_ABORTED: 5022 trace_xdp_exception(priv->dev, prog, act); 5023 fallthrough; 5024 case XDP_DROP: 5025 res = STMMAC_XDP_CONSUMED; 5026 break; 5027 } 5028 5029 return res; 5030 } 5031 5032 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5033 struct xdp_buff *xdp) 5034 { 5035 struct bpf_prog *prog; 5036 int res; 5037 5038 prog = READ_ONCE(priv->xdp_prog); 5039 if (!prog) { 5040 res = STMMAC_XDP_PASS; 5041 goto out; 5042 } 5043 5044 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5045 out: 5046 return ERR_PTR(-res); 5047 } 5048 5049 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5050 int xdp_status) 5051 { 5052 int cpu = smp_processor_id(); 5053 int queue; 5054 5055 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5056 5057 if (xdp_status & STMMAC_XDP_TX) 5058 stmmac_tx_timer_arm(priv, queue); 5059 5060 if (xdp_status & STMMAC_XDP_REDIRECT) 5061 xdp_do_flush(); 5062 } 5063 5064 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5065 struct xdp_buff *xdp) 5066 { 5067 unsigned int metasize = xdp->data - xdp->data_meta; 5068 unsigned int datasize = xdp->data_end - xdp->data; 5069 struct sk_buff *skb; 5070 5071 skb = napi_alloc_skb(&ch->rxtx_napi, 5072 xdp->data_end - xdp->data_hard_start); 5073 if (unlikely(!skb)) 5074 return NULL; 5075 5076 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5077 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5078 if (metasize) 5079 skb_metadata_set(skb, metasize); 5080 5081 return skb; 5082 } 5083 5084 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5085 struct dma_desc *p, struct dma_desc *np, 5086 struct xdp_buff *xdp) 5087 { 5088 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5089 struct stmmac_channel *ch = &priv->channel[queue]; 5090 unsigned int len = xdp->data_end - xdp->data; 5091 enum pkt_hash_types hash_type; 5092 int coe = priv->hw->rx_csum; 5093 struct sk_buff *skb; 5094 u32 hash; 5095 5096 skb = stmmac_construct_skb_zc(ch, xdp); 5097 if (!skb) { 5098 priv->xstats.rx_dropped++; 5099 return; 5100 } 5101 5102 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5103 if (priv->hw->hw_vlan_en) 5104 /* MAC level stripping. */ 5105 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5106 else 5107 /* Driver level stripping. */ 5108 stmmac_rx_vlan(priv->dev, skb); 5109 skb->protocol = eth_type_trans(skb, priv->dev); 5110 5111 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5112 skb_checksum_none_assert(skb); 5113 else 5114 skb->ip_summed = CHECKSUM_UNNECESSARY; 5115 5116 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5117 skb_set_hash(skb, hash, hash_type); 5118 5119 skb_record_rx_queue(skb, queue); 5120 napi_gro_receive(&ch->rxtx_napi, skb); 5121 5122 u64_stats_update_begin(&rxq_stats->napi_syncp); 5123 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5124 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5125 u64_stats_update_end(&rxq_stats->napi_syncp); 5126 } 5127 5128 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5129 { 5130 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5131 unsigned int entry = rx_q->dirty_rx; 5132 struct dma_desc *rx_desc = NULL; 5133 bool ret = true; 5134 5135 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5136 5137 while (budget-- > 0 && entry != rx_q->cur_rx) { 5138 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5139 dma_addr_t dma_addr; 5140 bool use_rx_wd; 5141 5142 if (!buf->xdp) { 5143 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5144 if (!buf->xdp) { 5145 ret = false; 5146 break; 5147 } 5148 } 5149 5150 if (priv->extend_desc) 5151 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 5152 else 5153 rx_desc = rx_q->dma_rx + entry; 5154 5155 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5156 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5157 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5158 stmmac_refill_desc3(priv, rx_q, rx_desc); 5159 5160 rx_q->rx_count_frames++; 5161 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5162 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5163 rx_q->rx_count_frames = 0; 5164 5165 use_rx_wd = !priv->rx_coal_frames[queue]; 5166 use_rx_wd |= rx_q->rx_count_frames > 0; 5167 if (!priv->use_riwt) 5168 use_rx_wd = false; 5169 5170 dma_wmb(); 5171 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5172 5173 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 5174 } 5175 5176 if (rx_desc) { 5177 rx_q->dirty_rx = entry; 5178 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 5179 (rx_q->dirty_rx * sizeof(struct dma_desc)); 5180 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 5181 } 5182 5183 return ret; 5184 } 5185 5186 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5187 { 5188 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5189 * to represent incoming packet, whereas cb field in the same structure 5190 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5191 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5192 */ 5193 return (struct stmmac_xdp_buff *)xdp; 5194 } 5195 5196 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5197 { 5198 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5199 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5200 unsigned int count = 0, error = 0, len = 0; 5201 int dirty = stmmac_rx_dirty(priv, queue); 5202 unsigned int next_entry = rx_q->cur_rx; 5203 u32 rx_errors = 0, rx_dropped = 0; 5204 unsigned int desc_size; 5205 struct bpf_prog *prog; 5206 bool failure = false; 5207 int xdp_status = 0; 5208 int status = 0; 5209 5210 if (netif_msg_rx_status(priv)) { 5211 void *rx_head; 5212 5213 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5214 if (priv->extend_desc) { 5215 rx_head = (void *)rx_q->dma_erx; 5216 desc_size = sizeof(struct dma_extended_desc); 5217 } else { 5218 rx_head = (void *)rx_q->dma_rx; 5219 desc_size = sizeof(struct dma_desc); 5220 } 5221 5222 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5223 rx_q->dma_rx_phy, desc_size); 5224 } 5225 while (count < limit) { 5226 struct stmmac_rx_buffer *buf; 5227 struct stmmac_xdp_buff *ctx; 5228 unsigned int buf1_len = 0; 5229 struct dma_desc *np, *p; 5230 int entry; 5231 int res; 5232 5233 if (!count && rx_q->state_saved) { 5234 error = rx_q->state.error; 5235 len = rx_q->state.len; 5236 } else { 5237 rx_q->state_saved = false; 5238 error = 0; 5239 len = 0; 5240 } 5241 5242 if (count >= limit) 5243 break; 5244 5245 read_again: 5246 buf1_len = 0; 5247 entry = next_entry; 5248 buf = &rx_q->buf_pool[entry]; 5249 5250 if (dirty >= STMMAC_RX_FILL_BATCH) { 5251 failure = failure || 5252 !stmmac_rx_refill_zc(priv, queue, dirty); 5253 dirty = 0; 5254 } 5255 5256 if (priv->extend_desc) 5257 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5258 else 5259 p = rx_q->dma_rx + entry; 5260 5261 /* read the status of the incoming frame */ 5262 status = stmmac_rx_status(priv, &priv->xstats, p); 5263 /* check if managed by the DMA otherwise go ahead */ 5264 if (unlikely(status & dma_own)) 5265 break; 5266 5267 /* Prefetch the next RX descriptor */ 5268 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5269 priv->dma_conf.dma_rx_size); 5270 next_entry = rx_q->cur_rx; 5271 5272 if (priv->extend_desc) 5273 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5274 else 5275 np = rx_q->dma_rx + next_entry; 5276 5277 prefetch(np); 5278 5279 /* Ensure a valid XSK buffer before proceed */ 5280 if (!buf->xdp) 5281 break; 5282 5283 if (priv->extend_desc) 5284 stmmac_rx_extended_status(priv, &priv->xstats, 5285 rx_q->dma_erx + entry); 5286 if (unlikely(status == discard_frame)) { 5287 xsk_buff_free(buf->xdp); 5288 buf->xdp = NULL; 5289 dirty++; 5290 error = 1; 5291 if (!priv->hwts_rx_en) 5292 rx_errors++; 5293 } 5294 5295 if (unlikely(error && (status & rx_not_ls))) 5296 goto read_again; 5297 if (unlikely(error)) { 5298 count++; 5299 continue; 5300 } 5301 5302 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5303 if (likely(status & rx_not_ls)) { 5304 xsk_buff_free(buf->xdp); 5305 buf->xdp = NULL; 5306 dirty++; 5307 count++; 5308 goto read_again; 5309 } 5310 5311 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5312 ctx->priv = priv; 5313 ctx->desc = p; 5314 ctx->ndesc = np; 5315 5316 /* XDP ZC Frame only support primary buffers for now */ 5317 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5318 len += buf1_len; 5319 5320 /* ACS is disabled; strip manually. */ 5321 if (likely(!(status & rx_not_ls))) { 5322 buf1_len -= ETH_FCS_LEN; 5323 len -= ETH_FCS_LEN; 5324 } 5325 5326 /* RX buffer is good and fit into a XSK pool buffer */ 5327 buf->xdp->data_end = buf->xdp->data + buf1_len; 5328 xsk_buff_dma_sync_for_cpu(buf->xdp); 5329 5330 prog = READ_ONCE(priv->xdp_prog); 5331 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5332 5333 switch (res) { 5334 case STMMAC_XDP_PASS: 5335 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5336 xsk_buff_free(buf->xdp); 5337 break; 5338 case STMMAC_XDP_CONSUMED: 5339 xsk_buff_free(buf->xdp); 5340 rx_dropped++; 5341 break; 5342 case STMMAC_XDP_TX: 5343 case STMMAC_XDP_REDIRECT: 5344 xdp_status |= res; 5345 break; 5346 } 5347 5348 buf->xdp = NULL; 5349 dirty++; 5350 count++; 5351 } 5352 5353 if (status & rx_not_ls) { 5354 rx_q->state_saved = true; 5355 rx_q->state.error = error; 5356 rx_q->state.len = len; 5357 } 5358 5359 stmmac_finalize_xdp_rx(priv, xdp_status); 5360 5361 u64_stats_update_begin(&rxq_stats->napi_syncp); 5362 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5363 u64_stats_update_end(&rxq_stats->napi_syncp); 5364 5365 priv->xstats.rx_dropped += rx_dropped; 5366 priv->xstats.rx_errors += rx_errors; 5367 5368 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5369 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5370 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5371 else 5372 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5373 5374 return (int)count; 5375 } 5376 5377 return failure ? limit : (int)count; 5378 } 5379 5380 /** 5381 * stmmac_rx - manage the receive process 5382 * @priv: driver private structure 5383 * @limit: napi bugget 5384 * @queue: RX queue index. 5385 * Description : this the function called by the napi poll method. 5386 * It gets all the frames inside the ring. 5387 */ 5388 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5389 { 5390 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5391 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5392 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5393 struct stmmac_channel *ch = &priv->channel[queue]; 5394 unsigned int count = 0, error = 0, len = 0; 5395 int status = 0, coe = priv->hw->rx_csum; 5396 unsigned int next_entry = rx_q->cur_rx; 5397 enum dma_data_direction dma_dir; 5398 unsigned int desc_size; 5399 struct sk_buff *skb = NULL; 5400 struct stmmac_xdp_buff ctx; 5401 int xdp_status = 0; 5402 int buf_sz; 5403 5404 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5405 buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5406 limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); 5407 5408 if (netif_msg_rx_status(priv)) { 5409 void *rx_head; 5410 5411 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5412 if (priv->extend_desc) { 5413 rx_head = (void *)rx_q->dma_erx; 5414 desc_size = sizeof(struct dma_extended_desc); 5415 } else { 5416 rx_head = (void *)rx_q->dma_rx; 5417 desc_size = sizeof(struct dma_desc); 5418 } 5419 5420 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5421 rx_q->dma_rx_phy, desc_size); 5422 } 5423 while (count < limit) { 5424 unsigned int buf1_len = 0, buf2_len = 0; 5425 enum pkt_hash_types hash_type; 5426 struct stmmac_rx_buffer *buf; 5427 struct dma_desc *np, *p; 5428 int entry; 5429 u32 hash; 5430 5431 if (!count && rx_q->state_saved) { 5432 skb = rx_q->state.skb; 5433 error = rx_q->state.error; 5434 len = rx_q->state.len; 5435 } else { 5436 rx_q->state_saved = false; 5437 skb = NULL; 5438 error = 0; 5439 len = 0; 5440 } 5441 5442 read_again: 5443 if (count >= limit) 5444 break; 5445 5446 buf1_len = 0; 5447 buf2_len = 0; 5448 entry = next_entry; 5449 buf = &rx_q->buf_pool[entry]; 5450 5451 if (priv->extend_desc) 5452 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5453 else 5454 p = rx_q->dma_rx + entry; 5455 5456 /* read the status of the incoming frame */ 5457 status = stmmac_rx_status(priv, &priv->xstats, p); 5458 /* check if managed by the DMA otherwise go ahead */ 5459 if (unlikely(status & dma_own)) 5460 break; 5461 5462 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5463 priv->dma_conf.dma_rx_size); 5464 next_entry = rx_q->cur_rx; 5465 5466 if (priv->extend_desc) 5467 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5468 else 5469 np = rx_q->dma_rx + next_entry; 5470 5471 prefetch(np); 5472 5473 if (priv->extend_desc) 5474 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5475 if (unlikely(status == discard_frame)) { 5476 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5477 buf->page = NULL; 5478 error = 1; 5479 if (!priv->hwts_rx_en) 5480 rx_errors++; 5481 } 5482 5483 if (unlikely(error && (status & rx_not_ls))) 5484 goto read_again; 5485 if (unlikely(error)) { 5486 dev_kfree_skb(skb); 5487 skb = NULL; 5488 count++; 5489 continue; 5490 } 5491 5492 /* Buffer is good. Go on. */ 5493 5494 prefetch(page_address(buf->page) + buf->page_offset); 5495 if (buf->sec_page) 5496 prefetch(page_address(buf->sec_page)); 5497 5498 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5499 len += buf1_len; 5500 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5501 len += buf2_len; 5502 5503 /* ACS is disabled; strip manually. */ 5504 if (likely(!(status & rx_not_ls))) { 5505 if (buf2_len) { 5506 buf2_len -= ETH_FCS_LEN; 5507 len -= ETH_FCS_LEN; 5508 } else if (buf1_len) { 5509 buf1_len -= ETH_FCS_LEN; 5510 len -= ETH_FCS_LEN; 5511 } 5512 } 5513 5514 if (!skb) { 5515 unsigned int pre_len, sync_len; 5516 5517 dma_sync_single_for_cpu(priv->device, buf->addr, 5518 buf1_len, dma_dir); 5519 5520 xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq); 5521 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5522 buf->page_offset, buf1_len, true); 5523 5524 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5525 buf->page_offset; 5526 5527 ctx.priv = priv; 5528 ctx.desc = p; 5529 ctx.ndesc = np; 5530 5531 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5532 /* Due xdp_adjust_tail: DMA sync for_device 5533 * cover max len CPU touch 5534 */ 5535 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5536 buf->page_offset; 5537 sync_len = max(sync_len, pre_len); 5538 5539 /* For Not XDP_PASS verdict */ 5540 if (IS_ERR(skb)) { 5541 unsigned int xdp_res = -PTR_ERR(skb); 5542 5543 if (xdp_res & STMMAC_XDP_CONSUMED) { 5544 page_pool_put_page(rx_q->page_pool, 5545 virt_to_head_page(ctx.xdp.data), 5546 sync_len, true); 5547 buf->page = NULL; 5548 rx_dropped++; 5549 5550 /* Clear skb as it was set as 5551 * status by XDP program. 5552 */ 5553 skb = NULL; 5554 5555 if (unlikely((status & rx_not_ls))) 5556 goto read_again; 5557 5558 count++; 5559 continue; 5560 } else if (xdp_res & (STMMAC_XDP_TX | 5561 STMMAC_XDP_REDIRECT)) { 5562 xdp_status |= xdp_res; 5563 buf->page = NULL; 5564 skb = NULL; 5565 count++; 5566 continue; 5567 } 5568 } 5569 } 5570 5571 if (!skb) { 5572 /* XDP program may expand or reduce tail */ 5573 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5574 5575 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); 5576 if (!skb) { 5577 rx_dropped++; 5578 count++; 5579 goto drain_data; 5580 } 5581 5582 /* XDP program may adjust header */ 5583 skb_copy_to_linear_data(skb, ctx.xdp.data, buf1_len); 5584 skb_put(skb, buf1_len); 5585 5586 /* Data payload copied into SKB, page ready for recycle */ 5587 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5588 buf->page = NULL; 5589 } else if (buf1_len) { 5590 dma_sync_single_for_cpu(priv->device, buf->addr, 5591 buf1_len, dma_dir); 5592 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5593 buf->page, buf->page_offset, buf1_len, 5594 priv->dma_conf.dma_buf_sz); 5595 5596 /* Data payload appended into SKB */ 5597 skb_mark_for_recycle(skb); 5598 buf->page = NULL; 5599 } 5600 5601 if (buf2_len) { 5602 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5603 buf2_len, dma_dir); 5604 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5605 buf->sec_page, 0, buf2_len, 5606 priv->dma_conf.dma_buf_sz); 5607 5608 /* Data payload appended into SKB */ 5609 skb_mark_for_recycle(skb); 5610 buf->sec_page = NULL; 5611 } 5612 5613 drain_data: 5614 if (likely(status & rx_not_ls)) 5615 goto read_again; 5616 if (!skb) 5617 continue; 5618 5619 /* Got entire packet into SKB. Finish it. */ 5620 5621 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5622 5623 if (priv->hw->hw_vlan_en) 5624 /* MAC level stripping. */ 5625 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5626 else 5627 /* Driver level stripping. */ 5628 stmmac_rx_vlan(priv->dev, skb); 5629 5630 skb->protocol = eth_type_trans(skb, priv->dev); 5631 5632 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5633 skb_checksum_none_assert(skb); 5634 else 5635 skb->ip_summed = CHECKSUM_UNNECESSARY; 5636 5637 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5638 skb_set_hash(skb, hash, hash_type); 5639 5640 skb_record_rx_queue(skb, queue); 5641 napi_gro_receive(&ch->rx_napi, skb); 5642 skb = NULL; 5643 5644 rx_packets++; 5645 rx_bytes += len; 5646 count++; 5647 } 5648 5649 if (status & rx_not_ls || skb) { 5650 rx_q->state_saved = true; 5651 rx_q->state.skb = skb; 5652 rx_q->state.error = error; 5653 rx_q->state.len = len; 5654 } 5655 5656 stmmac_finalize_xdp_rx(priv, xdp_status); 5657 5658 stmmac_rx_refill(priv, queue); 5659 5660 u64_stats_update_begin(&rxq_stats->napi_syncp); 5661 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5662 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5663 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5664 u64_stats_update_end(&rxq_stats->napi_syncp); 5665 5666 priv->xstats.rx_dropped += rx_dropped; 5667 priv->xstats.rx_errors += rx_errors; 5668 5669 return count; 5670 } 5671 5672 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5673 { 5674 struct stmmac_channel *ch = 5675 container_of(napi, struct stmmac_channel, rx_napi); 5676 struct stmmac_priv *priv = ch->priv_data; 5677 struct stmmac_rxq_stats *rxq_stats; 5678 u32 chan = ch->index; 5679 int work_done; 5680 5681 rxq_stats = &priv->xstats.rxq_stats[chan]; 5682 u64_stats_update_begin(&rxq_stats->napi_syncp); 5683 u64_stats_inc(&rxq_stats->napi.poll); 5684 u64_stats_update_end(&rxq_stats->napi_syncp); 5685 5686 work_done = stmmac_rx(priv, budget, chan); 5687 if (work_done < budget && napi_complete_done(napi, work_done)) { 5688 unsigned long flags; 5689 5690 spin_lock_irqsave(&ch->lock, flags); 5691 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5692 spin_unlock_irqrestore(&ch->lock, flags); 5693 } 5694 5695 return work_done; 5696 } 5697 5698 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5699 { 5700 struct stmmac_channel *ch = 5701 container_of(napi, struct stmmac_channel, tx_napi); 5702 struct stmmac_priv *priv = ch->priv_data; 5703 struct stmmac_txq_stats *txq_stats; 5704 bool pending_packets = false; 5705 u32 chan = ch->index; 5706 int work_done; 5707 5708 txq_stats = &priv->xstats.txq_stats[chan]; 5709 u64_stats_update_begin(&txq_stats->napi_syncp); 5710 u64_stats_inc(&txq_stats->napi.poll); 5711 u64_stats_update_end(&txq_stats->napi_syncp); 5712 5713 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 5714 work_done = min(work_done, budget); 5715 5716 if (work_done < budget && napi_complete_done(napi, work_done)) { 5717 unsigned long flags; 5718 5719 spin_lock_irqsave(&ch->lock, flags); 5720 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5721 spin_unlock_irqrestore(&ch->lock, flags); 5722 } 5723 5724 /* TX still have packet to handle, check if we need to arm tx timer */ 5725 if (pending_packets) 5726 stmmac_tx_timer_arm(priv, chan); 5727 5728 return work_done; 5729 } 5730 5731 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5732 { 5733 struct stmmac_channel *ch = 5734 container_of(napi, struct stmmac_channel, rxtx_napi); 5735 struct stmmac_priv *priv = ch->priv_data; 5736 bool tx_pending_packets = false; 5737 int rx_done, tx_done, rxtx_done; 5738 struct stmmac_rxq_stats *rxq_stats; 5739 struct stmmac_txq_stats *txq_stats; 5740 u32 chan = ch->index; 5741 5742 rxq_stats = &priv->xstats.rxq_stats[chan]; 5743 u64_stats_update_begin(&rxq_stats->napi_syncp); 5744 u64_stats_inc(&rxq_stats->napi.poll); 5745 u64_stats_update_end(&rxq_stats->napi_syncp); 5746 5747 txq_stats = &priv->xstats.txq_stats[chan]; 5748 u64_stats_update_begin(&txq_stats->napi_syncp); 5749 u64_stats_inc(&txq_stats->napi.poll); 5750 u64_stats_update_end(&txq_stats->napi_syncp); 5751 5752 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 5753 tx_done = min(tx_done, budget); 5754 5755 rx_done = stmmac_rx_zc(priv, budget, chan); 5756 5757 rxtx_done = max(tx_done, rx_done); 5758 5759 /* If either TX or RX work is not complete, return budget 5760 * and keep pooling 5761 */ 5762 if (rxtx_done >= budget) 5763 return budget; 5764 5765 /* all work done, exit the polling mode */ 5766 if (napi_complete_done(napi, rxtx_done)) { 5767 unsigned long flags; 5768 5769 spin_lock_irqsave(&ch->lock, flags); 5770 /* Both RX and TX work done are compelte, 5771 * so enable both RX & TX IRQs. 5772 */ 5773 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5774 spin_unlock_irqrestore(&ch->lock, flags); 5775 } 5776 5777 /* TX still have packet to handle, check if we need to arm tx timer */ 5778 if (tx_pending_packets) 5779 stmmac_tx_timer_arm(priv, chan); 5780 5781 return min(rxtx_done, budget - 1); 5782 } 5783 5784 /** 5785 * stmmac_tx_timeout 5786 * @dev : Pointer to net device structure 5787 * @txqueue: the index of the hanging transmit queue 5788 * Description: this function is called when a packet transmission fails to 5789 * complete within a reasonable time. The driver will mark the error in the 5790 * netdev structure and arrange for the device to be reset to a sane state 5791 * in order to transmit a new packet. 5792 */ 5793 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5794 { 5795 struct stmmac_priv *priv = netdev_priv(dev); 5796 5797 stmmac_global_err(priv); 5798 } 5799 5800 /** 5801 * stmmac_set_rx_mode - entry point for multicast addressing 5802 * @dev : pointer to the device structure 5803 * Description: 5804 * This function is a driver entry point which gets called by the kernel 5805 * whenever multicast addresses must be enabled/disabled. 5806 * Return value: 5807 * void. 5808 */ 5809 static void stmmac_set_rx_mode(struct net_device *dev) 5810 { 5811 struct stmmac_priv *priv = netdev_priv(dev); 5812 5813 stmmac_set_filter(priv, priv->hw, dev); 5814 } 5815 5816 /** 5817 * stmmac_change_mtu - entry point to change MTU size for the device. 5818 * @dev : device pointer. 5819 * @new_mtu : the new MTU size for the device. 5820 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5821 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5822 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5823 * Return value: 5824 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5825 * file on failure. 5826 */ 5827 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5828 { 5829 struct stmmac_priv *priv = netdev_priv(dev); 5830 int txfifosz = priv->plat->tx_fifo_size; 5831 struct stmmac_dma_conf *dma_conf; 5832 const int mtu = new_mtu; 5833 int ret; 5834 5835 if (txfifosz == 0) 5836 txfifosz = priv->dma_cap.tx_fifo_size; 5837 5838 txfifosz /= priv->plat->tx_queues_to_use; 5839 5840 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5841 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 5842 return -EINVAL; 5843 } 5844 5845 new_mtu = STMMAC_ALIGN(new_mtu); 5846 5847 /* If condition true, FIFO is too small or MTU too large */ 5848 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 5849 return -EINVAL; 5850 5851 if (netif_running(dev)) { 5852 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 5853 /* Try to allocate the new DMA conf with the new mtu */ 5854 dma_conf = stmmac_setup_dma_desc(priv, mtu); 5855 if (IS_ERR(dma_conf)) { 5856 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 5857 mtu); 5858 return PTR_ERR(dma_conf); 5859 } 5860 5861 stmmac_release(dev); 5862 5863 ret = __stmmac_open(dev, dma_conf); 5864 if (ret) { 5865 free_dma_desc_resources(priv, dma_conf); 5866 kfree(dma_conf); 5867 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 5868 return ret; 5869 } 5870 5871 kfree(dma_conf); 5872 5873 stmmac_set_rx_mode(dev); 5874 } 5875 5876 WRITE_ONCE(dev->mtu, mtu); 5877 netdev_update_features(dev); 5878 5879 return 0; 5880 } 5881 5882 static netdev_features_t stmmac_fix_features(struct net_device *dev, 5883 netdev_features_t features) 5884 { 5885 struct stmmac_priv *priv = netdev_priv(dev); 5886 5887 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 5888 features &= ~NETIF_F_RXCSUM; 5889 5890 if (!priv->plat->tx_coe) 5891 features &= ~NETIF_F_CSUM_MASK; 5892 5893 /* Some GMAC devices have a bugged Jumbo frame support that 5894 * needs to have the Tx COE disabled for oversized frames 5895 * (due to limited buffer sizes). In this case we disable 5896 * the TX csum insertion in the TDES and not use SF. 5897 */ 5898 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 5899 features &= ~NETIF_F_CSUM_MASK; 5900 5901 /* Disable tso if asked by ethtool */ 5902 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 5903 if (features & NETIF_F_TSO) 5904 priv->tso = true; 5905 else 5906 priv->tso = false; 5907 } 5908 5909 return features; 5910 } 5911 5912 static int stmmac_set_features(struct net_device *netdev, 5913 netdev_features_t features) 5914 { 5915 struct stmmac_priv *priv = netdev_priv(netdev); 5916 5917 /* Keep the COE Type in case of csum is supporting */ 5918 if (features & NETIF_F_RXCSUM) 5919 priv->hw->rx_csum = priv->plat->rx_coe; 5920 else 5921 priv->hw->rx_csum = 0; 5922 /* No check needed because rx_coe has been set before and it will be 5923 * fixed in case of issue. 5924 */ 5925 stmmac_rx_ipc(priv, priv->hw); 5926 5927 if (priv->sph_cap) { 5928 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; 5929 u32 chan; 5930 5931 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 5932 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 5933 } 5934 5935 if (features & NETIF_F_HW_VLAN_CTAG_RX) 5936 priv->hw->hw_vlan_en = true; 5937 else 5938 priv->hw->hw_vlan_en = false; 5939 5940 stmmac_set_hw_vlan_mode(priv, priv->hw); 5941 5942 return 0; 5943 } 5944 5945 static void stmmac_common_interrupt(struct stmmac_priv *priv) 5946 { 5947 u32 rx_cnt = priv->plat->rx_queues_to_use; 5948 u32 tx_cnt = priv->plat->tx_queues_to_use; 5949 u32 queues_count; 5950 u32 queue; 5951 bool xmac; 5952 5953 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 5954 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 5955 5956 if (priv->irq_wake) 5957 pm_wakeup_event(priv->device, 0); 5958 5959 if (priv->dma_cap.estsel) 5960 stmmac_est_irq_status(priv, priv, priv->dev, 5961 &priv->xstats, tx_cnt); 5962 5963 if (stmmac_fpe_supported(priv)) 5964 stmmac_fpe_irq_status(priv); 5965 5966 /* To handle GMAC own interrupts */ 5967 if ((priv->plat->has_gmac) || xmac) { 5968 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 5969 5970 if (unlikely(status)) { 5971 /* For LPI we need to save the tx status */ 5972 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 5973 priv->tx_path_in_lpi_mode = true; 5974 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 5975 priv->tx_path_in_lpi_mode = false; 5976 } 5977 5978 for (queue = 0; queue < queues_count; queue++) 5979 stmmac_host_mtl_irq_status(priv, priv->hw, queue); 5980 5981 /* PCS link status */ 5982 if (priv->hw->pcs && 5983 !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { 5984 if (priv->xstats.pcs_link) 5985 netif_carrier_on(priv->dev); 5986 else 5987 netif_carrier_off(priv->dev); 5988 } 5989 5990 stmmac_timestamp_interrupt(priv, priv); 5991 } 5992 } 5993 5994 /** 5995 * stmmac_interrupt - main ISR 5996 * @irq: interrupt number. 5997 * @dev_id: to pass the net device pointer. 5998 * Description: this is the main driver interrupt service routine. 5999 * It can call: 6000 * o DMA service routine (to manage incoming frame reception and transmission 6001 * status) 6002 * o Core interrupts to manage: remote wake-up, management counter, LPI 6003 * interrupts. 6004 */ 6005 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6006 { 6007 struct net_device *dev = (struct net_device *)dev_id; 6008 struct stmmac_priv *priv = netdev_priv(dev); 6009 6010 /* Check if adapter is up */ 6011 if (test_bit(STMMAC_DOWN, &priv->state)) 6012 return IRQ_HANDLED; 6013 6014 /* Check ASP error if it isn't delivered via an individual IRQ */ 6015 if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) 6016 return IRQ_HANDLED; 6017 6018 /* To handle Common interrupts */ 6019 stmmac_common_interrupt(priv); 6020 6021 /* To handle DMA interrupts */ 6022 stmmac_dma_interrupt(priv); 6023 6024 return IRQ_HANDLED; 6025 } 6026 6027 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6028 { 6029 struct net_device *dev = (struct net_device *)dev_id; 6030 struct stmmac_priv *priv = netdev_priv(dev); 6031 6032 /* Check if adapter is up */ 6033 if (test_bit(STMMAC_DOWN, &priv->state)) 6034 return IRQ_HANDLED; 6035 6036 /* To handle Common interrupts */ 6037 stmmac_common_interrupt(priv); 6038 6039 return IRQ_HANDLED; 6040 } 6041 6042 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6043 { 6044 struct net_device *dev = (struct net_device *)dev_id; 6045 struct stmmac_priv *priv = netdev_priv(dev); 6046 6047 /* Check if adapter is up */ 6048 if (test_bit(STMMAC_DOWN, &priv->state)) 6049 return IRQ_HANDLED; 6050 6051 /* Check if a fatal error happened */ 6052 stmmac_safety_feat_interrupt(priv); 6053 6054 return IRQ_HANDLED; 6055 } 6056 6057 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6058 { 6059 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6060 struct stmmac_dma_conf *dma_conf; 6061 int chan = tx_q->queue_index; 6062 struct stmmac_priv *priv; 6063 int status; 6064 6065 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6066 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6067 6068 /* Check if adapter is up */ 6069 if (test_bit(STMMAC_DOWN, &priv->state)) 6070 return IRQ_HANDLED; 6071 6072 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6073 6074 if (unlikely(status & tx_hard_error_bump_tc)) { 6075 /* Try to bump up the dma threshold on this failure */ 6076 stmmac_bump_dma_threshold(priv, chan); 6077 } else if (unlikely(status == tx_hard_error)) { 6078 stmmac_tx_err(priv, chan); 6079 } 6080 6081 return IRQ_HANDLED; 6082 } 6083 6084 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6085 { 6086 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6087 struct stmmac_dma_conf *dma_conf; 6088 int chan = rx_q->queue_index; 6089 struct stmmac_priv *priv; 6090 6091 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6092 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6093 6094 /* Check if adapter is up */ 6095 if (test_bit(STMMAC_DOWN, &priv->state)) 6096 return IRQ_HANDLED; 6097 6098 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6099 6100 return IRQ_HANDLED; 6101 } 6102 6103 /** 6104 * stmmac_ioctl - Entry point for the Ioctl 6105 * @dev: Device pointer. 6106 * @rq: An IOCTL specefic structure, that can contain a pointer to 6107 * a proprietary structure used to pass information to the driver. 6108 * @cmd: IOCTL command 6109 * Description: 6110 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6111 */ 6112 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6113 { 6114 struct stmmac_priv *priv = netdev_priv (dev); 6115 int ret = -EOPNOTSUPP; 6116 6117 if (!netif_running(dev)) 6118 return -EINVAL; 6119 6120 switch (cmd) { 6121 case SIOCGMIIPHY: 6122 case SIOCGMIIREG: 6123 case SIOCSMIIREG: 6124 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6125 break; 6126 case SIOCSHWTSTAMP: 6127 ret = stmmac_hwtstamp_set(dev, rq); 6128 break; 6129 case SIOCGHWTSTAMP: 6130 ret = stmmac_hwtstamp_get(dev, rq); 6131 break; 6132 default: 6133 break; 6134 } 6135 6136 return ret; 6137 } 6138 6139 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6140 void *cb_priv) 6141 { 6142 struct stmmac_priv *priv = cb_priv; 6143 int ret = -EOPNOTSUPP; 6144 6145 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6146 return ret; 6147 6148 __stmmac_disable_all_queues(priv); 6149 6150 switch (type) { 6151 case TC_SETUP_CLSU32: 6152 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6153 break; 6154 case TC_SETUP_CLSFLOWER: 6155 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6156 break; 6157 default: 6158 break; 6159 } 6160 6161 stmmac_enable_all_queues(priv); 6162 return ret; 6163 } 6164 6165 static LIST_HEAD(stmmac_block_cb_list); 6166 6167 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6168 void *type_data) 6169 { 6170 struct stmmac_priv *priv = netdev_priv(ndev); 6171 6172 switch (type) { 6173 case TC_QUERY_CAPS: 6174 return stmmac_tc_query_caps(priv, priv, type_data); 6175 case TC_SETUP_QDISC_MQPRIO: 6176 return stmmac_tc_setup_mqprio(priv, priv, type_data); 6177 case TC_SETUP_BLOCK: 6178 return flow_block_cb_setup_simple(type_data, 6179 &stmmac_block_cb_list, 6180 stmmac_setup_tc_block_cb, 6181 priv, priv, true); 6182 case TC_SETUP_QDISC_CBS: 6183 return stmmac_tc_setup_cbs(priv, priv, type_data); 6184 case TC_SETUP_QDISC_TAPRIO: 6185 return stmmac_tc_setup_taprio(priv, priv, type_data); 6186 case TC_SETUP_QDISC_ETF: 6187 return stmmac_tc_setup_etf(priv, priv, type_data); 6188 default: 6189 return -EOPNOTSUPP; 6190 } 6191 } 6192 6193 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6194 struct net_device *sb_dev) 6195 { 6196 int gso = skb_shinfo(skb)->gso_type; 6197 6198 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6199 /* 6200 * There is no way to determine the number of TSO/USO 6201 * capable Queues. Let's use always the Queue 0 6202 * because if TSO/USO is supported then at least this 6203 * one will be capable. 6204 */ 6205 return 0; 6206 } 6207 6208 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6209 } 6210 6211 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6212 { 6213 struct stmmac_priv *priv = netdev_priv(ndev); 6214 int ret = 0; 6215 6216 ret = pm_runtime_resume_and_get(priv->device); 6217 if (ret < 0) 6218 return ret; 6219 6220 ret = eth_mac_addr(ndev, addr); 6221 if (ret) 6222 goto set_mac_error; 6223 6224 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6225 6226 set_mac_error: 6227 pm_runtime_put(priv->device); 6228 6229 return ret; 6230 } 6231 6232 #ifdef CONFIG_DEBUG_FS 6233 static struct dentry *stmmac_fs_dir; 6234 6235 static void sysfs_display_ring(void *head, int size, int extend_desc, 6236 struct seq_file *seq, dma_addr_t dma_phy_addr) 6237 { 6238 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6239 struct dma_desc *p = (struct dma_desc *)head; 6240 unsigned int desc_size; 6241 dma_addr_t dma_addr; 6242 int i; 6243 6244 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6245 for (i = 0; i < size; i++) { 6246 dma_addr = dma_phy_addr + i * desc_size; 6247 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6248 i, &dma_addr, 6249 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6250 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6251 if (extend_desc) 6252 p = &(++ep)->basic; 6253 else 6254 p++; 6255 } 6256 } 6257 6258 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6259 { 6260 struct net_device *dev = seq->private; 6261 struct stmmac_priv *priv = netdev_priv(dev); 6262 u32 rx_count = priv->plat->rx_queues_to_use; 6263 u32 tx_count = priv->plat->tx_queues_to_use; 6264 u32 queue; 6265 6266 if ((dev->flags & IFF_UP) == 0) 6267 return 0; 6268 6269 for (queue = 0; queue < rx_count; queue++) { 6270 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6271 6272 seq_printf(seq, "RX Queue %d:\n", queue); 6273 6274 if (priv->extend_desc) { 6275 seq_printf(seq, "Extended descriptor ring:\n"); 6276 sysfs_display_ring((void *)rx_q->dma_erx, 6277 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6278 } else { 6279 seq_printf(seq, "Descriptor ring:\n"); 6280 sysfs_display_ring((void *)rx_q->dma_rx, 6281 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6282 } 6283 } 6284 6285 for (queue = 0; queue < tx_count; queue++) { 6286 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6287 6288 seq_printf(seq, "TX Queue %d:\n", queue); 6289 6290 if (priv->extend_desc) { 6291 seq_printf(seq, "Extended descriptor ring:\n"); 6292 sysfs_display_ring((void *)tx_q->dma_etx, 6293 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6294 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6295 seq_printf(seq, "Descriptor ring:\n"); 6296 sysfs_display_ring((void *)tx_q->dma_tx, 6297 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6298 } 6299 } 6300 6301 return 0; 6302 } 6303 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6304 6305 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6306 { 6307 static const char * const dwxgmac_timestamp_source[] = { 6308 "None", 6309 "Internal", 6310 "External", 6311 "Both", 6312 }; 6313 static const char * const dwxgmac_safety_feature_desc[] = { 6314 "No", 6315 "All Safety Features with ECC and Parity", 6316 "All Safety Features without ECC or Parity", 6317 "All Safety Features with Parity Only", 6318 "ECC Only", 6319 "UNDEFINED", 6320 "UNDEFINED", 6321 "UNDEFINED", 6322 }; 6323 struct net_device *dev = seq->private; 6324 struct stmmac_priv *priv = netdev_priv(dev); 6325 6326 if (!priv->hw_cap_support) { 6327 seq_printf(seq, "DMA HW features not supported\n"); 6328 return 0; 6329 } 6330 6331 seq_printf(seq, "==============================\n"); 6332 seq_printf(seq, "\tDMA HW features\n"); 6333 seq_printf(seq, "==============================\n"); 6334 6335 seq_printf(seq, "\t10/100 Mbps: %s\n", 6336 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6337 seq_printf(seq, "\t1000 Mbps: %s\n", 6338 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6339 seq_printf(seq, "\tHalf duplex: %s\n", 6340 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6341 if (priv->plat->has_xgmac) { 6342 seq_printf(seq, 6343 "\tNumber of Additional MAC address registers: %d\n", 6344 priv->dma_cap.multi_addr); 6345 } else { 6346 seq_printf(seq, "\tHash Filter: %s\n", 6347 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6348 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6349 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6350 } 6351 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6352 (priv->dma_cap.pcs) ? "Y" : "N"); 6353 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6354 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6355 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6356 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6357 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6358 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6359 seq_printf(seq, "\tRMON module: %s\n", 6360 (priv->dma_cap.rmon) ? "Y" : "N"); 6361 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6362 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6363 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6364 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6365 if (priv->plat->has_xgmac) 6366 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6367 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6368 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6369 (priv->dma_cap.eee) ? "Y" : "N"); 6370 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6371 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6372 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6373 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6374 priv->plat->has_xgmac) { 6375 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6376 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6377 } else { 6378 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6379 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6380 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6381 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6382 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6383 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6384 } 6385 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6386 priv->dma_cap.number_rx_channel); 6387 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6388 priv->dma_cap.number_tx_channel); 6389 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6390 priv->dma_cap.number_rx_queues); 6391 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6392 priv->dma_cap.number_tx_queues); 6393 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6394 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6395 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6396 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6397 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6398 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6399 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6400 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6401 priv->dma_cap.pps_out_num); 6402 seq_printf(seq, "\tSafety Features: %s\n", 6403 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6404 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6405 priv->dma_cap.frpsel ? "Y" : "N"); 6406 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6407 priv->dma_cap.host_dma_width); 6408 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6409 priv->dma_cap.rssen ? "Y" : "N"); 6410 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6411 priv->dma_cap.vlhash ? "Y" : "N"); 6412 seq_printf(seq, "\tSplit Header: %s\n", 6413 priv->dma_cap.sphen ? "Y" : "N"); 6414 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6415 priv->dma_cap.vlins ? "Y" : "N"); 6416 seq_printf(seq, "\tDouble VLAN: %s\n", 6417 priv->dma_cap.dvlan ? "Y" : "N"); 6418 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6419 priv->dma_cap.l3l4fnum); 6420 seq_printf(seq, "\tARP Offloading: %s\n", 6421 priv->dma_cap.arpoffsel ? "Y" : "N"); 6422 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6423 priv->dma_cap.estsel ? "Y" : "N"); 6424 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6425 priv->dma_cap.fpesel ? "Y" : "N"); 6426 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6427 priv->dma_cap.tbssel ? "Y" : "N"); 6428 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6429 priv->dma_cap.tbs_ch_num); 6430 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6431 priv->dma_cap.sgfsel ? "Y" : "N"); 6432 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6433 BIT(priv->dma_cap.ttsfd) >> 1); 6434 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6435 priv->dma_cap.numtc); 6436 seq_printf(seq, "\tDCB Feature: %s\n", 6437 priv->dma_cap.dcben ? "Y" : "N"); 6438 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6439 priv->dma_cap.advthword ? "Y" : "N"); 6440 seq_printf(seq, "\tPTP Offload: %s\n", 6441 priv->dma_cap.ptoen ? "Y" : "N"); 6442 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6443 priv->dma_cap.osten ? "Y" : "N"); 6444 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6445 priv->dma_cap.pfcen ? "Y" : "N"); 6446 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6447 BIT(priv->dma_cap.frpes) << 6); 6448 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6449 BIT(priv->dma_cap.frpbs) << 6); 6450 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6451 priv->dma_cap.frppipe_num); 6452 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6453 priv->dma_cap.nrvf_num ? 6454 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6455 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6456 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6457 seq_printf(seq, "\tDepth of GCL: %lu\n", 6458 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6459 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6460 priv->dma_cap.cbtisel ? "Y" : "N"); 6461 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6462 priv->dma_cap.aux_snapshot_n); 6463 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6464 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6465 seq_printf(seq, "\tEnhanced DMA: %s\n", 6466 priv->dma_cap.edma ? "Y" : "N"); 6467 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6468 priv->dma_cap.ediffc ? "Y" : "N"); 6469 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6470 priv->dma_cap.vxn ? "Y" : "N"); 6471 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6472 priv->dma_cap.dbgmem ? "Y" : "N"); 6473 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6474 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6475 return 0; 6476 } 6477 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6478 6479 /* Use network device events to rename debugfs file entries. 6480 */ 6481 static int stmmac_device_event(struct notifier_block *unused, 6482 unsigned long event, void *ptr) 6483 { 6484 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6485 struct stmmac_priv *priv = netdev_priv(dev); 6486 6487 if (dev->netdev_ops != &stmmac_netdev_ops) 6488 goto done; 6489 6490 switch (event) { 6491 case NETDEV_CHANGENAME: 6492 if (priv->dbgfs_dir) 6493 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, 6494 priv->dbgfs_dir, 6495 stmmac_fs_dir, 6496 dev->name); 6497 break; 6498 } 6499 done: 6500 return NOTIFY_DONE; 6501 } 6502 6503 static struct notifier_block stmmac_notifier = { 6504 .notifier_call = stmmac_device_event, 6505 }; 6506 6507 static void stmmac_init_fs(struct net_device *dev) 6508 { 6509 struct stmmac_priv *priv = netdev_priv(dev); 6510 6511 rtnl_lock(); 6512 6513 /* Create per netdev entries */ 6514 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6515 6516 /* Entry to report DMA RX/TX rings */ 6517 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6518 &stmmac_rings_status_fops); 6519 6520 /* Entry to report the DMA HW features */ 6521 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6522 &stmmac_dma_cap_fops); 6523 6524 rtnl_unlock(); 6525 } 6526 6527 static void stmmac_exit_fs(struct net_device *dev) 6528 { 6529 struct stmmac_priv *priv = netdev_priv(dev); 6530 6531 debugfs_remove_recursive(priv->dbgfs_dir); 6532 } 6533 #endif /* CONFIG_DEBUG_FS */ 6534 6535 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6536 { 6537 unsigned char *data = (unsigned char *)&vid_le; 6538 unsigned char data_byte = 0; 6539 u32 crc = ~0x0; 6540 u32 temp = 0; 6541 int i, bits; 6542 6543 bits = get_bitmask_order(VLAN_VID_MASK); 6544 for (i = 0; i < bits; i++) { 6545 if ((i % 8) == 0) 6546 data_byte = data[i / 8]; 6547 6548 temp = ((crc & 1) ^ data_byte) & 1; 6549 crc >>= 1; 6550 data_byte >>= 1; 6551 6552 if (temp) 6553 crc ^= 0xedb88320; 6554 } 6555 6556 return crc; 6557 } 6558 6559 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6560 { 6561 u32 crc, hash = 0; 6562 u16 pmatch = 0; 6563 int count = 0; 6564 u16 vid = 0; 6565 6566 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6567 __le16 vid_le = cpu_to_le16(vid); 6568 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6569 hash |= (1 << crc); 6570 count++; 6571 } 6572 6573 if (!priv->dma_cap.vlhash) { 6574 if (count > 2) /* VID = 0 always passes filter */ 6575 return -EOPNOTSUPP; 6576 6577 pmatch = vid; 6578 hash = 0; 6579 } 6580 6581 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6582 } 6583 6584 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6585 { 6586 struct stmmac_priv *priv = netdev_priv(ndev); 6587 bool is_double = false; 6588 int ret; 6589 6590 ret = pm_runtime_resume_and_get(priv->device); 6591 if (ret < 0) 6592 return ret; 6593 6594 if (be16_to_cpu(proto) == ETH_P_8021AD) 6595 is_double = true; 6596 6597 set_bit(vid, priv->active_vlans); 6598 ret = stmmac_vlan_update(priv, is_double); 6599 if (ret) { 6600 clear_bit(vid, priv->active_vlans); 6601 goto err_pm_put; 6602 } 6603 6604 if (priv->hw->num_vlan) { 6605 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6606 if (ret) 6607 goto err_pm_put; 6608 } 6609 err_pm_put: 6610 pm_runtime_put(priv->device); 6611 6612 return ret; 6613 } 6614 6615 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6616 { 6617 struct stmmac_priv *priv = netdev_priv(ndev); 6618 bool is_double = false; 6619 int ret; 6620 6621 ret = pm_runtime_resume_and_get(priv->device); 6622 if (ret < 0) 6623 return ret; 6624 6625 if (be16_to_cpu(proto) == ETH_P_8021AD) 6626 is_double = true; 6627 6628 clear_bit(vid, priv->active_vlans); 6629 6630 if (priv->hw->num_vlan) { 6631 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6632 if (ret) 6633 goto del_vlan_error; 6634 } 6635 6636 ret = stmmac_vlan_update(priv, is_double); 6637 6638 del_vlan_error: 6639 pm_runtime_put(priv->device); 6640 6641 return ret; 6642 } 6643 6644 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6645 { 6646 struct stmmac_priv *priv = netdev_priv(dev); 6647 6648 switch (bpf->command) { 6649 case XDP_SETUP_PROG: 6650 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6651 case XDP_SETUP_XSK_POOL: 6652 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6653 bpf->xsk.queue_id); 6654 default: 6655 return -EOPNOTSUPP; 6656 } 6657 } 6658 6659 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6660 struct xdp_frame **frames, u32 flags) 6661 { 6662 struct stmmac_priv *priv = netdev_priv(dev); 6663 int cpu = smp_processor_id(); 6664 struct netdev_queue *nq; 6665 int i, nxmit = 0; 6666 int queue; 6667 6668 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6669 return -ENETDOWN; 6670 6671 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6672 return -EINVAL; 6673 6674 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6675 nq = netdev_get_tx_queue(priv->dev, queue); 6676 6677 __netif_tx_lock(nq, cpu); 6678 /* Avoids TX time-out as we are sharing with slow path */ 6679 txq_trans_cond_update(nq); 6680 6681 for (i = 0; i < num_frames; i++) { 6682 int res; 6683 6684 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6685 if (res == STMMAC_XDP_CONSUMED) 6686 break; 6687 6688 nxmit++; 6689 } 6690 6691 if (flags & XDP_XMIT_FLUSH) { 6692 stmmac_flush_tx_descriptors(priv, queue); 6693 stmmac_tx_timer_arm(priv, queue); 6694 } 6695 6696 __netif_tx_unlock(nq); 6697 6698 return nxmit; 6699 } 6700 6701 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6702 { 6703 struct stmmac_channel *ch = &priv->channel[queue]; 6704 unsigned long flags; 6705 6706 spin_lock_irqsave(&ch->lock, flags); 6707 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6708 spin_unlock_irqrestore(&ch->lock, flags); 6709 6710 stmmac_stop_rx_dma(priv, queue); 6711 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6712 } 6713 6714 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6715 { 6716 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6717 struct stmmac_channel *ch = &priv->channel[queue]; 6718 unsigned long flags; 6719 u32 buf_size; 6720 int ret; 6721 6722 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6723 if (ret) { 6724 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6725 return; 6726 } 6727 6728 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 6729 if (ret) { 6730 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6731 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6732 return; 6733 } 6734 6735 stmmac_reset_rx_queue(priv, queue); 6736 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 6737 6738 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6739 rx_q->dma_rx_phy, rx_q->queue_index); 6740 6741 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6742 sizeof(struct dma_desc)); 6743 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6744 rx_q->rx_tail_addr, rx_q->queue_index); 6745 6746 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6747 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6748 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6749 buf_size, 6750 rx_q->queue_index); 6751 } else { 6752 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6753 priv->dma_conf.dma_buf_sz, 6754 rx_q->queue_index); 6755 } 6756 6757 stmmac_start_rx_dma(priv, queue); 6758 6759 spin_lock_irqsave(&ch->lock, flags); 6760 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6761 spin_unlock_irqrestore(&ch->lock, flags); 6762 } 6763 6764 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6765 { 6766 struct stmmac_channel *ch = &priv->channel[queue]; 6767 unsigned long flags; 6768 6769 spin_lock_irqsave(&ch->lock, flags); 6770 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6771 spin_unlock_irqrestore(&ch->lock, flags); 6772 6773 stmmac_stop_tx_dma(priv, queue); 6774 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6775 } 6776 6777 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6778 { 6779 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6780 struct stmmac_channel *ch = &priv->channel[queue]; 6781 unsigned long flags; 6782 int ret; 6783 6784 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6785 if (ret) { 6786 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6787 return; 6788 } 6789 6790 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 6791 if (ret) { 6792 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6793 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6794 return; 6795 } 6796 6797 stmmac_reset_tx_queue(priv, queue); 6798 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 6799 6800 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6801 tx_q->dma_tx_phy, tx_q->queue_index); 6802 6803 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6804 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6805 6806 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6807 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6808 tx_q->tx_tail_addr, tx_q->queue_index); 6809 6810 stmmac_start_tx_dma(priv, queue); 6811 6812 spin_lock_irqsave(&ch->lock, flags); 6813 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6814 spin_unlock_irqrestore(&ch->lock, flags); 6815 } 6816 6817 void stmmac_xdp_release(struct net_device *dev) 6818 { 6819 struct stmmac_priv *priv = netdev_priv(dev); 6820 u32 chan; 6821 6822 /* Ensure tx function is not running */ 6823 netif_tx_disable(dev); 6824 6825 /* Disable NAPI process */ 6826 stmmac_disable_all_queues(priv); 6827 6828 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6829 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6830 6831 /* Free the IRQ lines */ 6832 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 6833 6834 /* Stop TX/RX DMA channels */ 6835 stmmac_stop_all_dma(priv); 6836 6837 /* Release and free the Rx/Tx resources */ 6838 free_dma_desc_resources(priv, &priv->dma_conf); 6839 6840 /* Disable the MAC Rx/Tx */ 6841 stmmac_mac_set(priv, priv->ioaddr, false); 6842 6843 /* set trans_start so we don't get spurious 6844 * watchdogs during reset 6845 */ 6846 netif_trans_update(dev); 6847 netif_carrier_off(dev); 6848 } 6849 6850 int stmmac_xdp_open(struct net_device *dev) 6851 { 6852 struct stmmac_priv *priv = netdev_priv(dev); 6853 u32 rx_cnt = priv->plat->rx_queues_to_use; 6854 u32 tx_cnt = priv->plat->tx_queues_to_use; 6855 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 6856 struct stmmac_rx_queue *rx_q; 6857 struct stmmac_tx_queue *tx_q; 6858 u32 buf_size; 6859 bool sph_en; 6860 u32 chan; 6861 int ret; 6862 6863 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 6864 if (ret < 0) { 6865 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 6866 __func__); 6867 goto dma_desc_error; 6868 } 6869 6870 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 6871 if (ret < 0) { 6872 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 6873 __func__); 6874 goto init_error; 6875 } 6876 6877 stmmac_reset_queues_param(priv); 6878 6879 /* DMA CSR Channel configuration */ 6880 for (chan = 0; chan < dma_csr_ch; chan++) { 6881 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 6882 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6883 } 6884 6885 /* Adjust Split header */ 6886 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 6887 6888 /* DMA RX Channel Configuration */ 6889 for (chan = 0; chan < rx_cnt; chan++) { 6890 rx_q = &priv->dma_conf.rx_queue[chan]; 6891 6892 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6893 rx_q->dma_rx_phy, chan); 6894 6895 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 6896 (rx_q->buf_alloc_num * 6897 sizeof(struct dma_desc)); 6898 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6899 rx_q->rx_tail_addr, chan); 6900 6901 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6902 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6903 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6904 buf_size, 6905 rx_q->queue_index); 6906 } else { 6907 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6908 priv->dma_conf.dma_buf_sz, 6909 rx_q->queue_index); 6910 } 6911 6912 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6913 } 6914 6915 /* DMA TX Channel Configuration */ 6916 for (chan = 0; chan < tx_cnt; chan++) { 6917 tx_q = &priv->dma_conf.tx_queue[chan]; 6918 6919 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6920 tx_q->dma_tx_phy, chan); 6921 6922 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6923 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6924 tx_q->tx_tail_addr, chan); 6925 6926 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6927 tx_q->txtimer.function = stmmac_tx_timer; 6928 } 6929 6930 /* Enable the MAC Rx/Tx */ 6931 stmmac_mac_set(priv, priv->ioaddr, true); 6932 6933 /* Start Rx & Tx DMA Channels */ 6934 stmmac_start_all_dma(priv); 6935 6936 ret = stmmac_request_irq(dev); 6937 if (ret) 6938 goto irq_error; 6939 6940 /* Enable NAPI process*/ 6941 stmmac_enable_all_queues(priv); 6942 netif_carrier_on(dev); 6943 netif_tx_start_all_queues(dev); 6944 stmmac_enable_all_dma_irq(priv); 6945 6946 return 0; 6947 6948 irq_error: 6949 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6950 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6951 6952 stmmac_hw_teardown(dev); 6953 init_error: 6954 free_dma_desc_resources(priv, &priv->dma_conf); 6955 dma_desc_error: 6956 return ret; 6957 } 6958 6959 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 6960 { 6961 struct stmmac_priv *priv = netdev_priv(dev); 6962 struct stmmac_rx_queue *rx_q; 6963 struct stmmac_tx_queue *tx_q; 6964 struct stmmac_channel *ch; 6965 6966 if (test_bit(STMMAC_DOWN, &priv->state) || 6967 !netif_carrier_ok(priv->dev)) 6968 return -ENETDOWN; 6969 6970 if (!stmmac_xdp_is_enabled(priv)) 6971 return -EINVAL; 6972 6973 if (queue >= priv->plat->rx_queues_to_use || 6974 queue >= priv->plat->tx_queues_to_use) 6975 return -EINVAL; 6976 6977 rx_q = &priv->dma_conf.rx_queue[queue]; 6978 tx_q = &priv->dma_conf.tx_queue[queue]; 6979 ch = &priv->channel[queue]; 6980 6981 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 6982 return -EINVAL; 6983 6984 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 6985 /* EQoS does not have per-DMA channel SW interrupt, 6986 * so we schedule RX Napi straight-away. 6987 */ 6988 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 6989 __napi_schedule(&ch->rxtx_napi); 6990 } 6991 6992 return 0; 6993 } 6994 6995 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 6996 { 6997 struct stmmac_priv *priv = netdev_priv(dev); 6998 u32 tx_cnt = priv->plat->tx_queues_to_use; 6999 u32 rx_cnt = priv->plat->rx_queues_to_use; 7000 unsigned int start; 7001 int q; 7002 7003 for (q = 0; q < tx_cnt; q++) { 7004 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7005 u64 tx_packets; 7006 u64 tx_bytes; 7007 7008 do { 7009 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 7010 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 7011 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 7012 do { 7013 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 7014 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 7015 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 7016 7017 stats->tx_packets += tx_packets; 7018 stats->tx_bytes += tx_bytes; 7019 } 7020 7021 for (q = 0; q < rx_cnt; q++) { 7022 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7023 u64 rx_packets; 7024 u64 rx_bytes; 7025 7026 do { 7027 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 7028 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 7029 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 7030 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 7031 7032 stats->rx_packets += rx_packets; 7033 stats->rx_bytes += rx_bytes; 7034 } 7035 7036 stats->rx_dropped = priv->xstats.rx_dropped; 7037 stats->rx_errors = priv->xstats.rx_errors; 7038 stats->tx_dropped = priv->xstats.tx_dropped; 7039 stats->tx_errors = priv->xstats.tx_errors; 7040 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7041 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7042 stats->rx_length_errors = priv->xstats.rx_length; 7043 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7044 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7045 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7046 } 7047 7048 static const struct net_device_ops stmmac_netdev_ops = { 7049 .ndo_open = stmmac_open, 7050 .ndo_start_xmit = stmmac_xmit, 7051 .ndo_stop = stmmac_release, 7052 .ndo_change_mtu = stmmac_change_mtu, 7053 .ndo_fix_features = stmmac_fix_features, 7054 .ndo_set_features = stmmac_set_features, 7055 .ndo_set_rx_mode = stmmac_set_rx_mode, 7056 .ndo_tx_timeout = stmmac_tx_timeout, 7057 .ndo_eth_ioctl = stmmac_ioctl, 7058 .ndo_get_stats64 = stmmac_get_stats64, 7059 .ndo_setup_tc = stmmac_setup_tc, 7060 .ndo_select_queue = stmmac_select_queue, 7061 .ndo_set_mac_address = stmmac_set_mac_address, 7062 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7063 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7064 .ndo_bpf = stmmac_bpf, 7065 .ndo_xdp_xmit = stmmac_xdp_xmit, 7066 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7067 }; 7068 7069 static void stmmac_reset_subtask(struct stmmac_priv *priv) 7070 { 7071 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7072 return; 7073 if (test_bit(STMMAC_DOWN, &priv->state)) 7074 return; 7075 7076 netdev_err(priv->dev, "Reset adapter.\n"); 7077 7078 rtnl_lock(); 7079 netif_trans_update(priv->dev); 7080 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7081 usleep_range(1000, 2000); 7082 7083 set_bit(STMMAC_DOWN, &priv->state); 7084 dev_close(priv->dev); 7085 dev_open(priv->dev, NULL); 7086 clear_bit(STMMAC_DOWN, &priv->state); 7087 clear_bit(STMMAC_RESETING, &priv->state); 7088 rtnl_unlock(); 7089 } 7090 7091 static void stmmac_service_task(struct work_struct *work) 7092 { 7093 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7094 service_task); 7095 7096 stmmac_reset_subtask(priv); 7097 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7098 } 7099 7100 /** 7101 * stmmac_hw_init - Init the MAC device 7102 * @priv: driver private structure 7103 * Description: this function is to configure the MAC device according to 7104 * some platform parameters or the HW capability register. It prepares the 7105 * driver to use either ring or chain modes and to setup either enhanced or 7106 * normal descriptors. 7107 */ 7108 static int stmmac_hw_init(struct stmmac_priv *priv) 7109 { 7110 int ret; 7111 7112 /* dwmac-sun8i only work in chain mode */ 7113 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7114 chain_mode = 1; 7115 priv->chain_mode = chain_mode; 7116 7117 /* Initialize HW Interface */ 7118 ret = stmmac_hwif_init(priv); 7119 if (ret) 7120 return ret; 7121 7122 /* Get the HW capability (new GMAC newer than 3.50a) */ 7123 priv->hw_cap_support = stmmac_get_hw_features(priv); 7124 if (priv->hw_cap_support) { 7125 dev_info(priv->device, "DMA HW capability register supported\n"); 7126 7127 /* We can override some gmac/dma configuration fields: e.g. 7128 * enh_desc, tx_coe (e.g. that are passed through the 7129 * platform) with the values from the HW capability 7130 * register (if supported). 7131 */ 7132 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7133 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7134 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7135 priv->hw->pmt = priv->plat->pmt; 7136 if (priv->dma_cap.hash_tb_sz) { 7137 priv->hw->multicast_filter_bins = 7138 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7139 priv->hw->mcast_bits_log2 = 7140 ilog2(priv->hw->multicast_filter_bins); 7141 } 7142 7143 /* TXCOE doesn't work in thresh DMA mode */ 7144 if (priv->plat->force_thresh_dma_mode) 7145 priv->plat->tx_coe = 0; 7146 else 7147 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7148 7149 /* In case of GMAC4 rx_coe is from HW cap register. */ 7150 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7151 7152 if (priv->dma_cap.rx_coe_type2) 7153 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7154 else if (priv->dma_cap.rx_coe_type1) 7155 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7156 7157 } else { 7158 dev_info(priv->device, "No HW DMA feature register supported\n"); 7159 } 7160 7161 if (priv->plat->rx_coe) { 7162 priv->hw->rx_csum = priv->plat->rx_coe; 7163 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7164 if (priv->synopsys_id < DWMAC_CORE_4_00) 7165 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7166 } 7167 if (priv->plat->tx_coe) 7168 dev_info(priv->device, "TX Checksum insertion supported\n"); 7169 7170 if (priv->plat->pmt) { 7171 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7172 device_set_wakeup_capable(priv->device, 1); 7173 } 7174 7175 if (priv->dma_cap.tsoen) 7176 dev_info(priv->device, "TSO supported\n"); 7177 7178 priv->hw->vlan_fail_q_en = 7179 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7180 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7181 7182 /* Run HW quirks, if any */ 7183 if (priv->hwif_quirks) { 7184 ret = priv->hwif_quirks(priv); 7185 if (ret) 7186 return ret; 7187 } 7188 7189 /* Rx Watchdog is available in the COREs newer than the 3.40. 7190 * In some case, for example on bugged HW this feature 7191 * has to be disable and this can be done by passing the 7192 * riwt_off field from the platform. 7193 */ 7194 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 7195 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 7196 priv->use_riwt = 1; 7197 dev_info(priv->device, 7198 "Enable RX Mitigation via HW Watchdog Timer\n"); 7199 } 7200 7201 return 0; 7202 } 7203 7204 static void stmmac_napi_add(struct net_device *dev) 7205 { 7206 struct stmmac_priv *priv = netdev_priv(dev); 7207 u32 queue, maxq; 7208 7209 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7210 7211 for (queue = 0; queue < maxq; queue++) { 7212 struct stmmac_channel *ch = &priv->channel[queue]; 7213 7214 ch->priv_data = priv; 7215 ch->index = queue; 7216 spin_lock_init(&ch->lock); 7217 7218 if (queue < priv->plat->rx_queues_to_use) { 7219 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7220 } 7221 if (queue < priv->plat->tx_queues_to_use) { 7222 netif_napi_add_tx(dev, &ch->tx_napi, 7223 stmmac_napi_poll_tx); 7224 } 7225 if (queue < priv->plat->rx_queues_to_use && 7226 queue < priv->plat->tx_queues_to_use) { 7227 netif_napi_add(dev, &ch->rxtx_napi, 7228 stmmac_napi_poll_rxtx); 7229 } 7230 } 7231 } 7232 7233 static void stmmac_napi_del(struct net_device *dev) 7234 { 7235 struct stmmac_priv *priv = netdev_priv(dev); 7236 u32 queue, maxq; 7237 7238 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7239 7240 for (queue = 0; queue < maxq; queue++) { 7241 struct stmmac_channel *ch = &priv->channel[queue]; 7242 7243 if (queue < priv->plat->rx_queues_to_use) 7244 netif_napi_del(&ch->rx_napi); 7245 if (queue < priv->plat->tx_queues_to_use) 7246 netif_napi_del(&ch->tx_napi); 7247 if (queue < priv->plat->rx_queues_to_use && 7248 queue < priv->plat->tx_queues_to_use) { 7249 netif_napi_del(&ch->rxtx_napi); 7250 } 7251 } 7252 } 7253 7254 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 7255 { 7256 struct stmmac_priv *priv = netdev_priv(dev); 7257 int ret = 0, i; 7258 7259 if (netif_running(dev)) 7260 stmmac_release(dev); 7261 7262 stmmac_napi_del(dev); 7263 7264 priv->plat->rx_queues_to_use = rx_cnt; 7265 priv->plat->tx_queues_to_use = tx_cnt; 7266 if (!netif_is_rxfh_configured(dev)) 7267 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7268 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7269 rx_cnt); 7270 7271 stmmac_napi_add(dev); 7272 7273 if (netif_running(dev)) 7274 ret = stmmac_open(dev); 7275 7276 return ret; 7277 } 7278 7279 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7280 { 7281 struct stmmac_priv *priv = netdev_priv(dev); 7282 int ret = 0; 7283 7284 if (netif_running(dev)) 7285 stmmac_release(dev); 7286 7287 priv->dma_conf.dma_rx_size = rx_size; 7288 priv->dma_conf.dma_tx_size = tx_size; 7289 7290 if (netif_running(dev)) 7291 ret = stmmac_open(dev); 7292 7293 return ret; 7294 } 7295 7296 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7297 { 7298 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7299 struct dma_desc *desc_contains_ts = ctx->desc; 7300 struct stmmac_priv *priv = ctx->priv; 7301 struct dma_desc *ndesc = ctx->ndesc; 7302 struct dma_desc *desc = ctx->desc; 7303 u64 ns = 0; 7304 7305 if (!priv->hwts_rx_en) 7306 return -ENODATA; 7307 7308 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7309 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 7310 desc_contains_ts = ndesc; 7311 7312 /* Check if timestamp is available */ 7313 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7314 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7315 ns -= priv->plat->cdc_error_adj; 7316 *timestamp = ns_to_ktime(ns); 7317 return 0; 7318 } 7319 7320 return -ENODATA; 7321 } 7322 7323 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7324 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7325 }; 7326 7327 /** 7328 * stmmac_dvr_probe 7329 * @device: device pointer 7330 * @plat_dat: platform data pointer 7331 * @res: stmmac resource pointer 7332 * Description: this is the main probe function used to 7333 * call the alloc_etherdev, allocate the priv structure. 7334 * Return: 7335 * returns 0 on success, otherwise errno. 7336 */ 7337 int stmmac_dvr_probe(struct device *device, 7338 struct plat_stmmacenet_data *plat_dat, 7339 struct stmmac_resources *res) 7340 { 7341 struct net_device *ndev = NULL; 7342 struct stmmac_priv *priv; 7343 u32 rxq; 7344 int i, ret = 0; 7345 7346 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7347 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7348 if (!ndev) 7349 return -ENOMEM; 7350 7351 SET_NETDEV_DEV(ndev, device); 7352 7353 priv = netdev_priv(ndev); 7354 priv->device = device; 7355 priv->dev = ndev; 7356 7357 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7358 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7359 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7360 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7361 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7362 } 7363 7364 priv->xstats.pcpu_stats = 7365 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7366 if (!priv->xstats.pcpu_stats) 7367 return -ENOMEM; 7368 7369 stmmac_set_ethtool_ops(ndev); 7370 priv->pause = pause; 7371 priv->plat = plat_dat; 7372 priv->ioaddr = res->addr; 7373 priv->dev->base_addr = (unsigned long)res->addr; 7374 priv->plat->dma_cfg->multi_msi_en = 7375 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7376 7377 priv->dev->irq = res->irq; 7378 priv->wol_irq = res->wol_irq; 7379 priv->lpi_irq = res->lpi_irq; 7380 priv->sfty_irq = res->sfty_irq; 7381 priv->sfty_ce_irq = res->sfty_ce_irq; 7382 priv->sfty_ue_irq = res->sfty_ue_irq; 7383 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7384 priv->rx_irq[i] = res->rx_irq[i]; 7385 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7386 priv->tx_irq[i] = res->tx_irq[i]; 7387 7388 if (!is_zero_ether_addr(res->mac)) 7389 eth_hw_addr_set(priv->dev, res->mac); 7390 7391 dev_set_drvdata(device, priv->dev); 7392 7393 /* Verify driver arguments */ 7394 stmmac_verify_args(); 7395 7396 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7397 if (!priv->af_xdp_zc_qps) 7398 return -ENOMEM; 7399 7400 /* Allocate workqueue */ 7401 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7402 if (!priv->wq) { 7403 dev_err(priv->device, "failed to create workqueue\n"); 7404 ret = -ENOMEM; 7405 goto error_wq_init; 7406 } 7407 7408 INIT_WORK(&priv->service_task, stmmac_service_task); 7409 7410 /* Override with kernel parameters if supplied XXX CRS XXX 7411 * this needs to have multiple instances 7412 */ 7413 if ((phyaddr >= 0) && (phyaddr <= 31)) 7414 priv->plat->phy_addr = phyaddr; 7415 7416 if (priv->plat->stmmac_rst) { 7417 ret = reset_control_assert(priv->plat->stmmac_rst); 7418 reset_control_deassert(priv->plat->stmmac_rst); 7419 /* Some reset controllers have only reset callback instead of 7420 * assert + deassert callbacks pair. 7421 */ 7422 if (ret == -ENOTSUPP) 7423 reset_control_reset(priv->plat->stmmac_rst); 7424 } 7425 7426 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7427 if (ret == -ENOTSUPP) 7428 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7429 ERR_PTR(ret)); 7430 7431 /* Wait a bit for the reset to take effect */ 7432 udelay(10); 7433 7434 /* Init MAC and get the capabilities */ 7435 ret = stmmac_hw_init(priv); 7436 if (ret) 7437 goto error_hw_init; 7438 7439 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7440 */ 7441 if (priv->synopsys_id < DWMAC_CORE_5_20) 7442 priv->plat->dma_cfg->dche = false; 7443 7444 stmmac_check_ether_addr(priv); 7445 7446 ndev->netdev_ops = &stmmac_netdev_ops; 7447 7448 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7449 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7450 7451 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7452 NETIF_F_RXCSUM; 7453 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7454 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7455 7456 ret = stmmac_tc_init(priv, priv); 7457 if (!ret) { 7458 ndev->hw_features |= NETIF_F_HW_TC; 7459 } 7460 7461 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 7462 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7463 if (priv->plat->has_gmac4) 7464 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7465 priv->tso = true; 7466 dev_info(priv->device, "TSO feature enabled\n"); 7467 } 7468 7469 if (priv->dma_cap.sphen && 7470 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7471 ndev->hw_features |= NETIF_F_GRO; 7472 priv->sph_cap = true; 7473 priv->sph = priv->sph_cap; 7474 dev_info(priv->device, "SPH feature enabled\n"); 7475 } 7476 7477 /* Ideally our host DMA address width is the same as for the 7478 * device. However, it may differ and then we have to use our 7479 * host DMA width for allocation and the device DMA width for 7480 * register handling. 7481 */ 7482 if (priv->plat->host_dma_width) 7483 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7484 else 7485 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7486 7487 if (priv->dma_cap.host_dma_width) { 7488 ret = dma_set_mask_and_coherent(device, 7489 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7490 if (!ret) { 7491 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7492 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7493 7494 /* 7495 * If more than 32 bits can be addressed, make sure to 7496 * enable enhanced addressing mode. 7497 */ 7498 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7499 priv->plat->dma_cfg->eame = true; 7500 } else { 7501 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7502 if (ret) { 7503 dev_err(priv->device, "Failed to set DMA Mask\n"); 7504 goto error_hw_init; 7505 } 7506 7507 priv->dma_cap.host_dma_width = 32; 7508 } 7509 } 7510 7511 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7512 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7513 #ifdef STMMAC_VLAN_TAG_USED 7514 /* Both mac100 and gmac support receive VLAN tag detection */ 7515 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7516 if (priv->plat->has_gmac4) { 7517 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 7518 priv->hw->hw_vlan_en = true; 7519 } 7520 if (priv->dma_cap.vlhash) { 7521 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7522 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7523 } 7524 if (priv->dma_cap.vlins) { 7525 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7526 if (priv->dma_cap.dvlan) 7527 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; 7528 } 7529 #endif 7530 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7531 7532 priv->xstats.threshold = tc; 7533 7534 /* Initialize RSS */ 7535 rxq = priv->plat->rx_queues_to_use; 7536 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7537 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7538 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7539 7540 if (priv->dma_cap.rssen && priv->plat->rss_en) 7541 ndev->features |= NETIF_F_RXHASH; 7542 7543 ndev->vlan_features |= ndev->features; 7544 7545 /* MTU range: 46 - hw-specific max */ 7546 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7547 if (priv->plat->has_xgmac) 7548 ndev->max_mtu = XGMAC_JUMBO_LEN; 7549 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 7550 ndev->max_mtu = JUMBO_LEN; 7551 else 7552 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7553 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 7554 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 7555 */ 7556 if ((priv->plat->maxmtu < ndev->max_mtu) && 7557 (priv->plat->maxmtu >= ndev->min_mtu)) 7558 ndev->max_mtu = priv->plat->maxmtu; 7559 else if (priv->plat->maxmtu < ndev->min_mtu) 7560 dev_warn(priv->device, 7561 "%s: warning: maxmtu having invalid value (%d)\n", 7562 __func__, priv->plat->maxmtu); 7563 7564 if (flow_ctrl) 7565 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 7566 7567 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 7568 7569 /* Setup channels NAPI */ 7570 stmmac_napi_add(ndev); 7571 7572 mutex_init(&priv->lock); 7573 7574 stmmac_fpe_init(priv); 7575 7576 /* If a specific clk_csr value is passed from the platform 7577 * this means that the CSR Clock Range selection cannot be 7578 * changed at run-time and it is fixed. Viceversa the driver'll try to 7579 * set the MDC clock dynamically according to the csr actual 7580 * clock input. 7581 */ 7582 if (priv->plat->clk_csr >= 0) 7583 priv->clk_csr = priv->plat->clk_csr; 7584 else 7585 stmmac_clk_csr_set(priv); 7586 7587 stmmac_check_pcs_mode(priv); 7588 7589 pm_runtime_get_noresume(device); 7590 pm_runtime_set_active(device); 7591 if (!pm_runtime_enabled(device)) 7592 pm_runtime_enable(device); 7593 7594 ret = stmmac_mdio_register(ndev); 7595 if (ret < 0) { 7596 dev_err_probe(priv->device, ret, 7597 "MDIO bus (id: %d) registration failed\n", 7598 priv->plat->bus_id); 7599 goto error_mdio_register; 7600 } 7601 7602 if (priv->plat->speed_mode_2500) 7603 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); 7604 7605 ret = stmmac_pcs_setup(ndev); 7606 if (ret) 7607 goto error_pcs_setup; 7608 7609 ret = stmmac_phy_setup(priv); 7610 if (ret) { 7611 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7612 goto error_phy_setup; 7613 } 7614 7615 ret = register_netdev(ndev); 7616 if (ret) { 7617 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7618 __func__, ret); 7619 goto error_netdev_register; 7620 } 7621 7622 #ifdef CONFIG_DEBUG_FS 7623 stmmac_init_fs(ndev); 7624 #endif 7625 7626 if (priv->plat->dump_debug_regs) 7627 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7628 7629 /* Let pm_runtime_put() disable the clocks. 7630 * If CONFIG_PM is not enabled, the clocks will stay powered. 7631 */ 7632 pm_runtime_put(device); 7633 7634 return ret; 7635 7636 error_netdev_register: 7637 phylink_destroy(priv->phylink); 7638 error_phy_setup: 7639 stmmac_pcs_clean(ndev); 7640 error_pcs_setup: 7641 stmmac_mdio_unregister(ndev); 7642 error_mdio_register: 7643 stmmac_napi_del(ndev); 7644 error_hw_init: 7645 destroy_workqueue(priv->wq); 7646 error_wq_init: 7647 bitmap_free(priv->af_xdp_zc_qps); 7648 7649 return ret; 7650 } 7651 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 7652 7653 /** 7654 * stmmac_dvr_remove 7655 * @dev: device pointer 7656 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 7657 * changes the link status, releases the DMA descriptor rings. 7658 */ 7659 void stmmac_dvr_remove(struct device *dev) 7660 { 7661 struct net_device *ndev = dev_get_drvdata(dev); 7662 struct stmmac_priv *priv = netdev_priv(ndev); 7663 7664 netdev_info(priv->dev, "%s: removing driver", __func__); 7665 7666 pm_runtime_get_sync(dev); 7667 7668 stmmac_stop_all_dma(priv); 7669 stmmac_mac_set(priv, priv->ioaddr, false); 7670 unregister_netdev(ndev); 7671 7672 #ifdef CONFIG_DEBUG_FS 7673 stmmac_exit_fs(ndev); 7674 #endif 7675 phylink_destroy(priv->phylink); 7676 if (priv->plat->stmmac_rst) 7677 reset_control_assert(priv->plat->stmmac_rst); 7678 reset_control_assert(priv->plat->stmmac_ahb_rst); 7679 7680 stmmac_pcs_clean(ndev); 7681 stmmac_mdio_unregister(ndev); 7682 7683 destroy_workqueue(priv->wq); 7684 mutex_destroy(&priv->lock); 7685 bitmap_free(priv->af_xdp_zc_qps); 7686 7687 pm_runtime_disable(dev); 7688 pm_runtime_put_noidle(dev); 7689 } 7690 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 7691 7692 /** 7693 * stmmac_suspend - suspend callback 7694 * @dev: device pointer 7695 * Description: this is the function to suspend the device and it is called 7696 * by the platform driver to stop the network queue, release the resources, 7697 * program the PMT register (for WoL), clean and release driver resources. 7698 */ 7699 int stmmac_suspend(struct device *dev) 7700 { 7701 struct net_device *ndev = dev_get_drvdata(dev); 7702 struct stmmac_priv *priv = netdev_priv(ndev); 7703 u32 chan; 7704 7705 if (!ndev || !netif_running(ndev)) 7706 return 0; 7707 7708 mutex_lock(&priv->lock); 7709 7710 netif_device_detach(ndev); 7711 7712 stmmac_disable_all_queues(priv); 7713 7714 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7715 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7716 7717 if (priv->eee_enabled) { 7718 priv->tx_path_in_lpi_mode = false; 7719 del_timer_sync(&priv->eee_ctrl_timer); 7720 } 7721 7722 /* Stop TX/RX DMA */ 7723 stmmac_stop_all_dma(priv); 7724 7725 if (priv->plat->serdes_powerdown) 7726 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7727 7728 /* Enable Power down mode by programming the PMT regs */ 7729 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7730 stmmac_pmt(priv, priv->hw, priv->wolopts); 7731 priv->irq_wake = 1; 7732 } else { 7733 stmmac_mac_set(priv, priv->ioaddr, false); 7734 pinctrl_pm_select_sleep_state(priv->device); 7735 } 7736 7737 mutex_unlock(&priv->lock); 7738 7739 rtnl_lock(); 7740 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7741 phylink_suspend(priv->phylink, true); 7742 } else { 7743 if (device_may_wakeup(priv->device)) 7744 phylink_speed_down(priv->phylink, false); 7745 phylink_suspend(priv->phylink, false); 7746 } 7747 rtnl_unlock(); 7748 7749 if (stmmac_fpe_supported(priv)) 7750 timer_shutdown_sync(&priv->fpe_cfg.verify_timer); 7751 7752 priv->speed = SPEED_UNKNOWN; 7753 return 0; 7754 } 7755 EXPORT_SYMBOL_GPL(stmmac_suspend); 7756 7757 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 7758 { 7759 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7760 7761 rx_q->cur_rx = 0; 7762 rx_q->dirty_rx = 0; 7763 } 7764 7765 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 7766 { 7767 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7768 7769 tx_q->cur_tx = 0; 7770 tx_q->dirty_tx = 0; 7771 tx_q->mss = 0; 7772 7773 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 7774 } 7775 7776 /** 7777 * stmmac_reset_queues_param - reset queue parameters 7778 * @priv: device pointer 7779 */ 7780 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 7781 { 7782 u32 rx_cnt = priv->plat->rx_queues_to_use; 7783 u32 tx_cnt = priv->plat->tx_queues_to_use; 7784 u32 queue; 7785 7786 for (queue = 0; queue < rx_cnt; queue++) 7787 stmmac_reset_rx_queue(priv, queue); 7788 7789 for (queue = 0; queue < tx_cnt; queue++) 7790 stmmac_reset_tx_queue(priv, queue); 7791 } 7792 7793 /** 7794 * stmmac_resume - resume callback 7795 * @dev: device pointer 7796 * Description: when resume this function is invoked to setup the DMA and CORE 7797 * in a usable state. 7798 */ 7799 int stmmac_resume(struct device *dev) 7800 { 7801 struct net_device *ndev = dev_get_drvdata(dev); 7802 struct stmmac_priv *priv = netdev_priv(ndev); 7803 int ret; 7804 7805 if (!netif_running(ndev)) 7806 return 0; 7807 7808 /* Power Down bit, into the PM register, is cleared 7809 * automatically as soon as a magic packet or a Wake-up frame 7810 * is received. Anyway, it's better to manually clear 7811 * this bit because it can generate problems while resuming 7812 * from another devices (e.g. serial console). 7813 */ 7814 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7815 mutex_lock(&priv->lock); 7816 stmmac_pmt(priv, priv->hw, 0); 7817 mutex_unlock(&priv->lock); 7818 priv->irq_wake = 0; 7819 } else { 7820 pinctrl_pm_select_default_state(priv->device); 7821 /* reset the phy so that it's ready */ 7822 if (priv->mii) 7823 stmmac_mdio_reset(priv->mii); 7824 } 7825 7826 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 7827 priv->plat->serdes_powerup) { 7828 ret = priv->plat->serdes_powerup(ndev, 7829 priv->plat->bsp_priv); 7830 7831 if (ret < 0) 7832 return ret; 7833 } 7834 7835 rtnl_lock(); 7836 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7837 phylink_resume(priv->phylink); 7838 } else { 7839 phylink_resume(priv->phylink); 7840 if (device_may_wakeup(priv->device)) 7841 phylink_speed_up(priv->phylink); 7842 } 7843 rtnl_unlock(); 7844 7845 rtnl_lock(); 7846 mutex_lock(&priv->lock); 7847 7848 stmmac_reset_queues_param(priv); 7849 7850 stmmac_free_tx_skbufs(priv); 7851 stmmac_clear_descriptors(priv, &priv->dma_conf); 7852 7853 stmmac_hw_setup(ndev, false); 7854 stmmac_init_coalesce(priv); 7855 stmmac_set_rx_mode(ndev); 7856 7857 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); 7858 7859 stmmac_enable_all_queues(priv); 7860 stmmac_enable_all_dma_irq(priv); 7861 7862 mutex_unlock(&priv->lock); 7863 rtnl_unlock(); 7864 7865 netif_device_attach(ndev); 7866 7867 return 0; 7868 } 7869 EXPORT_SYMBOL_GPL(stmmac_resume); 7870 7871 #ifndef MODULE 7872 static int __init stmmac_cmdline_opt(char *str) 7873 { 7874 char *opt; 7875 7876 if (!str || !*str) 7877 return 1; 7878 while ((opt = strsep(&str, ",")) != NULL) { 7879 if (!strncmp(opt, "debug:", 6)) { 7880 if (kstrtoint(opt + 6, 0, &debug)) 7881 goto err; 7882 } else if (!strncmp(opt, "phyaddr:", 8)) { 7883 if (kstrtoint(opt + 8, 0, &phyaddr)) 7884 goto err; 7885 } else if (!strncmp(opt, "buf_sz:", 7)) { 7886 if (kstrtoint(opt + 7, 0, &buf_sz)) 7887 goto err; 7888 } else if (!strncmp(opt, "tc:", 3)) { 7889 if (kstrtoint(opt + 3, 0, &tc)) 7890 goto err; 7891 } else if (!strncmp(opt, "watchdog:", 9)) { 7892 if (kstrtoint(opt + 9, 0, &watchdog)) 7893 goto err; 7894 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 7895 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 7896 goto err; 7897 } else if (!strncmp(opt, "pause:", 6)) { 7898 if (kstrtoint(opt + 6, 0, &pause)) 7899 goto err; 7900 } else if (!strncmp(opt, "eee_timer:", 10)) { 7901 if (kstrtoint(opt + 10, 0, &eee_timer)) 7902 goto err; 7903 } else if (!strncmp(opt, "chain_mode:", 11)) { 7904 if (kstrtoint(opt + 11, 0, &chain_mode)) 7905 goto err; 7906 } 7907 } 7908 return 1; 7909 7910 err: 7911 pr_err("%s: ERROR broken module parameter conversion", __func__); 7912 return 1; 7913 } 7914 7915 __setup("stmmaceth=", stmmac_cmdline_opt); 7916 #endif /* MODULE */ 7917 7918 static int __init stmmac_init(void) 7919 { 7920 #ifdef CONFIG_DEBUG_FS 7921 /* Create debugfs main directory if it doesn't exist yet */ 7922 if (!stmmac_fs_dir) 7923 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 7924 register_netdevice_notifier(&stmmac_notifier); 7925 #endif 7926 7927 return 0; 7928 } 7929 7930 static void __exit stmmac_exit(void) 7931 { 7932 #ifdef CONFIG_DEBUG_FS 7933 unregister_netdevice_notifier(&stmmac_notifier); 7934 debugfs_remove_recursive(stmmac_fs_dir); 7935 #endif 7936 } 7937 7938 module_init(stmmac_init) 7939 module_exit(stmmac_exit) 7940 7941 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 7942 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 7943 MODULE_LICENSE("GPL"); 7944