1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/clk.h> 18 #include <linux/kernel.h> 19 #include <linux/interrupt.h> 20 #include <linux/ip.h> 21 #include <linux/tcp.h> 22 #include <linux/skbuff.h> 23 #include <linux/ethtool.h> 24 #include <linux/if_ether.h> 25 #include <linux/crc32.h> 26 #include <linux/mii.h> 27 #include <linux/if.h> 28 #include <linux/if_vlan.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/slab.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/prefetch.h> 33 #include <linux/pinctrl/consumer.h> 34 #ifdef CONFIG_DEBUG_FS 35 #include <linux/debugfs.h> 36 #include <linux/seq_file.h> 37 #endif /* CONFIG_DEBUG_FS */ 38 #include <linux/net_tstamp.h> 39 #include <linux/phylink.h> 40 #include <linux/udp.h> 41 #include <linux/bpf_trace.h> 42 #include <net/page_pool/helpers.h> 43 #include <net/pkt_cls.h> 44 #include <net/xdp_sock_drv.h> 45 #include "stmmac_ptp.h" 46 #include "stmmac_fpe.h" 47 #include "stmmac.h" 48 #include "stmmac_xdp.h" 49 #include <linux/reset.h> 50 #include <linux/of_mdio.h> 51 #include "dwmac1000.h" 52 #include "dwxgmac2.h" 53 #include "hwif.h" 54 55 /* As long as the interface is active, we keep the timestamping counter enabled 56 * with fine resolution and binary rollover. This avoid non-monotonic behavior 57 * (clock jumps) when changing timestamping settings at runtime. 58 */ 59 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ 60 PTP_TCR_TSCTRLSSR) 61 62 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 63 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 64 65 /* Module parameters */ 66 #define TX_TIMEO 5000 67 static int watchdog = TX_TIMEO; 68 module_param(watchdog, int, 0644); 69 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 70 71 static int debug = -1; 72 module_param(debug, int, 0644); 73 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 74 75 static int phyaddr = -1; 76 module_param(phyaddr, int, 0444); 77 MODULE_PARM_DESC(phyaddr, "Physical device address"); 78 79 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 80 81 /* Limit to make sure XDP TX and slow path can coexist */ 82 #define STMMAC_XSK_TX_BUDGET_MAX 256 83 #define STMMAC_TX_XSK_AVAIL 16 84 #define STMMAC_RX_FILL_BATCH 16 85 86 #define STMMAC_XDP_PASS 0 87 #define STMMAC_XDP_CONSUMED BIT(0) 88 #define STMMAC_XDP_TX BIT(1) 89 #define STMMAC_XDP_REDIRECT BIT(2) 90 91 static int flow_ctrl = FLOW_AUTO; 92 module_param(flow_ctrl, int, 0644); 93 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 94 95 static int pause = PAUSE_TIME; 96 module_param(pause, int, 0644); 97 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 98 99 #define TC_DEFAULT 64 100 static int tc = TC_DEFAULT; 101 module_param(tc, int, 0644); 102 MODULE_PARM_DESC(tc, "DMA threshold control value"); 103 104 #define DEFAULT_BUFSIZE 1536 105 static int buf_sz = DEFAULT_BUFSIZE; 106 module_param(buf_sz, int, 0644); 107 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 108 109 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 110 NETIF_MSG_LINK | NETIF_MSG_IFUP | 111 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 112 113 #define STMMAC_DEFAULT_LPI_TIMER 1000 114 static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 115 module_param(eee_timer, uint, 0644); 116 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 117 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 118 119 /* By default the driver will use the ring mode to manage tx and rx descriptors, 120 * but allow user to force to use the chain instead of the ring 121 */ 122 static unsigned int chain_mode; 123 module_param(chain_mode, int, 0444); 124 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 125 126 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 127 /* For MSI interrupts handling */ 128 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 129 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 130 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 131 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 132 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 133 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 134 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 135 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 136 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 137 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 138 u32 rxmode, u32 chan); 139 140 #ifdef CONFIG_DEBUG_FS 141 static const struct net_device_ops stmmac_netdev_ops; 142 static void stmmac_init_fs(struct net_device *dev); 143 static void stmmac_exit_fs(struct net_device *dev); 144 #endif 145 146 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 147 148 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) 149 { 150 int ret = 0; 151 152 if (enabled) { 153 ret = clk_prepare_enable(priv->plat->stmmac_clk); 154 if (ret) 155 return ret; 156 ret = clk_prepare_enable(priv->plat->pclk); 157 if (ret) { 158 clk_disable_unprepare(priv->plat->stmmac_clk); 159 return ret; 160 } 161 if (priv->plat->clks_config) { 162 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); 163 if (ret) { 164 clk_disable_unprepare(priv->plat->stmmac_clk); 165 clk_disable_unprepare(priv->plat->pclk); 166 return ret; 167 } 168 } 169 } else { 170 clk_disable_unprepare(priv->plat->stmmac_clk); 171 clk_disable_unprepare(priv->plat->pclk); 172 if (priv->plat->clks_config) 173 priv->plat->clks_config(priv->plat->bsp_priv, enabled); 174 } 175 176 return ret; 177 } 178 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); 179 180 /** 181 * stmmac_verify_args - verify the driver parameters. 182 * Description: it checks the driver parameters and set a default in case of 183 * errors. 184 */ 185 static void stmmac_verify_args(void) 186 { 187 if (unlikely(watchdog < 0)) 188 watchdog = TX_TIMEO; 189 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 190 buf_sz = DEFAULT_BUFSIZE; 191 if (unlikely(flow_ctrl > 1)) 192 flow_ctrl = FLOW_AUTO; 193 else if (likely(flow_ctrl < 0)) 194 flow_ctrl = FLOW_OFF; 195 if (unlikely((pause < 0) || (pause > 0xffff))) 196 pause = PAUSE_TIME; 197 } 198 199 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 200 { 201 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 202 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 203 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 204 u32 queue; 205 206 for (queue = 0; queue < maxq; queue++) { 207 struct stmmac_channel *ch = &priv->channel[queue]; 208 209 if (stmmac_xdp_is_enabled(priv) && 210 test_bit(queue, priv->af_xdp_zc_qps)) { 211 napi_disable(&ch->rxtx_napi); 212 continue; 213 } 214 215 if (queue < rx_queues_cnt) 216 napi_disable(&ch->rx_napi); 217 if (queue < tx_queues_cnt) 218 napi_disable(&ch->tx_napi); 219 } 220 } 221 222 /** 223 * stmmac_disable_all_queues - Disable all queues 224 * @priv: driver private structure 225 */ 226 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 227 { 228 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 229 struct stmmac_rx_queue *rx_q; 230 u32 queue; 231 232 /* synchronize_rcu() needed for pending XDP buffers to drain */ 233 for (queue = 0; queue < rx_queues_cnt; queue++) { 234 rx_q = &priv->dma_conf.rx_queue[queue]; 235 if (rx_q->xsk_pool) { 236 synchronize_rcu(); 237 break; 238 } 239 } 240 241 __stmmac_disable_all_queues(priv); 242 } 243 244 /** 245 * stmmac_enable_all_queues - Enable all queues 246 * @priv: driver private structure 247 */ 248 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 249 { 250 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 251 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 252 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 253 u32 queue; 254 255 for (queue = 0; queue < maxq; queue++) { 256 struct stmmac_channel *ch = &priv->channel[queue]; 257 258 if (stmmac_xdp_is_enabled(priv) && 259 test_bit(queue, priv->af_xdp_zc_qps)) { 260 napi_enable(&ch->rxtx_napi); 261 continue; 262 } 263 264 if (queue < rx_queues_cnt) 265 napi_enable(&ch->rx_napi); 266 if (queue < tx_queues_cnt) 267 napi_enable(&ch->tx_napi); 268 } 269 } 270 271 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 272 { 273 if (!test_bit(STMMAC_DOWN, &priv->state) && 274 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 275 queue_work(priv->wq, &priv->service_task); 276 } 277 278 static void stmmac_global_err(struct stmmac_priv *priv) 279 { 280 netif_carrier_off(priv->dev); 281 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 282 stmmac_service_event_schedule(priv); 283 } 284 285 /** 286 * stmmac_clk_csr_set - dynamically set the MDC clock 287 * @priv: driver private structure 288 * Description: this is to dynamically set the MDC clock according to the csr 289 * clock input. 290 * Note: 291 * If a specific clk_csr value is passed from the platform 292 * this means that the CSR Clock Range selection cannot be 293 * changed at run-time and it is fixed (as reported in the driver 294 * documentation). Viceversa the driver will try to set the MDC 295 * clock dynamically according to the actual clock input. 296 */ 297 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 298 { 299 unsigned long clk_rate; 300 301 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 302 303 /* Platform provided default clk_csr would be assumed valid 304 * for all other cases except for the below mentioned ones. 305 * For values higher than the IEEE 802.3 specified frequency 306 * we can not estimate the proper divider as it is not known 307 * the frequency of clk_csr_i. So we do not change the default 308 * divider. 309 */ 310 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 311 if (clk_rate < CSR_F_35M) 312 priv->clk_csr = STMMAC_CSR_20_35M; 313 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 314 priv->clk_csr = STMMAC_CSR_35_60M; 315 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 316 priv->clk_csr = STMMAC_CSR_60_100M; 317 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 318 priv->clk_csr = STMMAC_CSR_100_150M; 319 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 320 priv->clk_csr = STMMAC_CSR_150_250M; 321 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) 322 priv->clk_csr = STMMAC_CSR_250_300M; 323 else if ((clk_rate >= CSR_F_300M) && (clk_rate < CSR_F_500M)) 324 priv->clk_csr = STMMAC_CSR_300_500M; 325 else if ((clk_rate >= CSR_F_500M) && (clk_rate < CSR_F_800M)) 326 priv->clk_csr = STMMAC_CSR_500_800M; 327 } 328 329 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { 330 if (clk_rate > 160000000) 331 priv->clk_csr = 0x03; 332 else if (clk_rate > 80000000) 333 priv->clk_csr = 0x02; 334 else if (clk_rate > 40000000) 335 priv->clk_csr = 0x01; 336 else 337 priv->clk_csr = 0; 338 } 339 340 if (priv->plat->has_xgmac) { 341 if (clk_rate > 400000000) 342 priv->clk_csr = 0x5; 343 else if (clk_rate > 350000000) 344 priv->clk_csr = 0x4; 345 else if (clk_rate > 300000000) 346 priv->clk_csr = 0x3; 347 else if (clk_rate > 250000000) 348 priv->clk_csr = 0x2; 349 else if (clk_rate > 150000000) 350 priv->clk_csr = 0x1; 351 else 352 priv->clk_csr = 0x0; 353 } 354 } 355 356 static void print_pkt(unsigned char *buf, int len) 357 { 358 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 359 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 360 } 361 362 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 363 { 364 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 365 u32 avail; 366 367 if (tx_q->dirty_tx > tx_q->cur_tx) 368 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 369 else 370 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 371 372 return avail; 373 } 374 375 /** 376 * stmmac_rx_dirty - Get RX queue dirty 377 * @priv: driver private structure 378 * @queue: RX queue index 379 */ 380 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 381 { 382 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 383 u32 dirty; 384 385 if (rx_q->dirty_rx <= rx_q->cur_rx) 386 dirty = rx_q->cur_rx - rx_q->dirty_rx; 387 else 388 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 389 390 return dirty; 391 } 392 393 static void stmmac_disable_hw_lpi_timer(struct stmmac_priv *priv) 394 { 395 stmmac_set_eee_lpi_timer(priv, priv->hw, 0); 396 } 397 398 static void stmmac_enable_hw_lpi_timer(struct stmmac_priv *priv) 399 { 400 stmmac_set_eee_lpi_timer(priv, priv->hw, priv->tx_lpi_timer); 401 } 402 403 static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) 404 { 405 u32 tx_cnt = priv->plat->tx_queues_to_use; 406 u32 queue; 407 408 /* check if all TX queues have the work finished */ 409 for (queue = 0; queue < tx_cnt; queue++) { 410 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 411 412 if (tx_q->dirty_tx != tx_q->cur_tx) 413 return true; /* still unfinished work */ 414 } 415 416 return false; 417 } 418 419 static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv) 420 { 421 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 422 } 423 424 /** 425 * stmmac_try_to_start_sw_lpi - check and enter in LPI mode 426 * @priv: driver private structure 427 * Description: this function is to verify and enter in LPI mode in case of 428 * EEE. 429 */ 430 static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv) 431 { 432 if (stmmac_eee_tx_busy(priv)) { 433 stmmac_restart_sw_lpi_timer(priv); 434 return; 435 } 436 437 /* Check and enter in LPI mode */ 438 if (!priv->tx_path_in_lpi_mode) 439 stmmac_set_eee_mode(priv, priv->hw, 440 priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); 441 } 442 443 /** 444 * stmmac_stop_sw_lpi - stop transmitting LPI 445 * @priv: driver private structure 446 * Description: When using software-controlled LPI, stop transmitting LPI state. 447 */ 448 static void stmmac_stop_sw_lpi(struct stmmac_priv *priv) 449 { 450 stmmac_reset_eee_mode(priv, priv->hw); 451 del_timer_sync(&priv->eee_ctrl_timer); 452 priv->tx_path_in_lpi_mode = false; 453 } 454 455 /** 456 * stmmac_eee_ctrl_timer - EEE TX SW timer. 457 * @t: timer_list struct containing private info 458 * Description: 459 * if there is no data transfer and if we are not in LPI state, 460 * then MAC Transmitter can be moved to LPI state. 461 */ 462 static void stmmac_eee_ctrl_timer(struct timer_list *t) 463 { 464 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 465 466 stmmac_try_to_start_sw_lpi(priv); 467 } 468 469 /** 470 * stmmac_eee_init - init EEE 471 * @priv: driver private structure 472 * @active: indicates whether EEE should be enabled. 473 * Description: 474 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 475 * can also manage EEE, this function enable the LPI state and start related 476 * timer. 477 */ 478 static void stmmac_eee_init(struct stmmac_priv *priv, bool active) 479 { 480 priv->eee_active = active; 481 482 /* Check if MAC core supports the EEE feature. */ 483 if (!priv->dma_cap.eee) { 484 priv->eee_enabled = false; 485 return; 486 } 487 488 mutex_lock(&priv->lock); 489 490 /* Check if it needs to be deactivated */ 491 if (!priv->eee_active) { 492 if (priv->eee_enabled) { 493 netdev_dbg(priv->dev, "disable EEE\n"); 494 priv->eee_sw_timer_en = false; 495 stmmac_disable_hw_lpi_timer(priv); 496 del_timer_sync(&priv->eee_ctrl_timer); 497 stmmac_set_eee_timer(priv, priv->hw, 0, 498 STMMAC_DEFAULT_TWT_LS); 499 if (priv->hw->xpcs) 500 xpcs_config_eee(priv->hw->xpcs, 501 priv->plat->mult_fact_100ns, 502 false); 503 } 504 priv->eee_enabled = false; 505 mutex_unlock(&priv->lock); 506 return; 507 } 508 509 if (priv->eee_active && !priv->eee_enabled) { 510 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 511 STMMAC_DEFAULT_TWT_LS); 512 if (priv->hw->xpcs) 513 xpcs_config_eee(priv->hw->xpcs, 514 priv->plat->mult_fact_100ns, 515 true); 516 } 517 518 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { 519 /* Use hardware LPI mode */ 520 del_timer_sync(&priv->eee_ctrl_timer); 521 priv->tx_path_in_lpi_mode = false; 522 priv->eee_sw_timer_en = false; 523 stmmac_enable_hw_lpi_timer(priv); 524 } else { 525 /* Use software LPI mode */ 526 priv->eee_sw_timer_en = true; 527 stmmac_disable_hw_lpi_timer(priv); 528 stmmac_restart_sw_lpi_timer(priv); 529 } 530 531 priv->eee_enabled = true; 532 533 mutex_unlock(&priv->lock); 534 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 535 } 536 537 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 538 * @priv: driver private structure 539 * @p : descriptor pointer 540 * @skb : the socket buffer 541 * Description : 542 * This function will read timestamp from the descriptor & pass it to stack. 543 * and also perform some sanity checks. 544 */ 545 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 546 struct dma_desc *p, struct sk_buff *skb) 547 { 548 struct skb_shared_hwtstamps shhwtstamp; 549 bool found = false; 550 u64 ns = 0; 551 552 if (!priv->hwts_tx_en) 553 return; 554 555 /* exit if skb doesn't support hw tstamp */ 556 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 557 return; 558 559 /* check tx tstamp status */ 560 if (stmmac_get_tx_timestamp_status(priv, p)) { 561 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 562 found = true; 563 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 564 found = true; 565 } 566 567 if (found) { 568 ns -= priv->plat->cdc_error_adj; 569 570 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 571 shhwtstamp.hwtstamp = ns_to_ktime(ns); 572 573 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 574 /* pass tstamp to stack */ 575 skb_tstamp_tx(skb, &shhwtstamp); 576 } 577 } 578 579 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 580 * @priv: driver private structure 581 * @p : descriptor pointer 582 * @np : next descriptor pointer 583 * @skb : the socket buffer 584 * Description : 585 * This function will read received packet's timestamp from the descriptor 586 * and pass it to stack. It also perform some sanity checks. 587 */ 588 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 589 struct dma_desc *np, struct sk_buff *skb) 590 { 591 struct skb_shared_hwtstamps *shhwtstamp = NULL; 592 struct dma_desc *desc = p; 593 u64 ns = 0; 594 595 if (!priv->hwts_rx_en) 596 return; 597 /* For GMAC4, the valid timestamp is from CTX next desc. */ 598 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 599 desc = np; 600 601 /* Check if timestamp is available */ 602 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 603 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 604 605 ns -= priv->plat->cdc_error_adj; 606 607 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 608 shhwtstamp = skb_hwtstamps(skb); 609 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 610 shhwtstamp->hwtstamp = ns_to_ktime(ns); 611 } else { 612 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 613 } 614 } 615 616 /** 617 * stmmac_hwtstamp_set - control hardware timestamping. 618 * @dev: device pointer. 619 * @ifr: An IOCTL specific structure, that can contain a pointer to 620 * a proprietary structure used to pass information to the driver. 621 * Description: 622 * This function configures the MAC to enable/disable both outgoing(TX) 623 * and incoming(RX) packets time stamping based on user input. 624 * Return Value: 625 * 0 on success and an appropriate -ve integer on failure. 626 */ 627 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 628 { 629 struct stmmac_priv *priv = netdev_priv(dev); 630 struct hwtstamp_config config; 631 u32 ptp_v2 = 0; 632 u32 tstamp_all = 0; 633 u32 ptp_over_ipv4_udp = 0; 634 u32 ptp_over_ipv6_udp = 0; 635 u32 ptp_over_ethernet = 0; 636 u32 snap_type_sel = 0; 637 u32 ts_master_en = 0; 638 u32 ts_event_en = 0; 639 640 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 641 netdev_alert(priv->dev, "No support for HW time stamping\n"); 642 priv->hwts_tx_en = 0; 643 priv->hwts_rx_en = 0; 644 645 return -EOPNOTSUPP; 646 } 647 648 if (copy_from_user(&config, ifr->ifr_data, 649 sizeof(config))) 650 return -EFAULT; 651 652 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 653 __func__, config.flags, config.tx_type, config.rx_filter); 654 655 if (config.tx_type != HWTSTAMP_TX_OFF && 656 config.tx_type != HWTSTAMP_TX_ON) 657 return -ERANGE; 658 659 if (priv->adv_ts) { 660 switch (config.rx_filter) { 661 case HWTSTAMP_FILTER_NONE: 662 /* time stamp no incoming packet at all */ 663 config.rx_filter = HWTSTAMP_FILTER_NONE; 664 break; 665 666 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 667 /* PTP v1, UDP, any kind of event packet */ 668 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 669 /* 'xmac' hardware can support Sync, Pdelay_Req and 670 * Pdelay_resp by setting bit14 and bits17/16 to 01 671 * This leaves Delay_Req timestamps out. 672 * Enable all events *and* general purpose message 673 * timestamping 674 */ 675 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 676 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 677 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 678 break; 679 680 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 681 /* PTP v1, UDP, Sync packet */ 682 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 683 /* take time stamp for SYNC messages only */ 684 ts_event_en = PTP_TCR_TSEVNTENA; 685 686 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 687 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 688 break; 689 690 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 691 /* PTP v1, UDP, Delay_req packet */ 692 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 693 /* take time stamp for Delay_Req messages only */ 694 ts_master_en = PTP_TCR_TSMSTRENA; 695 ts_event_en = PTP_TCR_TSEVNTENA; 696 697 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 698 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 699 break; 700 701 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 702 /* PTP v2, UDP, any kind of event packet */ 703 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 704 ptp_v2 = PTP_TCR_TSVER2ENA; 705 /* take time stamp for all event messages */ 706 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 707 708 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 709 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 710 break; 711 712 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 713 /* PTP v2, UDP, Sync packet */ 714 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 715 ptp_v2 = PTP_TCR_TSVER2ENA; 716 /* take time stamp for SYNC messages only */ 717 ts_event_en = PTP_TCR_TSEVNTENA; 718 719 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 720 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 721 break; 722 723 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 724 /* PTP v2, UDP, Delay_req packet */ 725 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 726 ptp_v2 = PTP_TCR_TSVER2ENA; 727 /* take time stamp for Delay_Req messages only */ 728 ts_master_en = PTP_TCR_TSMSTRENA; 729 ts_event_en = PTP_TCR_TSEVNTENA; 730 731 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 732 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 733 break; 734 735 case HWTSTAMP_FILTER_PTP_V2_EVENT: 736 /* PTP v2/802.AS1 any layer, any kind of event packet */ 737 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 738 ptp_v2 = PTP_TCR_TSVER2ENA; 739 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 740 if (priv->synopsys_id < DWMAC_CORE_4_10) 741 ts_event_en = PTP_TCR_TSEVNTENA; 742 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 743 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 744 ptp_over_ethernet = PTP_TCR_TSIPENA; 745 break; 746 747 case HWTSTAMP_FILTER_PTP_V2_SYNC: 748 /* PTP v2/802.AS1, any layer, Sync packet */ 749 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 750 ptp_v2 = PTP_TCR_TSVER2ENA; 751 /* take time stamp for SYNC messages only */ 752 ts_event_en = PTP_TCR_TSEVNTENA; 753 754 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 755 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 756 ptp_over_ethernet = PTP_TCR_TSIPENA; 757 break; 758 759 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 760 /* PTP v2/802.AS1, any layer, Delay_req packet */ 761 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 762 ptp_v2 = PTP_TCR_TSVER2ENA; 763 /* take time stamp for Delay_Req messages only */ 764 ts_master_en = PTP_TCR_TSMSTRENA; 765 ts_event_en = PTP_TCR_TSEVNTENA; 766 767 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 768 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 769 ptp_over_ethernet = PTP_TCR_TSIPENA; 770 break; 771 772 case HWTSTAMP_FILTER_NTP_ALL: 773 case HWTSTAMP_FILTER_ALL: 774 /* time stamp any incoming packet */ 775 config.rx_filter = HWTSTAMP_FILTER_ALL; 776 tstamp_all = PTP_TCR_TSENALL; 777 break; 778 779 default: 780 return -ERANGE; 781 } 782 } else { 783 switch (config.rx_filter) { 784 case HWTSTAMP_FILTER_NONE: 785 config.rx_filter = HWTSTAMP_FILTER_NONE; 786 break; 787 default: 788 /* PTP v1, UDP, any kind of event packet */ 789 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 790 break; 791 } 792 } 793 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 794 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 795 796 priv->systime_flags = STMMAC_HWTS_ACTIVE; 797 798 if (priv->hwts_tx_en || priv->hwts_rx_en) { 799 priv->systime_flags |= tstamp_all | ptp_v2 | 800 ptp_over_ethernet | ptp_over_ipv6_udp | 801 ptp_over_ipv4_udp | ts_event_en | 802 ts_master_en | snap_type_sel; 803 } 804 805 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 806 807 memcpy(&priv->tstamp_config, &config, sizeof(config)); 808 809 return copy_to_user(ifr->ifr_data, &config, 810 sizeof(config)) ? -EFAULT : 0; 811 } 812 813 /** 814 * stmmac_hwtstamp_get - read hardware timestamping. 815 * @dev: device pointer. 816 * @ifr: An IOCTL specific structure, that can contain a pointer to 817 * a proprietary structure used to pass information to the driver. 818 * Description: 819 * This function obtain the current hardware timestamping settings 820 * as requested. 821 */ 822 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 823 { 824 struct stmmac_priv *priv = netdev_priv(dev); 825 struct hwtstamp_config *config = &priv->tstamp_config; 826 827 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 828 return -EOPNOTSUPP; 829 830 return copy_to_user(ifr->ifr_data, config, 831 sizeof(*config)) ? -EFAULT : 0; 832 } 833 834 /** 835 * stmmac_init_tstamp_counter - init hardware timestamping counter 836 * @priv: driver private structure 837 * @systime_flags: timestamping flags 838 * Description: 839 * Initialize hardware counter for packet timestamping. 840 * This is valid as long as the interface is open and not suspended. 841 * Will be rerun after resuming from suspend, case in which the timestamping 842 * flags updated by stmmac_hwtstamp_set() also need to be restored. 843 */ 844 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) 845 { 846 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 847 struct timespec64 now; 848 u32 sec_inc = 0; 849 u64 temp = 0; 850 851 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 852 return -EOPNOTSUPP; 853 854 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 855 priv->systime_flags = systime_flags; 856 857 /* program Sub Second Increment reg */ 858 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 859 priv->plat->clk_ptp_rate, 860 xmac, &sec_inc); 861 temp = div_u64(1000000000ULL, sec_inc); 862 863 /* Store sub second increment for later use */ 864 priv->sub_second_inc = sec_inc; 865 866 /* calculate default added value: 867 * formula is : 868 * addend = (2^32)/freq_div_ratio; 869 * where, freq_div_ratio = 1e9ns/sec_inc 870 */ 871 temp = (u64)(temp << 32); 872 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 873 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 874 875 /* initialize system time */ 876 ktime_get_real_ts64(&now); 877 878 /* lower 32 bits of tv_sec are safe until y2106 */ 879 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 880 881 return 0; 882 } 883 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); 884 885 /** 886 * stmmac_init_ptp - init PTP 887 * @priv: driver private structure 888 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 889 * This is done by looking at the HW cap. register. 890 * This function also registers the ptp driver. 891 */ 892 static int stmmac_init_ptp(struct stmmac_priv *priv) 893 { 894 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 895 int ret; 896 897 if (priv->plat->ptp_clk_freq_config) 898 priv->plat->ptp_clk_freq_config(priv); 899 900 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); 901 if (ret) 902 return ret; 903 904 priv->adv_ts = 0; 905 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 906 if (xmac && priv->dma_cap.atime_stamp) 907 priv->adv_ts = 1; 908 /* Dwmac 3.x core with extend_desc can support adv_ts */ 909 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 910 priv->adv_ts = 1; 911 912 if (priv->dma_cap.time_stamp) 913 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 914 915 if (priv->adv_ts) 916 netdev_info(priv->dev, 917 "IEEE 1588-2008 Advanced Timestamp supported\n"); 918 919 priv->hwts_tx_en = 0; 920 priv->hwts_rx_en = 0; 921 922 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 923 stmmac_hwtstamp_correct_latency(priv, priv); 924 925 return 0; 926 } 927 928 static void stmmac_release_ptp(struct stmmac_priv *priv) 929 { 930 clk_disable_unprepare(priv->plat->clk_ptp_ref); 931 stmmac_ptp_unregister(priv); 932 } 933 934 /** 935 * stmmac_mac_flow_ctrl - Configure flow control in all queues 936 * @priv: driver private structure 937 * @duplex: duplex passed to the next function 938 * Description: It is used for configuring the flow control in all queues 939 */ 940 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 941 { 942 u32 tx_cnt = priv->plat->tx_queues_to_use; 943 944 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 945 priv->pause, tx_cnt); 946 } 947 948 static unsigned long stmmac_mac_get_caps(struct phylink_config *config, 949 phy_interface_t interface) 950 { 951 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 952 953 /* Refresh the MAC-specific capabilities */ 954 stmmac_mac_update_caps(priv); 955 956 config->mac_capabilities = priv->hw->link.caps; 957 958 if (priv->plat->max_speed) 959 phylink_limit_mac_speed(config, priv->plat->max_speed); 960 961 return config->mac_capabilities; 962 } 963 964 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 965 phy_interface_t interface) 966 { 967 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 968 struct phylink_pcs *pcs; 969 970 if (priv->plat->select_pcs) { 971 pcs = priv->plat->select_pcs(priv, interface); 972 if (!IS_ERR(pcs)) 973 return pcs; 974 } 975 976 return NULL; 977 } 978 979 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 980 const struct phylink_link_state *state) 981 { 982 /* Nothing to do, xpcs_config() handles everything */ 983 } 984 985 static void stmmac_mac_link_down(struct phylink_config *config, 986 unsigned int mode, phy_interface_t interface) 987 { 988 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 989 990 stmmac_mac_set(priv, priv->ioaddr, false); 991 if (priv->dma_cap.eee) 992 stmmac_set_eee_pls(priv, priv->hw, false); 993 994 if (stmmac_fpe_supported(priv)) 995 stmmac_fpe_link_state_handle(priv, false); 996 } 997 998 static void stmmac_mac_link_up(struct phylink_config *config, 999 struct phy_device *phy, 1000 unsigned int mode, phy_interface_t interface, 1001 int speed, int duplex, 1002 bool tx_pause, bool rx_pause) 1003 { 1004 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1005 u32 old_ctrl, ctrl; 1006 1007 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 1008 priv->plat->serdes_powerup) 1009 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 1010 1011 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1012 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1013 1014 if (interface == PHY_INTERFACE_MODE_USXGMII) { 1015 switch (speed) { 1016 case SPEED_10000: 1017 ctrl |= priv->hw->link.xgmii.speed10000; 1018 break; 1019 case SPEED_5000: 1020 ctrl |= priv->hw->link.xgmii.speed5000; 1021 break; 1022 case SPEED_2500: 1023 ctrl |= priv->hw->link.xgmii.speed2500; 1024 break; 1025 default: 1026 return; 1027 } 1028 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1029 switch (speed) { 1030 case SPEED_100000: 1031 ctrl |= priv->hw->link.xlgmii.speed100000; 1032 break; 1033 case SPEED_50000: 1034 ctrl |= priv->hw->link.xlgmii.speed50000; 1035 break; 1036 case SPEED_40000: 1037 ctrl |= priv->hw->link.xlgmii.speed40000; 1038 break; 1039 case SPEED_25000: 1040 ctrl |= priv->hw->link.xlgmii.speed25000; 1041 break; 1042 case SPEED_10000: 1043 ctrl |= priv->hw->link.xgmii.speed10000; 1044 break; 1045 case SPEED_2500: 1046 ctrl |= priv->hw->link.speed2500; 1047 break; 1048 case SPEED_1000: 1049 ctrl |= priv->hw->link.speed1000; 1050 break; 1051 default: 1052 return; 1053 } 1054 } else { 1055 switch (speed) { 1056 case SPEED_2500: 1057 ctrl |= priv->hw->link.speed2500; 1058 break; 1059 case SPEED_1000: 1060 ctrl |= priv->hw->link.speed1000; 1061 break; 1062 case SPEED_100: 1063 ctrl |= priv->hw->link.speed100; 1064 break; 1065 case SPEED_10: 1066 ctrl |= priv->hw->link.speed10; 1067 break; 1068 default: 1069 return; 1070 } 1071 } 1072 1073 priv->speed = speed; 1074 1075 if (priv->plat->fix_mac_speed) 1076 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); 1077 1078 if (!duplex) 1079 ctrl &= ~priv->hw->link.duplex; 1080 else 1081 ctrl |= priv->hw->link.duplex; 1082 1083 /* Flow Control operation */ 1084 if (rx_pause && tx_pause) 1085 priv->flow_ctrl = FLOW_AUTO; 1086 else if (rx_pause && !tx_pause) 1087 priv->flow_ctrl = FLOW_RX; 1088 else if (!rx_pause && tx_pause) 1089 priv->flow_ctrl = FLOW_TX; 1090 else 1091 priv->flow_ctrl = FLOW_OFF; 1092 1093 stmmac_mac_flow_ctrl(priv, duplex); 1094 1095 if (ctrl != old_ctrl) 1096 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1097 1098 stmmac_mac_set(priv, priv->ioaddr, true); 1099 if (priv->dma_cap.eee) 1100 stmmac_set_eee_pls(priv, priv->hw, true); 1101 1102 if (stmmac_fpe_supported(priv)) 1103 stmmac_fpe_link_state_handle(priv, true); 1104 1105 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1106 stmmac_hwtstamp_correct_latency(priv, priv); 1107 } 1108 1109 static void stmmac_mac_disable_tx_lpi(struct phylink_config *config) 1110 { 1111 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1112 1113 stmmac_eee_init(priv, false); 1114 } 1115 1116 static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 1117 bool tx_clk_stop) 1118 { 1119 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1120 1121 priv->tx_lpi_timer = timer; 1122 stmmac_eee_init(priv, true); 1123 1124 return 0; 1125 } 1126 1127 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1128 .mac_get_caps = stmmac_mac_get_caps, 1129 .mac_select_pcs = stmmac_mac_select_pcs, 1130 .mac_config = stmmac_mac_config, 1131 .mac_link_down = stmmac_mac_link_down, 1132 .mac_link_up = stmmac_mac_link_up, 1133 .mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi, 1134 .mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi, 1135 }; 1136 1137 /** 1138 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1139 * @priv: driver private structure 1140 * Description: this is to verify if the HW supports the PCS. 1141 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1142 * configured for the TBI, RTBI, or SGMII PHY interface. 1143 */ 1144 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1145 { 1146 int interface = priv->plat->mac_interface; 1147 1148 if (priv->dma_cap.pcs) { 1149 if ((interface == PHY_INTERFACE_MODE_RGMII) || 1150 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 1151 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 1152 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 1153 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 1154 priv->hw->pcs = STMMAC_PCS_RGMII; 1155 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 1156 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1157 priv->hw->pcs = STMMAC_PCS_SGMII; 1158 } 1159 } 1160 } 1161 1162 /** 1163 * stmmac_init_phy - PHY initialization 1164 * @dev: net device structure 1165 * Description: it initializes the driver's PHY state, and attaches the PHY 1166 * to the mac driver. 1167 * Return value: 1168 * 0 on success 1169 */ 1170 static int stmmac_init_phy(struct net_device *dev) 1171 { 1172 struct stmmac_priv *priv = netdev_priv(dev); 1173 struct fwnode_handle *phy_fwnode; 1174 struct fwnode_handle *fwnode; 1175 int ret; 1176 1177 if (!phylink_expects_phy(priv->phylink)) 1178 return 0; 1179 1180 fwnode = priv->plat->port_node; 1181 if (!fwnode) 1182 fwnode = dev_fwnode(priv->device); 1183 1184 if (fwnode) 1185 phy_fwnode = fwnode_get_phy_node(fwnode); 1186 else 1187 phy_fwnode = NULL; 1188 1189 /* Some DT bindings do not set-up the PHY handle. Let's try to 1190 * manually parse it 1191 */ 1192 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1193 int addr = priv->plat->phy_addr; 1194 struct phy_device *phydev; 1195 1196 if (addr < 0) { 1197 netdev_err(priv->dev, "no phy found\n"); 1198 return -ENODEV; 1199 } 1200 1201 phydev = mdiobus_get_phy(priv->mii, addr); 1202 if (!phydev) { 1203 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1204 return -ENODEV; 1205 } 1206 1207 ret = phylink_connect_phy(priv->phylink, phydev); 1208 } else { 1209 fwnode_handle_put(phy_fwnode); 1210 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); 1211 } 1212 1213 if (ret == 0) { 1214 struct ethtool_keee eee; 1215 1216 /* Configure phylib's copy of the LPI timer. Normally, 1217 * phylink_config.lpi_timer_default would do this, but there is 1218 * a chance that userspace could change the eee_timer setting 1219 * via sysfs before the first open. Thus, preserve existing 1220 * behaviour. 1221 */ 1222 if (!phylink_ethtool_get_eee(priv->phylink, &eee)) { 1223 eee.tx_lpi_timer = priv->tx_lpi_timer; 1224 phylink_ethtool_set_eee(priv->phylink, &eee); 1225 } 1226 } 1227 1228 if (!priv->plat->pmt) { 1229 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 1230 1231 phylink_ethtool_get_wol(priv->phylink, &wol); 1232 device_set_wakeup_capable(priv->device, !!wol.supported); 1233 device_set_wakeup_enable(priv->device, !!wol.wolopts); 1234 } 1235 1236 return ret; 1237 } 1238 1239 static int stmmac_phy_setup(struct stmmac_priv *priv) 1240 { 1241 struct stmmac_mdio_bus_data *mdio_bus_data; 1242 int mode = priv->plat->phy_interface; 1243 struct fwnode_handle *fwnode; 1244 struct phylink_pcs *pcs; 1245 struct phylink *phylink; 1246 1247 priv->phylink_config.dev = &priv->dev->dev; 1248 priv->phylink_config.type = PHYLINK_NETDEV; 1249 priv->phylink_config.mac_managed_pm = true; 1250 1251 /* Stmmac always requires an RX clock for hardware initialization */ 1252 priv->phylink_config.mac_requires_rxc = true; 1253 1254 if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) 1255 priv->phylink_config.eee_rx_clk_stop_enable = true; 1256 1257 mdio_bus_data = priv->plat->mdio_bus_data; 1258 if (mdio_bus_data) 1259 priv->phylink_config.default_an_inband = 1260 mdio_bus_data->default_an_inband; 1261 1262 /* Set the platform/firmware specified interface mode. Note, phylink 1263 * deals with the PHY interface mode, not the MAC interface mode. 1264 */ 1265 __set_bit(mode, priv->phylink_config.supported_interfaces); 1266 1267 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1268 if (priv->hw->xpcs) 1269 pcs = xpcs_to_phylink_pcs(priv->hw->xpcs); 1270 else 1271 pcs = priv->hw->phylink_pcs; 1272 1273 if (pcs) 1274 phy_interface_or(priv->phylink_config.supported_interfaces, 1275 priv->phylink_config.supported_interfaces, 1276 pcs->supported_interfaces); 1277 1278 if (priv->dma_cap.eee) { 1279 /* Assume all supported interfaces also support LPI */ 1280 memcpy(priv->phylink_config.lpi_interfaces, 1281 priv->phylink_config.supported_interfaces, 1282 sizeof(priv->phylink_config.lpi_interfaces)); 1283 1284 /* All full duplex speeds above 100Mbps are supported */ 1285 priv->phylink_config.lpi_capabilities = ~(MAC_1000FD - 1) | 1286 MAC_100FD; 1287 priv->phylink_config.lpi_timer_default = eee_timer * 1000; 1288 priv->phylink_config.eee_enabled_default = true; 1289 } 1290 1291 fwnode = priv->plat->port_node; 1292 if (!fwnode) 1293 fwnode = dev_fwnode(priv->device); 1294 1295 phylink = phylink_create(&priv->phylink_config, fwnode, 1296 mode, &stmmac_phylink_mac_ops); 1297 if (IS_ERR(phylink)) 1298 return PTR_ERR(phylink); 1299 1300 priv->phylink = phylink; 1301 return 0; 1302 } 1303 1304 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1305 struct stmmac_dma_conf *dma_conf) 1306 { 1307 u32 rx_cnt = priv->plat->rx_queues_to_use; 1308 unsigned int desc_size; 1309 void *head_rx; 1310 u32 queue; 1311 1312 /* Display RX rings */ 1313 for (queue = 0; queue < rx_cnt; queue++) { 1314 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1315 1316 pr_info("\tRX Queue %u rings\n", queue); 1317 1318 if (priv->extend_desc) { 1319 head_rx = (void *)rx_q->dma_erx; 1320 desc_size = sizeof(struct dma_extended_desc); 1321 } else { 1322 head_rx = (void *)rx_q->dma_rx; 1323 desc_size = sizeof(struct dma_desc); 1324 } 1325 1326 /* Display RX ring */ 1327 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1328 rx_q->dma_rx_phy, desc_size); 1329 } 1330 } 1331 1332 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1333 struct stmmac_dma_conf *dma_conf) 1334 { 1335 u32 tx_cnt = priv->plat->tx_queues_to_use; 1336 unsigned int desc_size; 1337 void *head_tx; 1338 u32 queue; 1339 1340 /* Display TX rings */ 1341 for (queue = 0; queue < tx_cnt; queue++) { 1342 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1343 1344 pr_info("\tTX Queue %d rings\n", queue); 1345 1346 if (priv->extend_desc) { 1347 head_tx = (void *)tx_q->dma_etx; 1348 desc_size = sizeof(struct dma_extended_desc); 1349 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1350 head_tx = (void *)tx_q->dma_entx; 1351 desc_size = sizeof(struct dma_edesc); 1352 } else { 1353 head_tx = (void *)tx_q->dma_tx; 1354 desc_size = sizeof(struct dma_desc); 1355 } 1356 1357 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1358 tx_q->dma_tx_phy, desc_size); 1359 } 1360 } 1361 1362 static void stmmac_display_rings(struct stmmac_priv *priv, 1363 struct stmmac_dma_conf *dma_conf) 1364 { 1365 /* Display RX ring */ 1366 stmmac_display_rx_rings(priv, dma_conf); 1367 1368 /* Display TX ring */ 1369 stmmac_display_tx_rings(priv, dma_conf); 1370 } 1371 1372 static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) 1373 { 1374 if (stmmac_xdp_is_enabled(priv)) 1375 return XDP_PACKET_HEADROOM; 1376 1377 return NET_SKB_PAD; 1378 } 1379 1380 static int stmmac_set_bfsize(int mtu, int bufsize) 1381 { 1382 int ret = bufsize; 1383 1384 if (mtu >= BUF_SIZE_8KiB) 1385 ret = BUF_SIZE_16KiB; 1386 else if (mtu >= BUF_SIZE_4KiB) 1387 ret = BUF_SIZE_8KiB; 1388 else if (mtu >= BUF_SIZE_2KiB) 1389 ret = BUF_SIZE_4KiB; 1390 else if (mtu > DEFAULT_BUFSIZE) 1391 ret = BUF_SIZE_2KiB; 1392 else 1393 ret = DEFAULT_BUFSIZE; 1394 1395 return ret; 1396 } 1397 1398 /** 1399 * stmmac_clear_rx_descriptors - clear RX descriptors 1400 * @priv: driver private structure 1401 * @dma_conf: structure to take the dma data 1402 * @queue: RX queue index 1403 * Description: this function is called to clear the RX descriptors 1404 * in case of both basic and extended descriptors are used. 1405 */ 1406 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1407 struct stmmac_dma_conf *dma_conf, 1408 u32 queue) 1409 { 1410 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1411 int i; 1412 1413 /* Clear the RX descriptors */ 1414 for (i = 0; i < dma_conf->dma_rx_size; i++) 1415 if (priv->extend_desc) 1416 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1417 priv->use_riwt, priv->mode, 1418 (i == dma_conf->dma_rx_size - 1), 1419 dma_conf->dma_buf_sz); 1420 else 1421 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1422 priv->use_riwt, priv->mode, 1423 (i == dma_conf->dma_rx_size - 1), 1424 dma_conf->dma_buf_sz); 1425 } 1426 1427 /** 1428 * stmmac_clear_tx_descriptors - clear tx descriptors 1429 * @priv: driver private structure 1430 * @dma_conf: structure to take the dma data 1431 * @queue: TX queue index. 1432 * Description: this function is called to clear the TX descriptors 1433 * in case of both basic and extended descriptors are used. 1434 */ 1435 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1436 struct stmmac_dma_conf *dma_conf, 1437 u32 queue) 1438 { 1439 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1440 int i; 1441 1442 /* Clear the TX descriptors */ 1443 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1444 int last = (i == (dma_conf->dma_tx_size - 1)); 1445 struct dma_desc *p; 1446 1447 if (priv->extend_desc) 1448 p = &tx_q->dma_etx[i].basic; 1449 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1450 p = &tx_q->dma_entx[i].basic; 1451 else 1452 p = &tx_q->dma_tx[i]; 1453 1454 stmmac_init_tx_desc(priv, p, priv->mode, last); 1455 } 1456 } 1457 1458 /** 1459 * stmmac_clear_descriptors - clear descriptors 1460 * @priv: driver private structure 1461 * @dma_conf: structure to take the dma data 1462 * Description: this function is called to clear the TX and RX descriptors 1463 * in case of both basic and extended descriptors are used. 1464 */ 1465 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1466 struct stmmac_dma_conf *dma_conf) 1467 { 1468 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1469 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1470 u32 queue; 1471 1472 /* Clear the RX descriptors */ 1473 for (queue = 0; queue < rx_queue_cnt; queue++) 1474 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1475 1476 /* Clear the TX descriptors */ 1477 for (queue = 0; queue < tx_queue_cnt; queue++) 1478 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1479 } 1480 1481 /** 1482 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1483 * @priv: driver private structure 1484 * @dma_conf: structure to take the dma data 1485 * @p: descriptor pointer 1486 * @i: descriptor index 1487 * @flags: gfp flag 1488 * @queue: RX queue index 1489 * Description: this function is called to allocate a receive buffer, perform 1490 * the DMA mapping and init the descriptor. 1491 */ 1492 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1493 struct stmmac_dma_conf *dma_conf, 1494 struct dma_desc *p, 1495 int i, gfp_t flags, u32 queue) 1496 { 1497 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1498 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1499 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1500 1501 if (priv->dma_cap.host_dma_width <= 32) 1502 gfp |= GFP_DMA32; 1503 1504 if (!buf->page) { 1505 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1506 if (!buf->page) 1507 return -ENOMEM; 1508 buf->page_offset = stmmac_rx_offset(priv); 1509 } 1510 1511 if (priv->sph && !buf->sec_page) { 1512 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1513 if (!buf->sec_page) 1514 return -ENOMEM; 1515 1516 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1517 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1518 } else { 1519 buf->sec_page = NULL; 1520 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1521 } 1522 1523 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1524 1525 stmmac_set_desc_addr(priv, p, buf->addr); 1526 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1527 stmmac_init_desc3(priv, p); 1528 1529 return 0; 1530 } 1531 1532 /** 1533 * stmmac_free_rx_buffer - free RX dma buffers 1534 * @priv: private structure 1535 * @rx_q: RX queue 1536 * @i: buffer index. 1537 */ 1538 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1539 struct stmmac_rx_queue *rx_q, 1540 int i) 1541 { 1542 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1543 1544 if (buf->page) 1545 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1546 buf->page = NULL; 1547 1548 if (buf->sec_page) 1549 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1550 buf->sec_page = NULL; 1551 } 1552 1553 /** 1554 * stmmac_free_tx_buffer - free RX dma buffers 1555 * @priv: private structure 1556 * @dma_conf: structure to take the dma data 1557 * @queue: RX queue index 1558 * @i: buffer index. 1559 */ 1560 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1561 struct stmmac_dma_conf *dma_conf, 1562 u32 queue, int i) 1563 { 1564 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1565 1566 if (tx_q->tx_skbuff_dma[i].buf && 1567 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1568 if (tx_q->tx_skbuff_dma[i].map_as_page) 1569 dma_unmap_page(priv->device, 1570 tx_q->tx_skbuff_dma[i].buf, 1571 tx_q->tx_skbuff_dma[i].len, 1572 DMA_TO_DEVICE); 1573 else 1574 dma_unmap_single(priv->device, 1575 tx_q->tx_skbuff_dma[i].buf, 1576 tx_q->tx_skbuff_dma[i].len, 1577 DMA_TO_DEVICE); 1578 } 1579 1580 if (tx_q->xdpf[i] && 1581 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1582 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1583 xdp_return_frame(tx_q->xdpf[i]); 1584 tx_q->xdpf[i] = NULL; 1585 } 1586 1587 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1588 tx_q->xsk_frames_done++; 1589 1590 if (tx_q->tx_skbuff[i] && 1591 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1592 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1593 tx_q->tx_skbuff[i] = NULL; 1594 } 1595 1596 tx_q->tx_skbuff_dma[i].buf = 0; 1597 tx_q->tx_skbuff_dma[i].map_as_page = false; 1598 } 1599 1600 /** 1601 * dma_free_rx_skbufs - free RX dma buffers 1602 * @priv: private structure 1603 * @dma_conf: structure to take the dma data 1604 * @queue: RX queue index 1605 */ 1606 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1607 struct stmmac_dma_conf *dma_conf, 1608 u32 queue) 1609 { 1610 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1611 int i; 1612 1613 for (i = 0; i < dma_conf->dma_rx_size; i++) 1614 stmmac_free_rx_buffer(priv, rx_q, i); 1615 } 1616 1617 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1618 struct stmmac_dma_conf *dma_conf, 1619 u32 queue, gfp_t flags) 1620 { 1621 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1622 int i; 1623 1624 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1625 struct dma_desc *p; 1626 int ret; 1627 1628 if (priv->extend_desc) 1629 p = &((rx_q->dma_erx + i)->basic); 1630 else 1631 p = rx_q->dma_rx + i; 1632 1633 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1634 queue); 1635 if (ret) 1636 return ret; 1637 1638 rx_q->buf_alloc_num++; 1639 } 1640 1641 return 0; 1642 } 1643 1644 /** 1645 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1646 * @priv: private structure 1647 * @dma_conf: structure to take the dma data 1648 * @queue: RX queue index 1649 */ 1650 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1651 struct stmmac_dma_conf *dma_conf, 1652 u32 queue) 1653 { 1654 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1655 int i; 1656 1657 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1658 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1659 1660 if (!buf->xdp) 1661 continue; 1662 1663 xsk_buff_free(buf->xdp); 1664 buf->xdp = NULL; 1665 } 1666 } 1667 1668 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1669 struct stmmac_dma_conf *dma_conf, 1670 u32 queue) 1671 { 1672 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1673 int i; 1674 1675 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1676 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1677 * use this macro to make sure no size violations. 1678 */ 1679 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1680 1681 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1682 struct stmmac_rx_buffer *buf; 1683 dma_addr_t dma_addr; 1684 struct dma_desc *p; 1685 1686 if (priv->extend_desc) 1687 p = (struct dma_desc *)(rx_q->dma_erx + i); 1688 else 1689 p = rx_q->dma_rx + i; 1690 1691 buf = &rx_q->buf_pool[i]; 1692 1693 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1694 if (!buf->xdp) 1695 return -ENOMEM; 1696 1697 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1698 stmmac_set_desc_addr(priv, p, dma_addr); 1699 rx_q->buf_alloc_num++; 1700 } 1701 1702 return 0; 1703 } 1704 1705 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1706 { 1707 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1708 return NULL; 1709 1710 return xsk_get_pool_from_qid(priv->dev, queue); 1711 } 1712 1713 /** 1714 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1715 * @priv: driver private structure 1716 * @dma_conf: structure to take the dma data 1717 * @queue: RX queue index 1718 * @flags: gfp flag. 1719 * Description: this function initializes the DMA RX descriptors 1720 * and allocates the socket buffers. It supports the chained and ring 1721 * modes. 1722 */ 1723 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1724 struct stmmac_dma_conf *dma_conf, 1725 u32 queue, gfp_t flags) 1726 { 1727 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1728 int ret; 1729 1730 netif_dbg(priv, probe, priv->dev, 1731 "(%s) dma_rx_phy=0x%08x\n", __func__, 1732 (u32)rx_q->dma_rx_phy); 1733 1734 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1735 1736 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1737 1738 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1739 1740 if (rx_q->xsk_pool) { 1741 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1742 MEM_TYPE_XSK_BUFF_POOL, 1743 NULL)); 1744 netdev_info(priv->dev, 1745 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1746 rx_q->queue_index); 1747 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1748 } else { 1749 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1750 MEM_TYPE_PAGE_POOL, 1751 rx_q->page_pool)); 1752 netdev_info(priv->dev, 1753 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1754 rx_q->queue_index); 1755 } 1756 1757 if (rx_q->xsk_pool) { 1758 /* RX XDP ZC buffer pool may not be populated, e.g. 1759 * xdpsock TX-only. 1760 */ 1761 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1762 } else { 1763 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1764 if (ret < 0) 1765 return -ENOMEM; 1766 } 1767 1768 /* Setup the chained descriptor addresses */ 1769 if (priv->mode == STMMAC_CHAIN_MODE) { 1770 if (priv->extend_desc) 1771 stmmac_mode_init(priv, rx_q->dma_erx, 1772 rx_q->dma_rx_phy, 1773 dma_conf->dma_rx_size, 1); 1774 else 1775 stmmac_mode_init(priv, rx_q->dma_rx, 1776 rx_q->dma_rx_phy, 1777 dma_conf->dma_rx_size, 0); 1778 } 1779 1780 return 0; 1781 } 1782 1783 static int init_dma_rx_desc_rings(struct net_device *dev, 1784 struct stmmac_dma_conf *dma_conf, 1785 gfp_t flags) 1786 { 1787 struct stmmac_priv *priv = netdev_priv(dev); 1788 u32 rx_count = priv->plat->rx_queues_to_use; 1789 int queue; 1790 int ret; 1791 1792 /* RX INITIALIZATION */ 1793 netif_dbg(priv, probe, priv->dev, 1794 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1795 1796 for (queue = 0; queue < rx_count; queue++) { 1797 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1798 if (ret) 1799 goto err_init_rx_buffers; 1800 } 1801 1802 return 0; 1803 1804 err_init_rx_buffers: 1805 while (queue >= 0) { 1806 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1807 1808 if (rx_q->xsk_pool) 1809 dma_free_rx_xskbufs(priv, dma_conf, queue); 1810 else 1811 dma_free_rx_skbufs(priv, dma_conf, queue); 1812 1813 rx_q->buf_alloc_num = 0; 1814 rx_q->xsk_pool = NULL; 1815 1816 queue--; 1817 } 1818 1819 return ret; 1820 } 1821 1822 /** 1823 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1824 * @priv: driver private structure 1825 * @dma_conf: structure to take the dma data 1826 * @queue: TX queue index 1827 * Description: this function initializes the DMA TX descriptors 1828 * and allocates the socket buffers. It supports the chained and ring 1829 * modes. 1830 */ 1831 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 1832 struct stmmac_dma_conf *dma_conf, 1833 u32 queue) 1834 { 1835 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1836 int i; 1837 1838 netif_dbg(priv, probe, priv->dev, 1839 "(%s) dma_tx_phy=0x%08x\n", __func__, 1840 (u32)tx_q->dma_tx_phy); 1841 1842 /* Setup the chained descriptor addresses */ 1843 if (priv->mode == STMMAC_CHAIN_MODE) { 1844 if (priv->extend_desc) 1845 stmmac_mode_init(priv, tx_q->dma_etx, 1846 tx_q->dma_tx_phy, 1847 dma_conf->dma_tx_size, 1); 1848 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1849 stmmac_mode_init(priv, tx_q->dma_tx, 1850 tx_q->dma_tx_phy, 1851 dma_conf->dma_tx_size, 0); 1852 } 1853 1854 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1855 1856 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1857 struct dma_desc *p; 1858 1859 if (priv->extend_desc) 1860 p = &((tx_q->dma_etx + i)->basic); 1861 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1862 p = &((tx_q->dma_entx + i)->basic); 1863 else 1864 p = tx_q->dma_tx + i; 1865 1866 stmmac_clear_desc(priv, p); 1867 1868 tx_q->tx_skbuff_dma[i].buf = 0; 1869 tx_q->tx_skbuff_dma[i].map_as_page = false; 1870 tx_q->tx_skbuff_dma[i].len = 0; 1871 tx_q->tx_skbuff_dma[i].last_segment = false; 1872 tx_q->tx_skbuff[i] = NULL; 1873 } 1874 1875 return 0; 1876 } 1877 1878 static int init_dma_tx_desc_rings(struct net_device *dev, 1879 struct stmmac_dma_conf *dma_conf) 1880 { 1881 struct stmmac_priv *priv = netdev_priv(dev); 1882 u32 tx_queue_cnt; 1883 u32 queue; 1884 1885 tx_queue_cnt = priv->plat->tx_queues_to_use; 1886 1887 for (queue = 0; queue < tx_queue_cnt; queue++) 1888 __init_dma_tx_desc_rings(priv, dma_conf, queue); 1889 1890 return 0; 1891 } 1892 1893 /** 1894 * init_dma_desc_rings - init the RX/TX descriptor rings 1895 * @dev: net device structure 1896 * @dma_conf: structure to take the dma data 1897 * @flags: gfp flag. 1898 * Description: this function initializes the DMA RX/TX descriptors 1899 * and allocates the socket buffers. It supports the chained and ring 1900 * modes. 1901 */ 1902 static int init_dma_desc_rings(struct net_device *dev, 1903 struct stmmac_dma_conf *dma_conf, 1904 gfp_t flags) 1905 { 1906 struct stmmac_priv *priv = netdev_priv(dev); 1907 int ret; 1908 1909 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 1910 if (ret) 1911 return ret; 1912 1913 ret = init_dma_tx_desc_rings(dev, dma_conf); 1914 1915 stmmac_clear_descriptors(priv, dma_conf); 1916 1917 if (netif_msg_hw(priv)) 1918 stmmac_display_rings(priv, dma_conf); 1919 1920 return ret; 1921 } 1922 1923 /** 1924 * dma_free_tx_skbufs - free TX dma buffers 1925 * @priv: private structure 1926 * @dma_conf: structure to take the dma data 1927 * @queue: TX queue index 1928 */ 1929 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 1930 struct stmmac_dma_conf *dma_conf, 1931 u32 queue) 1932 { 1933 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1934 int i; 1935 1936 tx_q->xsk_frames_done = 0; 1937 1938 for (i = 0; i < dma_conf->dma_tx_size; i++) 1939 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 1940 1941 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 1942 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 1943 tx_q->xsk_frames_done = 0; 1944 tx_q->xsk_pool = NULL; 1945 } 1946 } 1947 1948 /** 1949 * stmmac_free_tx_skbufs - free TX skb buffers 1950 * @priv: private structure 1951 */ 1952 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 1953 { 1954 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1955 u32 queue; 1956 1957 for (queue = 0; queue < tx_queue_cnt; queue++) 1958 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 1959 } 1960 1961 /** 1962 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 1963 * @priv: private structure 1964 * @dma_conf: structure to take the dma data 1965 * @queue: RX queue index 1966 */ 1967 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 1968 struct stmmac_dma_conf *dma_conf, 1969 u32 queue) 1970 { 1971 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1972 1973 /* Release the DMA RX socket buffers */ 1974 if (rx_q->xsk_pool) 1975 dma_free_rx_xskbufs(priv, dma_conf, queue); 1976 else 1977 dma_free_rx_skbufs(priv, dma_conf, queue); 1978 1979 rx_q->buf_alloc_num = 0; 1980 rx_q->xsk_pool = NULL; 1981 1982 /* Free DMA regions of consistent memory previously allocated */ 1983 if (!priv->extend_desc) 1984 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1985 sizeof(struct dma_desc), 1986 rx_q->dma_rx, rx_q->dma_rx_phy); 1987 else 1988 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1989 sizeof(struct dma_extended_desc), 1990 rx_q->dma_erx, rx_q->dma_rx_phy); 1991 1992 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 1993 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 1994 1995 kfree(rx_q->buf_pool); 1996 if (rx_q->page_pool) 1997 page_pool_destroy(rx_q->page_pool); 1998 } 1999 2000 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 2001 struct stmmac_dma_conf *dma_conf) 2002 { 2003 u32 rx_count = priv->plat->rx_queues_to_use; 2004 u32 queue; 2005 2006 /* Free RX queue resources */ 2007 for (queue = 0; queue < rx_count; queue++) 2008 __free_dma_rx_desc_resources(priv, dma_conf, queue); 2009 } 2010 2011 /** 2012 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 2013 * @priv: private structure 2014 * @dma_conf: structure to take the dma data 2015 * @queue: TX queue index 2016 */ 2017 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 2018 struct stmmac_dma_conf *dma_conf, 2019 u32 queue) 2020 { 2021 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2022 size_t size; 2023 void *addr; 2024 2025 /* Release the DMA TX socket buffers */ 2026 dma_free_tx_skbufs(priv, dma_conf, queue); 2027 2028 if (priv->extend_desc) { 2029 size = sizeof(struct dma_extended_desc); 2030 addr = tx_q->dma_etx; 2031 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 2032 size = sizeof(struct dma_edesc); 2033 addr = tx_q->dma_entx; 2034 } else { 2035 size = sizeof(struct dma_desc); 2036 addr = tx_q->dma_tx; 2037 } 2038 2039 size *= dma_conf->dma_tx_size; 2040 2041 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 2042 2043 kfree(tx_q->tx_skbuff_dma); 2044 kfree(tx_q->tx_skbuff); 2045 } 2046 2047 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2048 struct stmmac_dma_conf *dma_conf) 2049 { 2050 u32 tx_count = priv->plat->tx_queues_to_use; 2051 u32 queue; 2052 2053 /* Free TX queue resources */ 2054 for (queue = 0; queue < tx_count; queue++) 2055 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2056 } 2057 2058 /** 2059 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2060 * @priv: private structure 2061 * @dma_conf: structure to take the dma data 2062 * @queue: RX queue index 2063 * Description: according to which descriptor can be used (extend or basic) 2064 * this function allocates the resources for TX and RX paths. In case of 2065 * reception, for example, it pre-allocated the RX socket buffer in order to 2066 * allow zero-copy mechanism. 2067 */ 2068 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2069 struct stmmac_dma_conf *dma_conf, 2070 u32 queue) 2071 { 2072 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2073 struct stmmac_channel *ch = &priv->channel[queue]; 2074 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2075 struct page_pool_params pp_params = { 0 }; 2076 unsigned int dma_buf_sz_pad, num_pages; 2077 unsigned int napi_id; 2078 int ret; 2079 2080 dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + 2081 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2082 num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE); 2083 2084 rx_q->queue_index = queue; 2085 rx_q->priv_data = priv; 2086 rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE; 2087 2088 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2089 pp_params.pool_size = dma_conf->dma_rx_size; 2090 pp_params.order = order_base_2(num_pages); 2091 pp_params.nid = dev_to_node(priv->device); 2092 pp_params.dev = priv->device; 2093 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2094 pp_params.offset = stmmac_rx_offset(priv); 2095 pp_params.max_len = dma_conf->dma_buf_sz; 2096 2097 rx_q->page_pool = page_pool_create(&pp_params); 2098 if (IS_ERR(rx_q->page_pool)) { 2099 ret = PTR_ERR(rx_q->page_pool); 2100 rx_q->page_pool = NULL; 2101 return ret; 2102 } 2103 2104 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, 2105 sizeof(*rx_q->buf_pool), 2106 GFP_KERNEL); 2107 if (!rx_q->buf_pool) 2108 return -ENOMEM; 2109 2110 if (priv->extend_desc) { 2111 rx_q->dma_erx = dma_alloc_coherent(priv->device, 2112 dma_conf->dma_rx_size * 2113 sizeof(struct dma_extended_desc), 2114 &rx_q->dma_rx_phy, 2115 GFP_KERNEL); 2116 if (!rx_q->dma_erx) 2117 return -ENOMEM; 2118 2119 } else { 2120 rx_q->dma_rx = dma_alloc_coherent(priv->device, 2121 dma_conf->dma_rx_size * 2122 sizeof(struct dma_desc), 2123 &rx_q->dma_rx_phy, 2124 GFP_KERNEL); 2125 if (!rx_q->dma_rx) 2126 return -ENOMEM; 2127 } 2128 2129 if (stmmac_xdp_is_enabled(priv) && 2130 test_bit(queue, priv->af_xdp_zc_qps)) 2131 napi_id = ch->rxtx_napi.napi_id; 2132 else 2133 napi_id = ch->rx_napi.napi_id; 2134 2135 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 2136 rx_q->queue_index, 2137 napi_id); 2138 if (ret) { 2139 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2140 return -EINVAL; 2141 } 2142 2143 return 0; 2144 } 2145 2146 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2147 struct stmmac_dma_conf *dma_conf) 2148 { 2149 u32 rx_count = priv->plat->rx_queues_to_use; 2150 u32 queue; 2151 int ret; 2152 2153 /* RX queues buffers and DMA */ 2154 for (queue = 0; queue < rx_count; queue++) { 2155 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2156 if (ret) 2157 goto err_dma; 2158 } 2159 2160 return 0; 2161 2162 err_dma: 2163 free_dma_rx_desc_resources(priv, dma_conf); 2164 2165 return ret; 2166 } 2167 2168 /** 2169 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2170 * @priv: private structure 2171 * @dma_conf: structure to take the dma data 2172 * @queue: TX queue index 2173 * Description: according to which descriptor can be used (extend or basic) 2174 * this function allocates the resources for TX and RX paths. In case of 2175 * reception, for example, it pre-allocated the RX socket buffer in order to 2176 * allow zero-copy mechanism. 2177 */ 2178 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2179 struct stmmac_dma_conf *dma_conf, 2180 u32 queue) 2181 { 2182 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2183 size_t size; 2184 void *addr; 2185 2186 tx_q->queue_index = queue; 2187 tx_q->priv_data = priv; 2188 2189 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, 2190 sizeof(*tx_q->tx_skbuff_dma), 2191 GFP_KERNEL); 2192 if (!tx_q->tx_skbuff_dma) 2193 return -ENOMEM; 2194 2195 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, 2196 sizeof(struct sk_buff *), 2197 GFP_KERNEL); 2198 if (!tx_q->tx_skbuff) 2199 return -ENOMEM; 2200 2201 if (priv->extend_desc) 2202 size = sizeof(struct dma_extended_desc); 2203 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2204 size = sizeof(struct dma_edesc); 2205 else 2206 size = sizeof(struct dma_desc); 2207 2208 size *= dma_conf->dma_tx_size; 2209 2210 addr = dma_alloc_coherent(priv->device, size, 2211 &tx_q->dma_tx_phy, GFP_KERNEL); 2212 if (!addr) 2213 return -ENOMEM; 2214 2215 if (priv->extend_desc) 2216 tx_q->dma_etx = addr; 2217 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2218 tx_q->dma_entx = addr; 2219 else 2220 tx_q->dma_tx = addr; 2221 2222 return 0; 2223 } 2224 2225 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2226 struct stmmac_dma_conf *dma_conf) 2227 { 2228 u32 tx_count = priv->plat->tx_queues_to_use; 2229 u32 queue; 2230 int ret; 2231 2232 /* TX queues buffers and DMA */ 2233 for (queue = 0; queue < tx_count; queue++) { 2234 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2235 if (ret) 2236 goto err_dma; 2237 } 2238 2239 return 0; 2240 2241 err_dma: 2242 free_dma_tx_desc_resources(priv, dma_conf); 2243 return ret; 2244 } 2245 2246 /** 2247 * alloc_dma_desc_resources - alloc TX/RX resources. 2248 * @priv: private structure 2249 * @dma_conf: structure to take the dma data 2250 * Description: according to which descriptor can be used (extend or basic) 2251 * this function allocates the resources for TX and RX paths. In case of 2252 * reception, for example, it pre-allocated the RX socket buffer in order to 2253 * allow zero-copy mechanism. 2254 */ 2255 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2256 struct stmmac_dma_conf *dma_conf) 2257 { 2258 /* RX Allocation */ 2259 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2260 2261 if (ret) 2262 return ret; 2263 2264 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2265 2266 return ret; 2267 } 2268 2269 /** 2270 * free_dma_desc_resources - free dma desc resources 2271 * @priv: private structure 2272 * @dma_conf: structure to take the dma data 2273 */ 2274 static void free_dma_desc_resources(struct stmmac_priv *priv, 2275 struct stmmac_dma_conf *dma_conf) 2276 { 2277 /* Release the DMA TX socket buffers */ 2278 free_dma_tx_desc_resources(priv, dma_conf); 2279 2280 /* Release the DMA RX socket buffers later 2281 * to ensure all pending XDP_TX buffers are returned. 2282 */ 2283 free_dma_rx_desc_resources(priv, dma_conf); 2284 } 2285 2286 /** 2287 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2288 * @priv: driver private structure 2289 * Description: It is used for enabling the rx queues in the MAC 2290 */ 2291 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2292 { 2293 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2294 int queue; 2295 u8 mode; 2296 2297 for (queue = 0; queue < rx_queues_count; queue++) { 2298 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2299 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2300 } 2301 } 2302 2303 /** 2304 * stmmac_start_rx_dma - start RX DMA channel 2305 * @priv: driver private structure 2306 * @chan: RX channel index 2307 * Description: 2308 * This starts a RX DMA channel 2309 */ 2310 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2311 { 2312 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2313 stmmac_start_rx(priv, priv->ioaddr, chan); 2314 } 2315 2316 /** 2317 * stmmac_start_tx_dma - start TX DMA channel 2318 * @priv: driver private structure 2319 * @chan: TX channel index 2320 * Description: 2321 * This starts a TX DMA channel 2322 */ 2323 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2324 { 2325 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2326 stmmac_start_tx(priv, priv->ioaddr, chan); 2327 } 2328 2329 /** 2330 * stmmac_stop_rx_dma - stop RX DMA channel 2331 * @priv: driver private structure 2332 * @chan: RX channel index 2333 * Description: 2334 * This stops a RX DMA channel 2335 */ 2336 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2337 { 2338 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2339 stmmac_stop_rx(priv, priv->ioaddr, chan); 2340 } 2341 2342 /** 2343 * stmmac_stop_tx_dma - stop TX DMA channel 2344 * @priv: driver private structure 2345 * @chan: TX channel index 2346 * Description: 2347 * This stops a TX DMA channel 2348 */ 2349 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2350 { 2351 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2352 stmmac_stop_tx(priv, priv->ioaddr, chan); 2353 } 2354 2355 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2356 { 2357 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2358 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2359 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2360 u32 chan; 2361 2362 for (chan = 0; chan < dma_csr_ch; chan++) { 2363 struct stmmac_channel *ch = &priv->channel[chan]; 2364 unsigned long flags; 2365 2366 spin_lock_irqsave(&ch->lock, flags); 2367 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2368 spin_unlock_irqrestore(&ch->lock, flags); 2369 } 2370 } 2371 2372 /** 2373 * stmmac_start_all_dma - start all RX and TX DMA channels 2374 * @priv: driver private structure 2375 * Description: 2376 * This starts all the RX and TX DMA channels 2377 */ 2378 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2379 { 2380 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2381 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2382 u32 chan = 0; 2383 2384 for (chan = 0; chan < rx_channels_count; chan++) 2385 stmmac_start_rx_dma(priv, chan); 2386 2387 for (chan = 0; chan < tx_channels_count; chan++) 2388 stmmac_start_tx_dma(priv, chan); 2389 } 2390 2391 /** 2392 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2393 * @priv: driver private structure 2394 * Description: 2395 * This stops the RX and TX DMA channels 2396 */ 2397 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2398 { 2399 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2400 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2401 u32 chan = 0; 2402 2403 for (chan = 0; chan < rx_channels_count; chan++) 2404 stmmac_stop_rx_dma(priv, chan); 2405 2406 for (chan = 0; chan < tx_channels_count; chan++) 2407 stmmac_stop_tx_dma(priv, chan); 2408 } 2409 2410 /** 2411 * stmmac_dma_operation_mode - HW DMA operation mode 2412 * @priv: driver private structure 2413 * Description: it is used for configuring the DMA operation mode register in 2414 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2415 */ 2416 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2417 { 2418 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2419 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2420 int rxfifosz = priv->plat->rx_fifo_size; 2421 int txfifosz = priv->plat->tx_fifo_size; 2422 u32 txmode = 0; 2423 u32 rxmode = 0; 2424 u32 chan = 0; 2425 u8 qmode = 0; 2426 2427 if (rxfifosz == 0) 2428 rxfifosz = priv->dma_cap.rx_fifo_size; 2429 if (txfifosz == 0) 2430 txfifosz = priv->dma_cap.tx_fifo_size; 2431 2432 /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ 2433 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { 2434 rxfifosz /= rx_channels_count; 2435 txfifosz /= tx_channels_count; 2436 } 2437 2438 if (priv->plat->force_thresh_dma_mode) { 2439 txmode = tc; 2440 rxmode = tc; 2441 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2442 /* 2443 * In case of GMAC, SF mode can be enabled 2444 * to perform the TX COE in HW. This depends on: 2445 * 1) TX COE if actually supported 2446 * 2) There is no bugged Jumbo frame support 2447 * that needs to not insert csum in the TDES. 2448 */ 2449 txmode = SF_DMA_MODE; 2450 rxmode = SF_DMA_MODE; 2451 priv->xstats.threshold = SF_DMA_MODE; 2452 } else { 2453 txmode = tc; 2454 rxmode = SF_DMA_MODE; 2455 } 2456 2457 /* configure all channels */ 2458 for (chan = 0; chan < rx_channels_count; chan++) { 2459 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2460 u32 buf_size; 2461 2462 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2463 2464 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2465 rxfifosz, qmode); 2466 2467 if (rx_q->xsk_pool) { 2468 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2469 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2470 buf_size, 2471 chan); 2472 } else { 2473 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2474 priv->dma_conf.dma_buf_sz, 2475 chan); 2476 } 2477 } 2478 2479 for (chan = 0; chan < tx_channels_count; chan++) { 2480 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2481 2482 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2483 txfifosz, qmode); 2484 } 2485 } 2486 2487 static void stmmac_xsk_request_timestamp(void *_priv) 2488 { 2489 struct stmmac_metadata_request *meta_req = _priv; 2490 2491 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2492 *meta_req->set_ic = true; 2493 } 2494 2495 static u64 stmmac_xsk_fill_timestamp(void *_priv) 2496 { 2497 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2498 struct stmmac_priv *priv = tx_compl->priv; 2499 struct dma_desc *desc = tx_compl->desc; 2500 bool found = false; 2501 u64 ns = 0; 2502 2503 if (!priv->hwts_tx_en) 2504 return 0; 2505 2506 /* check tx tstamp status */ 2507 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2508 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2509 found = true; 2510 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2511 found = true; 2512 } 2513 2514 if (found) { 2515 ns -= priv->plat->cdc_error_adj; 2516 return ns_to_ktime(ns); 2517 } 2518 2519 return 0; 2520 } 2521 2522 static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2523 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2524 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2525 }; 2526 2527 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2528 { 2529 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2530 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2531 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2532 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2533 unsigned int entry = tx_q->cur_tx; 2534 struct dma_desc *tx_desc = NULL; 2535 struct xdp_desc xdp_desc; 2536 bool work_done = true; 2537 u32 tx_set_ic_bit = 0; 2538 2539 /* Avoids TX time-out as we are sharing with slow path */ 2540 txq_trans_cond_update(nq); 2541 2542 budget = min(budget, stmmac_tx_avail(priv, queue)); 2543 2544 while (budget-- > 0) { 2545 struct stmmac_metadata_request meta_req; 2546 struct xsk_tx_metadata *meta = NULL; 2547 dma_addr_t dma_addr; 2548 bool set_ic; 2549 2550 /* We are sharing with slow path and stop XSK TX desc submission when 2551 * available TX ring is less than threshold. 2552 */ 2553 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2554 !netif_carrier_ok(priv->dev)) { 2555 work_done = false; 2556 break; 2557 } 2558 2559 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2560 break; 2561 2562 if (priv->est && priv->est->enable && 2563 priv->est->max_sdu[queue] && 2564 xdp_desc.len > priv->est->max_sdu[queue]) { 2565 priv->xstats.max_sdu_txq_drop[queue]++; 2566 continue; 2567 } 2568 2569 if (likely(priv->extend_desc)) 2570 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2571 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2572 tx_desc = &tx_q->dma_entx[entry].basic; 2573 else 2574 tx_desc = tx_q->dma_tx + entry; 2575 2576 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2577 meta = xsk_buff_get_metadata(pool, xdp_desc.addr); 2578 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2579 2580 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2581 2582 /* To return XDP buffer to XSK pool, we simple call 2583 * xsk_tx_completed(), so we don't need to fill up 2584 * 'buf' and 'xdpf'. 2585 */ 2586 tx_q->tx_skbuff_dma[entry].buf = 0; 2587 tx_q->xdpf[entry] = NULL; 2588 2589 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2590 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2591 tx_q->tx_skbuff_dma[entry].last_segment = true; 2592 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2593 2594 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2595 2596 tx_q->tx_count_frames++; 2597 2598 if (!priv->tx_coal_frames[queue]) 2599 set_ic = false; 2600 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2601 set_ic = true; 2602 else 2603 set_ic = false; 2604 2605 meta_req.priv = priv; 2606 meta_req.tx_desc = tx_desc; 2607 meta_req.set_ic = &set_ic; 2608 xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops, 2609 &meta_req); 2610 if (set_ic) { 2611 tx_q->tx_count_frames = 0; 2612 stmmac_set_tx_ic(priv, tx_desc); 2613 tx_set_ic_bit++; 2614 } 2615 2616 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2617 true, priv->mode, true, true, 2618 xdp_desc.len); 2619 2620 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 2621 2622 xsk_tx_metadata_to_compl(meta, 2623 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2624 2625 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2626 entry = tx_q->cur_tx; 2627 } 2628 u64_stats_update_begin(&txq_stats->napi_syncp); 2629 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2630 u64_stats_update_end(&txq_stats->napi_syncp); 2631 2632 if (tx_desc) { 2633 stmmac_flush_tx_descriptors(priv, queue); 2634 xsk_tx_release(pool); 2635 } 2636 2637 /* Return true if all of the 3 conditions are met 2638 * a) TX Budget is still available 2639 * b) work_done = true when XSK TX desc peek is empty (no more 2640 * pending XSK TX for transmission) 2641 */ 2642 return !!budget && work_done; 2643 } 2644 2645 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2646 { 2647 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2648 tc += 64; 2649 2650 if (priv->plat->force_thresh_dma_mode) 2651 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2652 else 2653 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2654 chan); 2655 2656 priv->xstats.threshold = tc; 2657 } 2658 } 2659 2660 /** 2661 * stmmac_tx_clean - to manage the transmission completion 2662 * @priv: driver private structure 2663 * @budget: napi budget limiting this functions packet handling 2664 * @queue: TX queue index 2665 * @pending_packets: signal to arm the TX coal timer 2666 * Description: it reclaims the transmit resources after transmission completes. 2667 * If some packets still needs to be handled, due to TX coalesce, set 2668 * pending_packets to true to make NAPI arm the TX coal timer. 2669 */ 2670 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2671 bool *pending_packets) 2672 { 2673 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2674 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2675 unsigned int bytes_compl = 0, pkts_compl = 0; 2676 unsigned int entry, xmits = 0, count = 0; 2677 u32 tx_packets = 0, tx_errors = 0; 2678 2679 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2680 2681 tx_q->xsk_frames_done = 0; 2682 2683 entry = tx_q->dirty_tx; 2684 2685 /* Try to clean all TX complete frame in 1 shot */ 2686 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2687 struct xdp_frame *xdpf; 2688 struct sk_buff *skb; 2689 struct dma_desc *p; 2690 int status; 2691 2692 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2693 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2694 xdpf = tx_q->xdpf[entry]; 2695 skb = NULL; 2696 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2697 xdpf = NULL; 2698 skb = tx_q->tx_skbuff[entry]; 2699 } else { 2700 xdpf = NULL; 2701 skb = NULL; 2702 } 2703 2704 if (priv->extend_desc) 2705 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2706 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2707 p = &tx_q->dma_entx[entry].basic; 2708 else 2709 p = tx_q->dma_tx + entry; 2710 2711 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2712 /* Check if the descriptor is owned by the DMA */ 2713 if (unlikely(status & tx_dma_own)) 2714 break; 2715 2716 count++; 2717 2718 /* Make sure descriptor fields are read after reading 2719 * the own bit. 2720 */ 2721 dma_rmb(); 2722 2723 /* Just consider the last segment and ...*/ 2724 if (likely(!(status & tx_not_ls))) { 2725 /* ... verify the status error condition */ 2726 if (unlikely(status & tx_err)) { 2727 tx_errors++; 2728 if (unlikely(status & tx_err_bump_tc)) 2729 stmmac_bump_dma_threshold(priv, queue); 2730 } else { 2731 tx_packets++; 2732 } 2733 if (skb) { 2734 stmmac_get_tx_hwtstamp(priv, p, skb); 2735 } else if (tx_q->xsk_pool && 2736 xp_tx_metadata_enabled(tx_q->xsk_pool)) { 2737 struct stmmac_xsk_tx_complete tx_compl = { 2738 .priv = priv, 2739 .desc = p, 2740 }; 2741 2742 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2743 &stmmac_xsk_tx_metadata_ops, 2744 &tx_compl); 2745 } 2746 } 2747 2748 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2749 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2750 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2751 dma_unmap_page(priv->device, 2752 tx_q->tx_skbuff_dma[entry].buf, 2753 tx_q->tx_skbuff_dma[entry].len, 2754 DMA_TO_DEVICE); 2755 else 2756 dma_unmap_single(priv->device, 2757 tx_q->tx_skbuff_dma[entry].buf, 2758 tx_q->tx_skbuff_dma[entry].len, 2759 DMA_TO_DEVICE); 2760 tx_q->tx_skbuff_dma[entry].buf = 0; 2761 tx_q->tx_skbuff_dma[entry].len = 0; 2762 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2763 } 2764 2765 stmmac_clean_desc3(priv, tx_q, p); 2766 2767 tx_q->tx_skbuff_dma[entry].last_segment = false; 2768 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2769 2770 if (xdpf && 2771 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2772 xdp_return_frame_rx_napi(xdpf); 2773 tx_q->xdpf[entry] = NULL; 2774 } 2775 2776 if (xdpf && 2777 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2778 xdp_return_frame(xdpf); 2779 tx_q->xdpf[entry] = NULL; 2780 } 2781 2782 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2783 tx_q->xsk_frames_done++; 2784 2785 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2786 if (likely(skb)) { 2787 pkts_compl++; 2788 bytes_compl += skb->len; 2789 dev_consume_skb_any(skb); 2790 tx_q->tx_skbuff[entry] = NULL; 2791 } 2792 } 2793 2794 stmmac_release_tx_desc(priv, p, priv->mode); 2795 2796 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 2797 } 2798 tx_q->dirty_tx = entry; 2799 2800 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2801 pkts_compl, bytes_compl); 2802 2803 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2804 queue))) && 2805 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2806 2807 netif_dbg(priv, tx_done, priv->dev, 2808 "%s: restart transmit\n", __func__); 2809 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2810 } 2811 2812 if (tx_q->xsk_pool) { 2813 bool work_done; 2814 2815 if (tx_q->xsk_frames_done) 2816 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2817 2818 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2819 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2820 2821 /* For XSK TX, we try to send as many as possible. 2822 * If XSK work done (XSK TX desc empty and budget still 2823 * available), return "budget - 1" to reenable TX IRQ. 2824 * Else, return "budget" to make NAPI continue polling. 2825 */ 2826 work_done = stmmac_xdp_xmit_zc(priv, queue, 2827 STMMAC_XSK_TX_BUDGET_MAX); 2828 if (work_done) 2829 xmits = budget - 1; 2830 else 2831 xmits = budget; 2832 } 2833 2834 if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode) 2835 stmmac_restart_sw_lpi_timer(priv); 2836 2837 /* We still have pending packets, let's call for a new scheduling */ 2838 if (tx_q->dirty_tx != tx_q->cur_tx) 2839 *pending_packets = true; 2840 2841 u64_stats_update_begin(&txq_stats->napi_syncp); 2842 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2843 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2844 u64_stats_inc(&txq_stats->napi.tx_clean); 2845 u64_stats_update_end(&txq_stats->napi_syncp); 2846 2847 priv->xstats.tx_errors += tx_errors; 2848 2849 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2850 2851 /* Combine decisions from TX clean and XSK TX */ 2852 return max(count, xmits); 2853 } 2854 2855 /** 2856 * stmmac_tx_err - to manage the tx error 2857 * @priv: driver private structure 2858 * @chan: channel index 2859 * Description: it cleans the descriptors and restarts the transmission 2860 * in case of transmission errors. 2861 */ 2862 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2863 { 2864 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2865 2866 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2867 2868 stmmac_stop_tx_dma(priv, chan); 2869 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 2870 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 2871 stmmac_reset_tx_queue(priv, chan); 2872 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2873 tx_q->dma_tx_phy, chan); 2874 stmmac_start_tx_dma(priv, chan); 2875 2876 priv->xstats.tx_errors++; 2877 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2878 } 2879 2880 /** 2881 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2882 * @priv: driver private structure 2883 * @txmode: TX operating mode 2884 * @rxmode: RX operating mode 2885 * @chan: channel index 2886 * Description: it is used for configuring of the DMA operation mode in 2887 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2888 * mode. 2889 */ 2890 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2891 u32 rxmode, u32 chan) 2892 { 2893 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2894 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2895 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2896 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2897 int rxfifosz = priv->plat->rx_fifo_size; 2898 int txfifosz = priv->plat->tx_fifo_size; 2899 2900 if (rxfifosz == 0) 2901 rxfifosz = priv->dma_cap.rx_fifo_size; 2902 if (txfifosz == 0) 2903 txfifosz = priv->dma_cap.tx_fifo_size; 2904 2905 /* Adjust for real per queue fifo size */ 2906 rxfifosz /= rx_channels_count; 2907 txfifosz /= tx_channels_count; 2908 2909 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2910 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2911 } 2912 2913 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2914 { 2915 int ret; 2916 2917 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2918 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2919 if (ret && (ret != -EINVAL)) { 2920 stmmac_global_err(priv); 2921 return true; 2922 } 2923 2924 return false; 2925 } 2926 2927 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 2928 { 2929 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2930 &priv->xstats, chan, dir); 2931 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2932 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2933 struct stmmac_channel *ch = &priv->channel[chan]; 2934 struct napi_struct *rx_napi; 2935 struct napi_struct *tx_napi; 2936 unsigned long flags; 2937 2938 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 2939 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2940 2941 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 2942 if (napi_schedule_prep(rx_napi)) { 2943 spin_lock_irqsave(&ch->lock, flags); 2944 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 2945 spin_unlock_irqrestore(&ch->lock, flags); 2946 __napi_schedule(rx_napi); 2947 } 2948 } 2949 2950 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 2951 if (napi_schedule_prep(tx_napi)) { 2952 spin_lock_irqsave(&ch->lock, flags); 2953 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 2954 spin_unlock_irqrestore(&ch->lock, flags); 2955 __napi_schedule(tx_napi); 2956 } 2957 } 2958 2959 return status; 2960 } 2961 2962 /** 2963 * stmmac_dma_interrupt - DMA ISR 2964 * @priv: driver private structure 2965 * Description: this is the DMA ISR. It is called by the main ISR. 2966 * It calls the dwmac dma routine and schedule poll method in case of some 2967 * work can be done. 2968 */ 2969 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2970 { 2971 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2972 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2973 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2974 tx_channel_count : rx_channel_count; 2975 u32 chan; 2976 int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2977 2978 /* Make sure we never check beyond our status buffer. */ 2979 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2980 channels_to_check = ARRAY_SIZE(status); 2981 2982 for (chan = 0; chan < channels_to_check; chan++) 2983 status[chan] = stmmac_napi_check(priv, chan, 2984 DMA_DIR_RXTX); 2985 2986 for (chan = 0; chan < tx_channel_count; chan++) { 2987 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2988 /* Try to bump up the dma threshold on this failure */ 2989 stmmac_bump_dma_threshold(priv, chan); 2990 } else if (unlikely(status[chan] == tx_hard_error)) { 2991 stmmac_tx_err(priv, chan); 2992 } 2993 } 2994 } 2995 2996 /** 2997 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2998 * @priv: driver private structure 2999 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 3000 */ 3001 static void stmmac_mmc_setup(struct stmmac_priv *priv) 3002 { 3003 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 3004 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 3005 3006 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 3007 3008 if (priv->dma_cap.rmon) { 3009 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 3010 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 3011 } else 3012 netdev_info(priv->dev, "No MAC Management Counters available\n"); 3013 } 3014 3015 /** 3016 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 3017 * @priv: driver private structure 3018 * Description: 3019 * new GMAC chip generations have a new register to indicate the 3020 * presence of the optional feature/functions. 3021 * This can be also used to override the value passed through the 3022 * platform and necessary for old MAC10/100 and GMAC chips. 3023 */ 3024 static int stmmac_get_hw_features(struct stmmac_priv *priv) 3025 { 3026 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 3027 } 3028 3029 /** 3030 * stmmac_check_ether_addr - check if the MAC addr is valid 3031 * @priv: driver private structure 3032 * Description: 3033 * it is to verify if the MAC address is valid, in case of failures it 3034 * generates a random MAC address 3035 */ 3036 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 3037 { 3038 u8 addr[ETH_ALEN]; 3039 3040 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 3041 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 3042 if (is_valid_ether_addr(addr)) 3043 eth_hw_addr_set(priv->dev, addr); 3044 else 3045 eth_hw_addr_random(priv->dev); 3046 dev_info(priv->device, "device MAC address %pM\n", 3047 priv->dev->dev_addr); 3048 } 3049 } 3050 3051 /** 3052 * stmmac_init_dma_engine - DMA init. 3053 * @priv: driver private structure 3054 * Description: 3055 * It inits the DMA invoking the specific MAC/GMAC callback. 3056 * Some DMA parameters can be passed from the platform; 3057 * in case of these are not passed a default is kept for the MAC or GMAC. 3058 */ 3059 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 3060 { 3061 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3062 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3063 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 3064 struct stmmac_rx_queue *rx_q; 3065 struct stmmac_tx_queue *tx_q; 3066 u32 chan = 0; 3067 int ret = 0; 3068 3069 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 3070 dev_err(priv->device, "Invalid DMA configuration\n"); 3071 return -EINVAL; 3072 } 3073 3074 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 3075 priv->plat->dma_cfg->atds = 1; 3076 3077 ret = stmmac_reset(priv, priv->ioaddr); 3078 if (ret) { 3079 dev_err(priv->device, "Failed to reset the dma\n"); 3080 return ret; 3081 } 3082 3083 /* DMA Configuration */ 3084 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); 3085 3086 if (priv->plat->axi) 3087 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3088 3089 /* DMA CSR Channel configuration */ 3090 for (chan = 0; chan < dma_csr_ch; chan++) { 3091 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3092 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3093 } 3094 3095 /* DMA RX Channel Configuration */ 3096 for (chan = 0; chan < rx_channels_count; chan++) { 3097 rx_q = &priv->dma_conf.rx_queue[chan]; 3098 3099 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3100 rx_q->dma_rx_phy, chan); 3101 3102 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 3103 (rx_q->buf_alloc_num * 3104 sizeof(struct dma_desc)); 3105 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 3106 rx_q->rx_tail_addr, chan); 3107 } 3108 3109 /* DMA TX Channel Configuration */ 3110 for (chan = 0; chan < tx_channels_count; chan++) { 3111 tx_q = &priv->dma_conf.tx_queue[chan]; 3112 3113 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3114 tx_q->dma_tx_phy, chan); 3115 3116 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 3117 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 3118 tx_q->tx_tail_addr, chan); 3119 } 3120 3121 return ret; 3122 } 3123 3124 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3125 { 3126 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3127 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3128 struct stmmac_channel *ch; 3129 struct napi_struct *napi; 3130 3131 if (!tx_coal_timer) 3132 return; 3133 3134 ch = &priv->channel[tx_q->queue_index]; 3135 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3136 3137 /* Arm timer only if napi is not already scheduled. 3138 * Try to cancel any timer if napi is scheduled, timer will be armed 3139 * again in the next scheduled napi. 3140 */ 3141 if (unlikely(!napi_is_scheduled(napi))) 3142 hrtimer_start(&tx_q->txtimer, 3143 STMMAC_COAL_TIMER(tx_coal_timer), 3144 HRTIMER_MODE_REL); 3145 else 3146 hrtimer_try_to_cancel(&tx_q->txtimer); 3147 } 3148 3149 /** 3150 * stmmac_tx_timer - mitigation sw timer for tx. 3151 * @t: data pointer 3152 * Description: 3153 * This is the timer handler to directly invoke the stmmac_tx_clean. 3154 */ 3155 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3156 { 3157 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3158 struct stmmac_priv *priv = tx_q->priv_data; 3159 struct stmmac_channel *ch; 3160 struct napi_struct *napi; 3161 3162 ch = &priv->channel[tx_q->queue_index]; 3163 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3164 3165 if (likely(napi_schedule_prep(napi))) { 3166 unsigned long flags; 3167 3168 spin_lock_irqsave(&ch->lock, flags); 3169 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3170 spin_unlock_irqrestore(&ch->lock, flags); 3171 __napi_schedule(napi); 3172 } 3173 3174 return HRTIMER_NORESTART; 3175 } 3176 3177 /** 3178 * stmmac_init_coalesce - init mitigation options. 3179 * @priv: driver private structure 3180 * Description: 3181 * This inits the coalesce parameters: i.e. timer rate, 3182 * timer handler and default threshold used for enabling the 3183 * interrupt on completion bit. 3184 */ 3185 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3186 { 3187 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3188 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3189 u32 chan; 3190 3191 for (chan = 0; chan < tx_channel_count; chan++) { 3192 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3193 3194 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3195 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3196 3197 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3198 tx_q->txtimer.function = stmmac_tx_timer; 3199 } 3200 3201 for (chan = 0; chan < rx_channel_count; chan++) 3202 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3203 } 3204 3205 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3206 { 3207 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3208 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3209 u32 chan; 3210 3211 /* set TX ring length */ 3212 for (chan = 0; chan < tx_channels_count; chan++) 3213 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3214 (priv->dma_conf.dma_tx_size - 1), chan); 3215 3216 /* set RX ring length */ 3217 for (chan = 0; chan < rx_channels_count; chan++) 3218 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3219 (priv->dma_conf.dma_rx_size - 1), chan); 3220 } 3221 3222 /** 3223 * stmmac_set_tx_queue_weight - Set TX queue weight 3224 * @priv: driver private structure 3225 * Description: It is used for setting TX queues weight 3226 */ 3227 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3228 { 3229 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3230 u32 weight; 3231 u32 queue; 3232 3233 for (queue = 0; queue < tx_queues_count; queue++) { 3234 weight = priv->plat->tx_queues_cfg[queue].weight; 3235 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3236 } 3237 } 3238 3239 /** 3240 * stmmac_configure_cbs - Configure CBS in TX queue 3241 * @priv: driver private structure 3242 * Description: It is used for configuring CBS in AVB TX queues 3243 */ 3244 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3245 { 3246 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3247 u32 mode_to_use; 3248 u32 queue; 3249 3250 /* queue 0 is reserved for legacy traffic */ 3251 for (queue = 1; queue < tx_queues_count; queue++) { 3252 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3253 if (mode_to_use == MTL_QUEUE_DCB) 3254 continue; 3255 3256 stmmac_config_cbs(priv, priv->hw, 3257 priv->plat->tx_queues_cfg[queue].send_slope, 3258 priv->plat->tx_queues_cfg[queue].idle_slope, 3259 priv->plat->tx_queues_cfg[queue].high_credit, 3260 priv->plat->tx_queues_cfg[queue].low_credit, 3261 queue); 3262 } 3263 } 3264 3265 /** 3266 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3267 * @priv: driver private structure 3268 * Description: It is used for mapping RX queues to RX dma channels 3269 */ 3270 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3271 { 3272 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3273 u32 queue; 3274 u32 chan; 3275 3276 for (queue = 0; queue < rx_queues_count; queue++) { 3277 chan = priv->plat->rx_queues_cfg[queue].chan; 3278 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3279 } 3280 } 3281 3282 /** 3283 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3284 * @priv: driver private structure 3285 * Description: It is used for configuring the RX Queue Priority 3286 */ 3287 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3288 { 3289 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3290 u32 queue; 3291 u32 prio; 3292 3293 for (queue = 0; queue < rx_queues_count; queue++) { 3294 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3295 continue; 3296 3297 prio = priv->plat->rx_queues_cfg[queue].prio; 3298 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3299 } 3300 } 3301 3302 /** 3303 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3304 * @priv: driver private structure 3305 * Description: It is used for configuring the TX Queue Priority 3306 */ 3307 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3308 { 3309 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3310 u32 queue; 3311 u32 prio; 3312 3313 for (queue = 0; queue < tx_queues_count; queue++) { 3314 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3315 continue; 3316 3317 prio = priv->plat->tx_queues_cfg[queue].prio; 3318 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3319 } 3320 } 3321 3322 /** 3323 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3324 * @priv: driver private structure 3325 * Description: It is used for configuring the RX queue routing 3326 */ 3327 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3328 { 3329 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3330 u32 queue; 3331 u8 packet; 3332 3333 for (queue = 0; queue < rx_queues_count; queue++) { 3334 /* no specific packet type routing specified for the queue */ 3335 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3336 continue; 3337 3338 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3339 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3340 } 3341 } 3342 3343 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3344 { 3345 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3346 priv->rss.enable = false; 3347 return; 3348 } 3349 3350 if (priv->dev->features & NETIF_F_RXHASH) 3351 priv->rss.enable = true; 3352 else 3353 priv->rss.enable = false; 3354 3355 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3356 priv->plat->rx_queues_to_use); 3357 } 3358 3359 /** 3360 * stmmac_mtl_configuration - Configure MTL 3361 * @priv: driver private structure 3362 * Description: It is used for configurring MTL 3363 */ 3364 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3365 { 3366 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3367 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3368 3369 if (tx_queues_count > 1) 3370 stmmac_set_tx_queue_weight(priv); 3371 3372 /* Configure MTL RX algorithms */ 3373 if (rx_queues_count > 1) 3374 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3375 priv->plat->rx_sched_algorithm); 3376 3377 /* Configure MTL TX algorithms */ 3378 if (tx_queues_count > 1) 3379 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3380 priv->plat->tx_sched_algorithm); 3381 3382 /* Configure CBS in AVB TX queues */ 3383 if (tx_queues_count > 1) 3384 stmmac_configure_cbs(priv); 3385 3386 /* Map RX MTL to DMA channels */ 3387 stmmac_rx_queue_dma_chan_map(priv); 3388 3389 /* Enable MAC RX Queues */ 3390 stmmac_mac_enable_rx_queues(priv); 3391 3392 /* Set RX priorities */ 3393 if (rx_queues_count > 1) 3394 stmmac_mac_config_rx_queues_prio(priv); 3395 3396 /* Set TX priorities */ 3397 if (tx_queues_count > 1) 3398 stmmac_mac_config_tx_queues_prio(priv); 3399 3400 /* Set RX routing */ 3401 if (rx_queues_count > 1) 3402 stmmac_mac_config_rx_queues_routing(priv); 3403 3404 /* Receive Side Scaling */ 3405 if (rx_queues_count > 1) 3406 stmmac_mac_config_rss(priv); 3407 } 3408 3409 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3410 { 3411 if (priv->dma_cap.asp) { 3412 netdev_info(priv->dev, "Enabling Safety Features\n"); 3413 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3414 priv->plat->safety_feat_cfg); 3415 } else { 3416 netdev_info(priv->dev, "No Safety Features support found\n"); 3417 } 3418 } 3419 3420 /** 3421 * stmmac_hw_setup - setup mac in a usable state. 3422 * @dev : pointer to the device structure. 3423 * @ptp_register: register PTP if set 3424 * Description: 3425 * this is the main function to setup the HW in a usable state because the 3426 * dma engine is reset, the core registers are configured (e.g. AXI, 3427 * Checksum features, timers). The DMA is ready to start receiving and 3428 * transmitting. 3429 * Return value: 3430 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3431 * file on failure. 3432 */ 3433 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) 3434 { 3435 struct stmmac_priv *priv = netdev_priv(dev); 3436 u32 rx_cnt = priv->plat->rx_queues_to_use; 3437 u32 tx_cnt = priv->plat->tx_queues_to_use; 3438 bool sph_en; 3439 u32 chan; 3440 int ret; 3441 3442 /* Make sure RX clock is enabled */ 3443 if (priv->hw->phylink_pcs) 3444 phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs); 3445 3446 /* DMA initialization and SW reset */ 3447 ret = stmmac_init_dma_engine(priv); 3448 if (ret < 0) { 3449 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3450 __func__); 3451 return ret; 3452 } 3453 3454 /* Copy the MAC addr into the HW */ 3455 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3456 3457 /* PS and related bits will be programmed according to the speed */ 3458 if (priv->hw->pcs) { 3459 int speed = priv->plat->mac_port_sel_speed; 3460 3461 if ((speed == SPEED_10) || (speed == SPEED_100) || 3462 (speed == SPEED_1000)) { 3463 priv->hw->ps = speed; 3464 } else { 3465 dev_warn(priv->device, "invalid port speed\n"); 3466 priv->hw->ps = 0; 3467 } 3468 } 3469 3470 /* Initialize the MAC Core */ 3471 stmmac_core_init(priv, priv->hw, dev); 3472 3473 /* Initialize MTL*/ 3474 stmmac_mtl_configuration(priv); 3475 3476 /* Initialize Safety Features */ 3477 stmmac_safety_feat_configuration(priv); 3478 3479 ret = stmmac_rx_ipc(priv, priv->hw); 3480 if (!ret) { 3481 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3482 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3483 priv->hw->rx_csum = 0; 3484 } 3485 3486 /* Enable the MAC Rx/Tx */ 3487 stmmac_mac_set(priv, priv->ioaddr, true); 3488 3489 /* Set the HW DMA mode and the COE */ 3490 stmmac_dma_operation_mode(priv); 3491 3492 stmmac_mmc_setup(priv); 3493 3494 if (ptp_register) { 3495 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 3496 if (ret < 0) 3497 netdev_warn(priv->dev, 3498 "failed to enable PTP reference clock: %pe\n", 3499 ERR_PTR(ret)); 3500 } 3501 3502 ret = stmmac_init_ptp(priv); 3503 if (ret == -EOPNOTSUPP) 3504 netdev_info(priv->dev, "PTP not supported by HW\n"); 3505 else if (ret) 3506 netdev_warn(priv->dev, "PTP init failed\n"); 3507 else if (ptp_register) 3508 stmmac_ptp_register(priv); 3509 3510 if (priv->use_riwt) { 3511 u32 queue; 3512 3513 for (queue = 0; queue < rx_cnt; queue++) { 3514 if (!priv->rx_riwt[queue]) 3515 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3516 3517 stmmac_rx_watchdog(priv, priv->ioaddr, 3518 priv->rx_riwt[queue], queue); 3519 } 3520 } 3521 3522 if (priv->hw->pcs) 3523 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); 3524 3525 /* set TX and RX rings length */ 3526 stmmac_set_rings_length(priv); 3527 3528 /* Enable TSO */ 3529 if (priv->tso) { 3530 for (chan = 0; chan < tx_cnt; chan++) { 3531 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3532 3533 /* TSO and TBS cannot co-exist */ 3534 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3535 continue; 3536 3537 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3538 } 3539 } 3540 3541 /* Enable Split Header */ 3542 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 3543 for (chan = 0; chan < rx_cnt; chan++) 3544 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3545 3546 3547 /* VLAN Tag Insertion */ 3548 if (priv->dma_cap.vlins) 3549 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3550 3551 /* TBS */ 3552 for (chan = 0; chan < tx_cnt; chan++) { 3553 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3554 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3555 3556 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3557 } 3558 3559 /* Configure real RX and TX queues */ 3560 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3561 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3562 3563 /* Start the ball rolling... */ 3564 stmmac_start_all_dma(priv); 3565 3566 stmmac_set_hw_vlan_mode(priv, priv->hw); 3567 3568 return 0; 3569 } 3570 3571 static void stmmac_hw_teardown(struct net_device *dev) 3572 { 3573 struct stmmac_priv *priv = netdev_priv(dev); 3574 3575 clk_disable_unprepare(priv->plat->clk_ptp_ref); 3576 } 3577 3578 static void stmmac_free_irq(struct net_device *dev, 3579 enum request_irq_err irq_err, int irq_idx) 3580 { 3581 struct stmmac_priv *priv = netdev_priv(dev); 3582 int j; 3583 3584 switch (irq_err) { 3585 case REQ_IRQ_ERR_ALL: 3586 irq_idx = priv->plat->tx_queues_to_use; 3587 fallthrough; 3588 case REQ_IRQ_ERR_TX: 3589 for (j = irq_idx - 1; j >= 0; j--) { 3590 if (priv->tx_irq[j] > 0) { 3591 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3592 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); 3593 } 3594 } 3595 irq_idx = priv->plat->rx_queues_to_use; 3596 fallthrough; 3597 case REQ_IRQ_ERR_RX: 3598 for (j = irq_idx - 1; j >= 0; j--) { 3599 if (priv->rx_irq[j] > 0) { 3600 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3601 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); 3602 } 3603 } 3604 3605 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3606 free_irq(priv->sfty_ue_irq, dev); 3607 fallthrough; 3608 case REQ_IRQ_ERR_SFTY_UE: 3609 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3610 free_irq(priv->sfty_ce_irq, dev); 3611 fallthrough; 3612 case REQ_IRQ_ERR_SFTY_CE: 3613 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) 3614 free_irq(priv->lpi_irq, dev); 3615 fallthrough; 3616 case REQ_IRQ_ERR_LPI: 3617 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3618 free_irq(priv->wol_irq, dev); 3619 fallthrough; 3620 case REQ_IRQ_ERR_SFTY: 3621 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) 3622 free_irq(priv->sfty_irq, dev); 3623 fallthrough; 3624 case REQ_IRQ_ERR_WOL: 3625 free_irq(dev->irq, dev); 3626 fallthrough; 3627 case REQ_IRQ_ERR_MAC: 3628 case REQ_IRQ_ERR_NO: 3629 /* If MAC IRQ request error, no more IRQ to free */ 3630 break; 3631 } 3632 } 3633 3634 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3635 { 3636 struct stmmac_priv *priv = netdev_priv(dev); 3637 enum request_irq_err irq_err; 3638 cpumask_t cpu_mask; 3639 int irq_idx = 0; 3640 char *int_name; 3641 int ret; 3642 int i; 3643 3644 /* For common interrupt */ 3645 int_name = priv->int_name_mac; 3646 sprintf(int_name, "%s:%s", dev->name, "mac"); 3647 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3648 0, int_name, dev); 3649 if (unlikely(ret < 0)) { 3650 netdev_err(priv->dev, 3651 "%s: alloc mac MSI %d (error: %d)\n", 3652 __func__, dev->irq, ret); 3653 irq_err = REQ_IRQ_ERR_MAC; 3654 goto irq_error; 3655 } 3656 3657 /* Request the Wake IRQ in case of another line 3658 * is used for WoL 3659 */ 3660 priv->wol_irq_disabled = true; 3661 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3662 int_name = priv->int_name_wol; 3663 sprintf(int_name, "%s:%s", dev->name, "wol"); 3664 ret = request_irq(priv->wol_irq, 3665 stmmac_mac_interrupt, 3666 0, int_name, dev); 3667 if (unlikely(ret < 0)) { 3668 netdev_err(priv->dev, 3669 "%s: alloc wol MSI %d (error: %d)\n", 3670 __func__, priv->wol_irq, ret); 3671 irq_err = REQ_IRQ_ERR_WOL; 3672 goto irq_error; 3673 } 3674 } 3675 3676 /* Request the LPI IRQ in case of another line 3677 * is used for LPI 3678 */ 3679 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3680 int_name = priv->int_name_lpi; 3681 sprintf(int_name, "%s:%s", dev->name, "lpi"); 3682 ret = request_irq(priv->lpi_irq, 3683 stmmac_mac_interrupt, 3684 0, int_name, dev); 3685 if (unlikely(ret < 0)) { 3686 netdev_err(priv->dev, 3687 "%s: alloc lpi MSI %d (error: %d)\n", 3688 __func__, priv->lpi_irq, ret); 3689 irq_err = REQ_IRQ_ERR_LPI; 3690 goto irq_error; 3691 } 3692 } 3693 3694 /* Request the common Safety Feature Correctible/Uncorrectible 3695 * Error line in case of another line is used 3696 */ 3697 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3698 int_name = priv->int_name_sfty; 3699 sprintf(int_name, "%s:%s", dev->name, "safety"); 3700 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3701 0, int_name, dev); 3702 if (unlikely(ret < 0)) { 3703 netdev_err(priv->dev, 3704 "%s: alloc sfty MSI %d (error: %d)\n", 3705 __func__, priv->sfty_irq, ret); 3706 irq_err = REQ_IRQ_ERR_SFTY; 3707 goto irq_error; 3708 } 3709 } 3710 3711 /* Request the Safety Feature Correctible Error line in 3712 * case of another line is used 3713 */ 3714 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3715 int_name = priv->int_name_sfty_ce; 3716 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3717 ret = request_irq(priv->sfty_ce_irq, 3718 stmmac_safety_interrupt, 3719 0, int_name, dev); 3720 if (unlikely(ret < 0)) { 3721 netdev_err(priv->dev, 3722 "%s: alloc sfty ce MSI %d (error: %d)\n", 3723 __func__, priv->sfty_ce_irq, ret); 3724 irq_err = REQ_IRQ_ERR_SFTY_CE; 3725 goto irq_error; 3726 } 3727 } 3728 3729 /* Request the Safety Feature Uncorrectible Error line in 3730 * case of another line is used 3731 */ 3732 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3733 int_name = priv->int_name_sfty_ue; 3734 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3735 ret = request_irq(priv->sfty_ue_irq, 3736 stmmac_safety_interrupt, 3737 0, int_name, dev); 3738 if (unlikely(ret < 0)) { 3739 netdev_err(priv->dev, 3740 "%s: alloc sfty ue MSI %d (error: %d)\n", 3741 __func__, priv->sfty_ue_irq, ret); 3742 irq_err = REQ_IRQ_ERR_SFTY_UE; 3743 goto irq_error; 3744 } 3745 } 3746 3747 /* Request Rx MSI irq */ 3748 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3749 if (i >= MTL_MAX_RX_QUEUES) 3750 break; 3751 if (priv->rx_irq[i] == 0) 3752 continue; 3753 3754 int_name = priv->int_name_rx_irq[i]; 3755 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3756 ret = request_irq(priv->rx_irq[i], 3757 stmmac_msi_intr_rx, 3758 0, int_name, &priv->dma_conf.rx_queue[i]); 3759 if (unlikely(ret < 0)) { 3760 netdev_err(priv->dev, 3761 "%s: alloc rx-%d MSI %d (error: %d)\n", 3762 __func__, i, priv->rx_irq[i], ret); 3763 irq_err = REQ_IRQ_ERR_RX; 3764 irq_idx = i; 3765 goto irq_error; 3766 } 3767 cpumask_clear(&cpu_mask); 3768 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3769 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); 3770 } 3771 3772 /* Request Tx MSI irq */ 3773 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3774 if (i >= MTL_MAX_TX_QUEUES) 3775 break; 3776 if (priv->tx_irq[i] == 0) 3777 continue; 3778 3779 int_name = priv->int_name_tx_irq[i]; 3780 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3781 ret = request_irq(priv->tx_irq[i], 3782 stmmac_msi_intr_tx, 3783 0, int_name, &priv->dma_conf.tx_queue[i]); 3784 if (unlikely(ret < 0)) { 3785 netdev_err(priv->dev, 3786 "%s: alloc tx-%d MSI %d (error: %d)\n", 3787 __func__, i, priv->tx_irq[i], ret); 3788 irq_err = REQ_IRQ_ERR_TX; 3789 irq_idx = i; 3790 goto irq_error; 3791 } 3792 cpumask_clear(&cpu_mask); 3793 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3794 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); 3795 } 3796 3797 return 0; 3798 3799 irq_error: 3800 stmmac_free_irq(dev, irq_err, irq_idx); 3801 return ret; 3802 } 3803 3804 static int stmmac_request_irq_single(struct net_device *dev) 3805 { 3806 struct stmmac_priv *priv = netdev_priv(dev); 3807 enum request_irq_err irq_err; 3808 int ret; 3809 3810 ret = request_irq(dev->irq, stmmac_interrupt, 3811 IRQF_SHARED, dev->name, dev); 3812 if (unlikely(ret < 0)) { 3813 netdev_err(priv->dev, 3814 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3815 __func__, dev->irq, ret); 3816 irq_err = REQ_IRQ_ERR_MAC; 3817 goto irq_error; 3818 } 3819 3820 /* Request the Wake IRQ in case of another line 3821 * is used for WoL 3822 */ 3823 priv->wol_irq_disabled = true; 3824 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3825 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3826 IRQF_SHARED, dev->name, dev); 3827 if (unlikely(ret < 0)) { 3828 netdev_err(priv->dev, 3829 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3830 __func__, priv->wol_irq, ret); 3831 irq_err = REQ_IRQ_ERR_WOL; 3832 goto irq_error; 3833 } 3834 } 3835 3836 /* Request the IRQ lines */ 3837 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3838 ret = request_irq(priv->lpi_irq, stmmac_interrupt, 3839 IRQF_SHARED, dev->name, dev); 3840 if (unlikely(ret < 0)) { 3841 netdev_err(priv->dev, 3842 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 3843 __func__, priv->lpi_irq, ret); 3844 irq_err = REQ_IRQ_ERR_LPI; 3845 goto irq_error; 3846 } 3847 } 3848 3849 /* Request the common Safety Feature Correctible/Uncorrectible 3850 * Error line in case of another line is used 3851 */ 3852 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3853 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3854 IRQF_SHARED, dev->name, dev); 3855 if (unlikely(ret < 0)) { 3856 netdev_err(priv->dev, 3857 "%s: ERROR: allocating the sfty IRQ %d (%d)\n", 3858 __func__, priv->sfty_irq, ret); 3859 irq_err = REQ_IRQ_ERR_SFTY; 3860 goto irq_error; 3861 } 3862 } 3863 3864 return 0; 3865 3866 irq_error: 3867 stmmac_free_irq(dev, irq_err, 0); 3868 return ret; 3869 } 3870 3871 static int stmmac_request_irq(struct net_device *dev) 3872 { 3873 struct stmmac_priv *priv = netdev_priv(dev); 3874 int ret; 3875 3876 /* Request the IRQ lines */ 3877 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 3878 ret = stmmac_request_irq_multi_msi(dev); 3879 else 3880 ret = stmmac_request_irq_single(dev); 3881 3882 return ret; 3883 } 3884 3885 /** 3886 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 3887 * @priv: driver private structure 3888 * @mtu: MTU to setup the dma queue and buf with 3889 * Description: Allocate and generate a dma_conf based on the provided MTU. 3890 * Allocate the Tx/Rx DMA queue and init them. 3891 * Return value: 3892 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 3893 */ 3894 static struct stmmac_dma_conf * 3895 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 3896 { 3897 struct stmmac_dma_conf *dma_conf; 3898 int chan, bfsize, ret; 3899 3900 dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); 3901 if (!dma_conf) { 3902 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 3903 __func__); 3904 return ERR_PTR(-ENOMEM); 3905 } 3906 3907 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 3908 if (bfsize < 0) 3909 bfsize = 0; 3910 3911 if (bfsize < BUF_SIZE_16KiB) 3912 bfsize = stmmac_set_bfsize(mtu, 0); 3913 3914 dma_conf->dma_buf_sz = bfsize; 3915 /* Chose the tx/rx size from the already defined one in the 3916 * priv struct. (if defined) 3917 */ 3918 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 3919 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 3920 3921 if (!dma_conf->dma_tx_size) 3922 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3923 if (!dma_conf->dma_rx_size) 3924 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3925 3926 /* Earlier check for TBS */ 3927 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3928 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 3929 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3930 3931 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3932 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3933 } 3934 3935 ret = alloc_dma_desc_resources(priv, dma_conf); 3936 if (ret < 0) { 3937 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 3938 __func__); 3939 goto alloc_error; 3940 } 3941 3942 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 3943 if (ret < 0) { 3944 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 3945 __func__); 3946 goto init_error; 3947 } 3948 3949 return dma_conf; 3950 3951 init_error: 3952 free_dma_desc_resources(priv, dma_conf); 3953 alloc_error: 3954 kfree(dma_conf); 3955 return ERR_PTR(ret); 3956 } 3957 3958 /** 3959 * __stmmac_open - open entry point of the driver 3960 * @dev : pointer to the device structure. 3961 * @dma_conf : structure to take the dma data 3962 * Description: 3963 * This function is the open entry point of the driver. 3964 * Return value: 3965 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3966 * file on failure. 3967 */ 3968 static int __stmmac_open(struct net_device *dev, 3969 struct stmmac_dma_conf *dma_conf) 3970 { 3971 struct stmmac_priv *priv = netdev_priv(dev); 3972 int mode = priv->plat->phy_interface; 3973 u32 chan; 3974 int ret; 3975 3976 /* Initialise the tx lpi timer, converting from msec to usec */ 3977 if (!priv->tx_lpi_timer) 3978 priv->tx_lpi_timer = eee_timer * 1000; 3979 3980 ret = pm_runtime_resume_and_get(priv->device); 3981 if (ret < 0) 3982 return ret; 3983 3984 if ((!priv->hw->xpcs || 3985 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) { 3986 ret = stmmac_init_phy(dev); 3987 if (ret) { 3988 netdev_err(priv->dev, 3989 "%s: Cannot attach to PHY (error: %d)\n", 3990 __func__, ret); 3991 goto init_phy_error; 3992 } 3993 } 3994 3995 buf_sz = dma_conf->dma_buf_sz; 3996 for (int i = 0; i < MTL_MAX_TX_QUEUES; i++) 3997 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 3998 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 3999 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 4000 4001 stmmac_reset_queues_param(priv); 4002 4003 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 4004 priv->plat->serdes_powerup) { 4005 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); 4006 if (ret < 0) { 4007 netdev_err(priv->dev, "%s: Serdes powerup failed\n", 4008 __func__); 4009 goto init_error; 4010 } 4011 } 4012 4013 ret = stmmac_hw_setup(dev, true); 4014 if (ret < 0) { 4015 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 4016 goto init_error; 4017 } 4018 4019 stmmac_init_coalesce(priv); 4020 4021 phylink_start(priv->phylink); 4022 /* We may have called phylink_speed_down before */ 4023 phylink_speed_up(priv->phylink); 4024 4025 ret = stmmac_request_irq(dev); 4026 if (ret) 4027 goto irq_error; 4028 4029 stmmac_enable_all_queues(priv); 4030 netif_tx_start_all_queues(priv->dev); 4031 stmmac_enable_all_dma_irq(priv); 4032 4033 return 0; 4034 4035 irq_error: 4036 phylink_stop(priv->phylink); 4037 4038 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4039 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4040 4041 stmmac_hw_teardown(dev); 4042 init_error: 4043 phylink_disconnect_phy(priv->phylink); 4044 init_phy_error: 4045 pm_runtime_put(priv->device); 4046 return ret; 4047 } 4048 4049 static int stmmac_open(struct net_device *dev) 4050 { 4051 struct stmmac_priv *priv = netdev_priv(dev); 4052 struct stmmac_dma_conf *dma_conf; 4053 int ret; 4054 4055 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 4056 if (IS_ERR(dma_conf)) 4057 return PTR_ERR(dma_conf); 4058 4059 ret = __stmmac_open(dev, dma_conf); 4060 if (ret) 4061 free_dma_desc_resources(priv, dma_conf); 4062 4063 kfree(dma_conf); 4064 return ret; 4065 } 4066 4067 /** 4068 * stmmac_release - close entry point of the driver 4069 * @dev : device pointer. 4070 * Description: 4071 * This is the stop entry point of the driver. 4072 */ 4073 static int stmmac_release(struct net_device *dev) 4074 { 4075 struct stmmac_priv *priv = netdev_priv(dev); 4076 u32 chan; 4077 4078 if (device_may_wakeup(priv->device)) 4079 phylink_speed_down(priv->phylink, false); 4080 /* Stop and disconnect the PHY */ 4081 phylink_stop(priv->phylink); 4082 phylink_disconnect_phy(priv->phylink); 4083 4084 stmmac_disable_all_queues(priv); 4085 4086 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4087 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4088 4089 netif_tx_disable(dev); 4090 4091 /* Free the IRQ lines */ 4092 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4093 4094 /* Stop TX/RX DMA and clear the descriptors */ 4095 stmmac_stop_all_dma(priv); 4096 4097 /* Release and free the Rx/Tx resources */ 4098 free_dma_desc_resources(priv, &priv->dma_conf); 4099 4100 /* Disable the MAC Rx/Tx */ 4101 stmmac_mac_set(priv, priv->ioaddr, false); 4102 4103 /* Powerdown Serdes if there is */ 4104 if (priv->plat->serdes_powerdown) 4105 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); 4106 4107 stmmac_release_ptp(priv); 4108 4109 if (stmmac_fpe_supported(priv)) 4110 timer_shutdown_sync(&priv->fpe_cfg.verify_timer); 4111 4112 pm_runtime_put(priv->device); 4113 4114 return 0; 4115 } 4116 4117 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4118 struct stmmac_tx_queue *tx_q) 4119 { 4120 u16 tag = 0x0, inner_tag = 0x0; 4121 u32 inner_type = 0x0; 4122 struct dma_desc *p; 4123 4124 if (!priv->dma_cap.vlins) 4125 return false; 4126 if (!skb_vlan_tag_present(skb)) 4127 return false; 4128 if (skb->vlan_proto == htons(ETH_P_8021AD)) { 4129 inner_tag = skb_vlan_tag_get(skb); 4130 inner_type = STMMAC_VLAN_INSERT; 4131 } 4132 4133 tag = skb_vlan_tag_get(skb); 4134 4135 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4136 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4137 else 4138 p = &tx_q->dma_tx[tx_q->cur_tx]; 4139 4140 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) 4141 return false; 4142 4143 stmmac_set_tx_owner(priv, p); 4144 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4145 return true; 4146 } 4147 4148 /** 4149 * stmmac_tso_allocator - close entry point of the driver 4150 * @priv: driver private structure 4151 * @des: buffer start address 4152 * @total_len: total length to fill in descriptors 4153 * @last_segment: condition for the last descriptor 4154 * @queue: TX queue index 4155 * Description: 4156 * This function fills descriptor and request new descriptors according to 4157 * buffer length to fill 4158 */ 4159 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4160 int total_len, bool last_segment, u32 queue) 4161 { 4162 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4163 struct dma_desc *desc; 4164 u32 buff_size; 4165 int tmp_len; 4166 4167 tmp_len = total_len; 4168 4169 while (tmp_len > 0) { 4170 dma_addr_t curr_addr; 4171 4172 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4173 priv->dma_conf.dma_tx_size); 4174 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4175 4176 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4177 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4178 else 4179 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4180 4181 curr_addr = des + (total_len - tmp_len); 4182 stmmac_set_desc_addr(priv, desc, curr_addr); 4183 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4184 TSO_MAX_BUFF_SIZE : tmp_len; 4185 4186 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4187 0, 1, 4188 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4189 0, 0); 4190 4191 tmp_len -= TSO_MAX_BUFF_SIZE; 4192 } 4193 } 4194 4195 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4196 { 4197 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4198 int desc_size; 4199 4200 if (likely(priv->extend_desc)) 4201 desc_size = sizeof(struct dma_extended_desc); 4202 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4203 desc_size = sizeof(struct dma_edesc); 4204 else 4205 desc_size = sizeof(struct dma_desc); 4206 4207 /* The own bit must be the latest setting done when prepare the 4208 * descriptor and then barrier is needed to make sure that 4209 * all is coherent before granting the DMA engine. 4210 */ 4211 wmb(); 4212 4213 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 4214 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 4215 } 4216 4217 /** 4218 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4219 * @skb : the socket buffer 4220 * @dev : device pointer 4221 * Description: this is the transmit function that is called on TSO frames 4222 * (support available on GMAC4 and newer chips). 4223 * Diagram below show the ring programming in case of TSO frames: 4224 * 4225 * First Descriptor 4226 * -------- 4227 * | DES0 |---> buffer1 = L2/L3/L4 header 4228 * | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address 4229 * | | width is 32-bit, but we never use it. 4230 * | | Also can be used as the most-significant 8-bits or 16-bits of 4231 * | | buffer1 address pointer if the DMA AXI address width is 40-bit 4232 * | | or 48-bit, and we always use it. 4233 * | DES2 |---> buffer1 len 4234 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4235 * -------- 4236 * -------- 4237 * | DES0 |---> buffer1 = TCP Payload (can continue on next descr...) 4238 * | DES1 |---> same as the First Descriptor 4239 * | DES2 |---> buffer1 len 4240 * | DES3 | 4241 * -------- 4242 * | 4243 * ... 4244 * | 4245 * -------- 4246 * | DES0 |---> buffer1 = Split TCP Payload 4247 * | DES1 |---> same as the First Descriptor 4248 * | DES2 |---> buffer1 len 4249 * | DES3 | 4250 * -------- 4251 * 4252 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4253 */ 4254 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4255 { 4256 struct dma_desc *desc, *first, *mss_desc = NULL; 4257 struct stmmac_priv *priv = netdev_priv(dev); 4258 unsigned int first_entry, tx_packets; 4259 struct stmmac_txq_stats *txq_stats; 4260 struct stmmac_tx_queue *tx_q; 4261 u32 pay_len, mss, queue; 4262 int i, first_tx, nfrags; 4263 u8 proto_hdr_len, hdr; 4264 dma_addr_t des; 4265 bool set_ic; 4266 4267 /* Always insert VLAN tag to SKB payload for TSO frames. 4268 * 4269 * Never insert VLAN tag by HW, since segments splited by 4270 * TSO engine will be un-tagged by mistake. 4271 */ 4272 if (skb_vlan_tag_present(skb)) { 4273 skb = __vlan_hwaccel_push_inside(skb); 4274 if (unlikely(!skb)) { 4275 priv->xstats.tx_dropped++; 4276 return NETDEV_TX_OK; 4277 } 4278 } 4279 4280 nfrags = skb_shinfo(skb)->nr_frags; 4281 queue = skb_get_queue_mapping(skb); 4282 4283 tx_q = &priv->dma_conf.tx_queue[queue]; 4284 txq_stats = &priv->xstats.txq_stats[queue]; 4285 first_tx = tx_q->cur_tx; 4286 4287 /* Compute header lengths */ 4288 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 4289 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 4290 hdr = sizeof(struct udphdr); 4291 } else { 4292 proto_hdr_len = skb_tcp_all_headers(skb); 4293 hdr = tcp_hdrlen(skb); 4294 } 4295 4296 /* Desc availability based on threshold should be enough safe */ 4297 if (unlikely(stmmac_tx_avail(priv, queue) < 4298 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4299 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4300 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4301 queue)); 4302 /* This is a hard error, log it. */ 4303 netdev_err(priv->dev, 4304 "%s: Tx Ring full when queue awake\n", 4305 __func__); 4306 } 4307 return NETDEV_TX_BUSY; 4308 } 4309 4310 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4311 4312 mss = skb_shinfo(skb)->gso_size; 4313 4314 /* set new MSS value if needed */ 4315 if (mss != tx_q->mss) { 4316 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4317 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4318 else 4319 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4320 4321 stmmac_set_mss(priv, mss_desc, mss); 4322 tx_q->mss = mss; 4323 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4324 priv->dma_conf.dma_tx_size); 4325 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4326 } 4327 4328 if (netif_msg_tx_queued(priv)) { 4329 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4330 __func__, hdr, proto_hdr_len, pay_len, mss); 4331 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4332 skb->data_len); 4333 } 4334 4335 first_entry = tx_q->cur_tx; 4336 WARN_ON(tx_q->tx_skbuff[first_entry]); 4337 4338 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4339 desc = &tx_q->dma_entx[first_entry].basic; 4340 else 4341 desc = &tx_q->dma_tx[first_entry]; 4342 first = desc; 4343 4344 /* first descriptor: fill Headers on Buf1 */ 4345 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4346 DMA_TO_DEVICE); 4347 if (dma_mapping_error(priv->device, des)) 4348 goto dma_map_err; 4349 4350 stmmac_set_desc_addr(priv, first, des); 4351 stmmac_tso_allocator(priv, des + proto_hdr_len, pay_len, 4352 (nfrags == 0), queue); 4353 4354 /* In case two or more DMA transmit descriptors are allocated for this 4355 * non-paged SKB data, the DMA buffer address should be saved to 4356 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4357 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4358 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4359 * since the tail areas of the DMA buffer can be accessed by DMA engine 4360 * sooner or later. 4361 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4362 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4363 * this DMA buffer right after the DMA engine completely finishes the 4364 * full buffer transmission. 4365 */ 4366 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4367 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb); 4368 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false; 4369 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4370 4371 /* Prepare fragments */ 4372 for (i = 0; i < nfrags; i++) { 4373 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4374 4375 des = skb_frag_dma_map(priv->device, frag, 0, 4376 skb_frag_size(frag), 4377 DMA_TO_DEVICE); 4378 if (dma_mapping_error(priv->device, des)) 4379 goto dma_map_err; 4380 4381 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4382 (i == nfrags - 1), queue); 4383 4384 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4385 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4386 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4387 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4388 } 4389 4390 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4391 4392 /* Only the last descriptor gets to point to the skb. */ 4393 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4394 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4395 4396 /* Manage tx mitigation */ 4397 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4398 tx_q->tx_count_frames += tx_packets; 4399 4400 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4401 set_ic = true; 4402 else if (!priv->tx_coal_frames[queue]) 4403 set_ic = false; 4404 else if (tx_packets > priv->tx_coal_frames[queue]) 4405 set_ic = true; 4406 else if ((tx_q->tx_count_frames % 4407 priv->tx_coal_frames[queue]) < tx_packets) 4408 set_ic = true; 4409 else 4410 set_ic = false; 4411 4412 if (set_ic) { 4413 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4414 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4415 else 4416 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4417 4418 tx_q->tx_count_frames = 0; 4419 stmmac_set_tx_ic(priv, desc); 4420 } 4421 4422 /* We've used all descriptors we need for this skb, however, 4423 * advance cur_tx so that it references a fresh descriptor. 4424 * ndo_start_xmit will fill this descriptor the next time it's 4425 * called and stmmac_tx_clean may clean up to this descriptor. 4426 */ 4427 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4428 4429 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4430 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4431 __func__); 4432 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4433 } 4434 4435 u64_stats_update_begin(&txq_stats->q_syncp); 4436 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4437 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4438 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4439 if (set_ic) 4440 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4441 u64_stats_update_end(&txq_stats->q_syncp); 4442 4443 if (priv->sarc_type) 4444 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4445 4446 skb_tx_timestamp(skb); 4447 4448 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4449 priv->hwts_tx_en)) { 4450 /* declare that device is doing timestamping */ 4451 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4452 stmmac_enable_tx_timestamp(priv, first); 4453 } 4454 4455 /* Complete the first descriptor before granting the DMA */ 4456 stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, 4457 tx_q->tx_skbuff_dma[first_entry].last_segment, 4458 hdr / 4, (skb->len - proto_hdr_len)); 4459 4460 /* If context desc is used to change MSS */ 4461 if (mss_desc) { 4462 /* Make sure that first descriptor has been completely 4463 * written, including its own bit. This is because MSS is 4464 * actually before first descriptor, so we need to make 4465 * sure that MSS's own bit is the last thing written. 4466 */ 4467 dma_wmb(); 4468 stmmac_set_tx_owner(priv, mss_desc); 4469 } 4470 4471 if (netif_msg_pktdata(priv)) { 4472 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4473 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4474 tx_q->cur_tx, first, nfrags); 4475 pr_info(">>> frame to be transmitted: "); 4476 print_pkt(skb->data, skb_headlen(skb)); 4477 } 4478 4479 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4480 4481 stmmac_flush_tx_descriptors(priv, queue); 4482 stmmac_tx_timer_arm(priv, queue); 4483 4484 return NETDEV_TX_OK; 4485 4486 dma_map_err: 4487 dev_err(priv->device, "Tx dma map failed\n"); 4488 dev_kfree_skb(skb); 4489 priv->xstats.tx_dropped++; 4490 return NETDEV_TX_OK; 4491 } 4492 4493 /** 4494 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4495 * @skb: socket buffer to check 4496 * 4497 * Check if a packet has an ethertype that will trigger the IP header checks 4498 * and IP/TCP checksum engine of the stmmac core. 4499 * 4500 * Return: true if the ethertype can trigger the checksum engine, false 4501 * otherwise 4502 */ 4503 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4504 { 4505 int depth = 0; 4506 __be16 proto; 4507 4508 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4509 &depth); 4510 4511 return (depth <= ETH_HLEN) && 4512 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4513 } 4514 4515 /** 4516 * stmmac_xmit - Tx entry point of the driver 4517 * @skb : the socket buffer 4518 * @dev : device pointer 4519 * Description : this is the tx entry point of the driver. 4520 * It programs the chain or the ring and supports oversized frames 4521 * and SG feature. 4522 */ 4523 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4524 { 4525 unsigned int first_entry, tx_packets, enh_desc; 4526 struct stmmac_priv *priv = netdev_priv(dev); 4527 unsigned int nopaged_len = skb_headlen(skb); 4528 int i, csum_insertion = 0, is_jumbo = 0; 4529 u32 queue = skb_get_queue_mapping(skb); 4530 int nfrags = skb_shinfo(skb)->nr_frags; 4531 int gso = skb_shinfo(skb)->gso_type; 4532 struct stmmac_txq_stats *txq_stats; 4533 struct dma_edesc *tbs_desc = NULL; 4534 struct dma_desc *desc, *first; 4535 struct stmmac_tx_queue *tx_q; 4536 bool has_vlan, set_ic; 4537 int entry, first_tx; 4538 dma_addr_t des; 4539 4540 tx_q = &priv->dma_conf.tx_queue[queue]; 4541 txq_stats = &priv->xstats.txq_stats[queue]; 4542 first_tx = tx_q->cur_tx; 4543 4544 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4545 stmmac_stop_sw_lpi(priv); 4546 4547 /* Manage oversized TCP frames for GMAC4 device */ 4548 if (skb_is_gso(skb) && priv->tso) { 4549 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4550 return stmmac_tso_xmit(skb, dev); 4551 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) 4552 return stmmac_tso_xmit(skb, dev); 4553 } 4554 4555 if (priv->est && priv->est->enable && 4556 priv->est->max_sdu[queue] && 4557 skb->len > priv->est->max_sdu[queue]){ 4558 priv->xstats.max_sdu_txq_drop[queue]++; 4559 goto max_sdu_err; 4560 } 4561 4562 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4563 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4564 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4565 queue)); 4566 /* This is a hard error, log it. */ 4567 netdev_err(priv->dev, 4568 "%s: Tx Ring full when queue awake\n", 4569 __func__); 4570 } 4571 return NETDEV_TX_BUSY; 4572 } 4573 4574 /* Check if VLAN can be inserted by HW */ 4575 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4576 4577 entry = tx_q->cur_tx; 4578 first_entry = entry; 4579 WARN_ON(tx_q->tx_skbuff[first_entry]); 4580 4581 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4582 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 4583 * queues. In that case, checksum offloading for those queues that don't 4584 * support tx coe needs to fallback to software checksum calculation. 4585 * 4586 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 4587 * also have to be checksummed in software. 4588 */ 4589 if (csum_insertion && 4590 (priv->plat->tx_queues_cfg[queue].coe_unsupported || 4591 !stmmac_has_ip_ethertype(skb))) { 4592 if (unlikely(skb_checksum_help(skb))) 4593 goto dma_map_err; 4594 csum_insertion = !csum_insertion; 4595 } 4596 4597 if (likely(priv->extend_desc)) 4598 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4599 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4600 desc = &tx_q->dma_entx[entry].basic; 4601 else 4602 desc = tx_q->dma_tx + entry; 4603 4604 first = desc; 4605 4606 if (has_vlan) 4607 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4608 4609 enh_desc = priv->plat->enh_desc; 4610 /* To program the descriptors according to the size of the frame */ 4611 if (enh_desc) 4612 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4613 4614 if (unlikely(is_jumbo)) { 4615 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4616 if (unlikely(entry < 0) && (entry != -EINVAL)) 4617 goto dma_map_err; 4618 } 4619 4620 for (i = 0; i < nfrags; i++) { 4621 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4622 int len = skb_frag_size(frag); 4623 bool last_segment = (i == (nfrags - 1)); 4624 4625 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4626 WARN_ON(tx_q->tx_skbuff[entry]); 4627 4628 if (likely(priv->extend_desc)) 4629 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4630 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4631 desc = &tx_q->dma_entx[entry].basic; 4632 else 4633 desc = tx_q->dma_tx + entry; 4634 4635 des = skb_frag_dma_map(priv->device, frag, 0, len, 4636 DMA_TO_DEVICE); 4637 if (dma_mapping_error(priv->device, des)) 4638 goto dma_map_err; /* should reuse desc w/o issues */ 4639 4640 tx_q->tx_skbuff_dma[entry].buf = des; 4641 4642 stmmac_set_desc_addr(priv, desc, des); 4643 4644 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4645 tx_q->tx_skbuff_dma[entry].len = len; 4646 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4647 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4648 4649 /* Prepare the descriptor and set the own bit too */ 4650 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4651 priv->mode, 1, last_segment, skb->len); 4652 } 4653 4654 /* Only the last descriptor gets to point to the skb. */ 4655 tx_q->tx_skbuff[entry] = skb; 4656 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4657 4658 /* According to the coalesce parameter the IC bit for the latest 4659 * segment is reset and the timer re-started to clean the tx status. 4660 * This approach takes care about the fragments: desc is the first 4661 * element in case of no SG. 4662 */ 4663 tx_packets = (entry + 1) - first_tx; 4664 tx_q->tx_count_frames += tx_packets; 4665 4666 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4667 set_ic = true; 4668 else if (!priv->tx_coal_frames[queue]) 4669 set_ic = false; 4670 else if (tx_packets > priv->tx_coal_frames[queue]) 4671 set_ic = true; 4672 else if ((tx_q->tx_count_frames % 4673 priv->tx_coal_frames[queue]) < tx_packets) 4674 set_ic = true; 4675 else 4676 set_ic = false; 4677 4678 if (set_ic) { 4679 if (likely(priv->extend_desc)) 4680 desc = &tx_q->dma_etx[entry].basic; 4681 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4682 desc = &tx_q->dma_entx[entry].basic; 4683 else 4684 desc = &tx_q->dma_tx[entry]; 4685 4686 tx_q->tx_count_frames = 0; 4687 stmmac_set_tx_ic(priv, desc); 4688 } 4689 4690 /* We've used all descriptors we need for this skb, however, 4691 * advance cur_tx so that it references a fresh descriptor. 4692 * ndo_start_xmit will fill this descriptor the next time it's 4693 * called and stmmac_tx_clean may clean up to this descriptor. 4694 */ 4695 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4696 tx_q->cur_tx = entry; 4697 4698 if (netif_msg_pktdata(priv)) { 4699 netdev_dbg(priv->dev, 4700 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4701 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4702 entry, first, nfrags); 4703 4704 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4705 print_pkt(skb->data, skb->len); 4706 } 4707 4708 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4709 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4710 __func__); 4711 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4712 } 4713 4714 u64_stats_update_begin(&txq_stats->q_syncp); 4715 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4716 if (set_ic) 4717 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4718 u64_stats_update_end(&txq_stats->q_syncp); 4719 4720 if (priv->sarc_type) 4721 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4722 4723 skb_tx_timestamp(skb); 4724 4725 /* Ready to fill the first descriptor and set the OWN bit w/o any 4726 * problems because all the descriptors are actually ready to be 4727 * passed to the DMA engine. 4728 */ 4729 if (likely(!is_jumbo)) { 4730 bool last_segment = (nfrags == 0); 4731 4732 des = dma_map_single(priv->device, skb->data, 4733 nopaged_len, DMA_TO_DEVICE); 4734 if (dma_mapping_error(priv->device, des)) 4735 goto dma_map_err; 4736 4737 tx_q->tx_skbuff_dma[first_entry].buf = des; 4738 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4739 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4740 4741 stmmac_set_desc_addr(priv, first, des); 4742 4743 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4744 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4745 4746 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4747 priv->hwts_tx_en)) { 4748 /* declare that device is doing timestamping */ 4749 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4750 stmmac_enable_tx_timestamp(priv, first); 4751 } 4752 4753 /* Prepare the first descriptor setting the OWN bit too */ 4754 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4755 csum_insertion, priv->mode, 0, last_segment, 4756 skb->len); 4757 } 4758 4759 if (tx_q->tbs & STMMAC_TBS_EN) { 4760 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4761 4762 tbs_desc = &tx_q->dma_entx[first_entry]; 4763 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4764 } 4765 4766 stmmac_set_tx_owner(priv, first); 4767 4768 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4769 4770 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 4771 4772 stmmac_flush_tx_descriptors(priv, queue); 4773 stmmac_tx_timer_arm(priv, queue); 4774 4775 return NETDEV_TX_OK; 4776 4777 dma_map_err: 4778 netdev_err(priv->dev, "Tx DMA map failed\n"); 4779 max_sdu_err: 4780 dev_kfree_skb(skb); 4781 priv->xstats.tx_dropped++; 4782 return NETDEV_TX_OK; 4783 } 4784 4785 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4786 { 4787 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 4788 __be16 vlan_proto = veth->h_vlan_proto; 4789 u16 vlanid; 4790 4791 if ((vlan_proto == htons(ETH_P_8021Q) && 4792 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4793 (vlan_proto == htons(ETH_P_8021AD) && 4794 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4795 /* pop the vlan tag */ 4796 vlanid = ntohs(veth->h_vlan_TCI); 4797 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4798 skb_pull(skb, VLAN_HLEN); 4799 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4800 } 4801 } 4802 4803 /** 4804 * stmmac_rx_refill - refill used skb preallocated buffers 4805 * @priv: driver private structure 4806 * @queue: RX queue index 4807 * Description : this is to reallocate the skb for the reception process 4808 * that is based on zero-copy. 4809 */ 4810 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4811 { 4812 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4813 int dirty = stmmac_rx_dirty(priv, queue); 4814 unsigned int entry = rx_q->dirty_rx; 4815 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4816 4817 if (priv->dma_cap.host_dma_width <= 32) 4818 gfp |= GFP_DMA32; 4819 4820 while (dirty-- > 0) { 4821 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4822 struct dma_desc *p; 4823 bool use_rx_wd; 4824 4825 if (priv->extend_desc) 4826 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4827 else 4828 p = rx_q->dma_rx + entry; 4829 4830 if (!buf->page) { 4831 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4832 if (!buf->page) 4833 break; 4834 } 4835 4836 if (priv->sph && !buf->sec_page) { 4837 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4838 if (!buf->sec_page) 4839 break; 4840 4841 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4842 } 4843 4844 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4845 4846 stmmac_set_desc_addr(priv, p, buf->addr); 4847 if (priv->sph) 4848 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4849 else 4850 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4851 stmmac_refill_desc3(priv, rx_q, p); 4852 4853 rx_q->rx_count_frames++; 4854 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4855 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4856 rx_q->rx_count_frames = 0; 4857 4858 use_rx_wd = !priv->rx_coal_frames[queue]; 4859 use_rx_wd |= rx_q->rx_count_frames > 0; 4860 if (!priv->use_riwt) 4861 use_rx_wd = false; 4862 4863 dma_wmb(); 4864 stmmac_set_rx_owner(priv, p, use_rx_wd); 4865 4866 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 4867 } 4868 rx_q->dirty_rx = entry; 4869 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4870 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4871 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4872 } 4873 4874 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4875 struct dma_desc *p, 4876 int status, unsigned int len) 4877 { 4878 unsigned int plen = 0, hlen = 0; 4879 int coe = priv->hw->rx_csum; 4880 4881 /* Not first descriptor, buffer is always zero */ 4882 if (priv->sph && len) 4883 return 0; 4884 4885 /* First descriptor, get split header length */ 4886 stmmac_get_rx_header_len(priv, p, &hlen); 4887 if (priv->sph && hlen) { 4888 priv->xstats.rx_split_hdr_pkt_n++; 4889 return hlen; 4890 } 4891 4892 /* First descriptor, not last descriptor and not split header */ 4893 if (status & rx_not_ls) 4894 return priv->dma_conf.dma_buf_sz; 4895 4896 plen = stmmac_get_rx_frame_len(priv, p, coe); 4897 4898 /* First descriptor and last descriptor and not split header */ 4899 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 4900 } 4901 4902 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4903 struct dma_desc *p, 4904 int status, unsigned int len) 4905 { 4906 int coe = priv->hw->rx_csum; 4907 unsigned int plen = 0; 4908 4909 /* Not split header, buffer is not available */ 4910 if (!priv->sph) 4911 return 0; 4912 4913 /* Not last descriptor */ 4914 if (status & rx_not_ls) 4915 return priv->dma_conf.dma_buf_sz; 4916 4917 plen = stmmac_get_rx_frame_len(priv, p, coe); 4918 4919 /* Last descriptor */ 4920 return plen - len; 4921 } 4922 4923 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 4924 struct xdp_frame *xdpf, bool dma_map) 4925 { 4926 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 4927 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4928 unsigned int entry = tx_q->cur_tx; 4929 struct dma_desc *tx_desc; 4930 dma_addr_t dma_addr; 4931 bool set_ic; 4932 4933 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 4934 return STMMAC_XDP_CONSUMED; 4935 4936 if (priv->est && priv->est->enable && 4937 priv->est->max_sdu[queue] && 4938 xdpf->len > priv->est->max_sdu[queue]) { 4939 priv->xstats.max_sdu_txq_drop[queue]++; 4940 return STMMAC_XDP_CONSUMED; 4941 } 4942 4943 if (likely(priv->extend_desc)) 4944 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4945 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4946 tx_desc = &tx_q->dma_entx[entry].basic; 4947 else 4948 tx_desc = tx_q->dma_tx + entry; 4949 4950 if (dma_map) { 4951 dma_addr = dma_map_single(priv->device, xdpf->data, 4952 xdpf->len, DMA_TO_DEVICE); 4953 if (dma_mapping_error(priv->device, dma_addr)) 4954 return STMMAC_XDP_CONSUMED; 4955 4956 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 4957 } else { 4958 struct page *page = virt_to_page(xdpf->data); 4959 4960 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 4961 xdpf->headroom; 4962 dma_sync_single_for_device(priv->device, dma_addr, 4963 xdpf->len, DMA_BIDIRECTIONAL); 4964 4965 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 4966 } 4967 4968 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 4969 tx_q->tx_skbuff_dma[entry].map_as_page = false; 4970 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 4971 tx_q->tx_skbuff_dma[entry].last_segment = true; 4972 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 4973 4974 tx_q->xdpf[entry] = xdpf; 4975 4976 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 4977 4978 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 4979 true, priv->mode, true, true, 4980 xdpf->len); 4981 4982 tx_q->tx_count_frames++; 4983 4984 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 4985 set_ic = true; 4986 else 4987 set_ic = false; 4988 4989 if (set_ic) { 4990 tx_q->tx_count_frames = 0; 4991 stmmac_set_tx_ic(priv, tx_desc); 4992 u64_stats_update_begin(&txq_stats->q_syncp); 4993 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4994 u64_stats_update_end(&txq_stats->q_syncp); 4995 } 4996 4997 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 4998 4999 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 5000 tx_q->cur_tx = entry; 5001 5002 return STMMAC_XDP_TX; 5003 } 5004 5005 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 5006 int cpu) 5007 { 5008 int index = cpu; 5009 5010 if (unlikely(index < 0)) 5011 index = 0; 5012 5013 while (index >= priv->plat->tx_queues_to_use) 5014 index -= priv->plat->tx_queues_to_use; 5015 5016 return index; 5017 } 5018 5019 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 5020 struct xdp_buff *xdp) 5021 { 5022 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 5023 int cpu = smp_processor_id(); 5024 struct netdev_queue *nq; 5025 int queue; 5026 int res; 5027 5028 if (unlikely(!xdpf)) 5029 return STMMAC_XDP_CONSUMED; 5030 5031 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5032 nq = netdev_get_tx_queue(priv->dev, queue); 5033 5034 __netif_tx_lock(nq, cpu); 5035 /* Avoids TX time-out as we are sharing with slow path */ 5036 txq_trans_cond_update(nq); 5037 5038 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); 5039 if (res == STMMAC_XDP_TX) 5040 stmmac_flush_tx_descriptors(priv, queue); 5041 5042 __netif_tx_unlock(nq); 5043 5044 return res; 5045 } 5046 5047 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 5048 struct bpf_prog *prog, 5049 struct xdp_buff *xdp) 5050 { 5051 u32 act; 5052 int res; 5053 5054 act = bpf_prog_run_xdp(prog, xdp); 5055 switch (act) { 5056 case XDP_PASS: 5057 res = STMMAC_XDP_PASS; 5058 break; 5059 case XDP_TX: 5060 res = stmmac_xdp_xmit_back(priv, xdp); 5061 break; 5062 case XDP_REDIRECT: 5063 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 5064 res = STMMAC_XDP_CONSUMED; 5065 else 5066 res = STMMAC_XDP_REDIRECT; 5067 break; 5068 default: 5069 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5070 fallthrough; 5071 case XDP_ABORTED: 5072 trace_xdp_exception(priv->dev, prog, act); 5073 fallthrough; 5074 case XDP_DROP: 5075 res = STMMAC_XDP_CONSUMED; 5076 break; 5077 } 5078 5079 return res; 5080 } 5081 5082 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5083 struct xdp_buff *xdp) 5084 { 5085 struct bpf_prog *prog; 5086 int res; 5087 5088 prog = READ_ONCE(priv->xdp_prog); 5089 if (!prog) { 5090 res = STMMAC_XDP_PASS; 5091 goto out; 5092 } 5093 5094 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5095 out: 5096 return ERR_PTR(-res); 5097 } 5098 5099 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5100 int xdp_status) 5101 { 5102 int cpu = smp_processor_id(); 5103 int queue; 5104 5105 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5106 5107 if (xdp_status & STMMAC_XDP_TX) 5108 stmmac_tx_timer_arm(priv, queue); 5109 5110 if (xdp_status & STMMAC_XDP_REDIRECT) 5111 xdp_do_flush(); 5112 } 5113 5114 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5115 struct xdp_buff *xdp) 5116 { 5117 unsigned int metasize = xdp->data - xdp->data_meta; 5118 unsigned int datasize = xdp->data_end - xdp->data; 5119 struct sk_buff *skb; 5120 5121 skb = napi_alloc_skb(&ch->rxtx_napi, 5122 xdp->data_end - xdp->data_hard_start); 5123 if (unlikely(!skb)) 5124 return NULL; 5125 5126 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5127 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5128 if (metasize) 5129 skb_metadata_set(skb, metasize); 5130 5131 return skb; 5132 } 5133 5134 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5135 struct dma_desc *p, struct dma_desc *np, 5136 struct xdp_buff *xdp) 5137 { 5138 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5139 struct stmmac_channel *ch = &priv->channel[queue]; 5140 unsigned int len = xdp->data_end - xdp->data; 5141 enum pkt_hash_types hash_type; 5142 int coe = priv->hw->rx_csum; 5143 struct sk_buff *skb; 5144 u32 hash; 5145 5146 skb = stmmac_construct_skb_zc(ch, xdp); 5147 if (!skb) { 5148 priv->xstats.rx_dropped++; 5149 return; 5150 } 5151 5152 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5153 if (priv->hw->hw_vlan_en) 5154 /* MAC level stripping. */ 5155 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5156 else 5157 /* Driver level stripping. */ 5158 stmmac_rx_vlan(priv->dev, skb); 5159 skb->protocol = eth_type_trans(skb, priv->dev); 5160 5161 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5162 skb_checksum_none_assert(skb); 5163 else 5164 skb->ip_summed = CHECKSUM_UNNECESSARY; 5165 5166 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5167 skb_set_hash(skb, hash, hash_type); 5168 5169 skb_record_rx_queue(skb, queue); 5170 napi_gro_receive(&ch->rxtx_napi, skb); 5171 5172 u64_stats_update_begin(&rxq_stats->napi_syncp); 5173 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5174 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5175 u64_stats_update_end(&rxq_stats->napi_syncp); 5176 } 5177 5178 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5179 { 5180 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5181 unsigned int entry = rx_q->dirty_rx; 5182 struct dma_desc *rx_desc = NULL; 5183 bool ret = true; 5184 5185 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5186 5187 while (budget-- > 0 && entry != rx_q->cur_rx) { 5188 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5189 dma_addr_t dma_addr; 5190 bool use_rx_wd; 5191 5192 if (!buf->xdp) { 5193 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5194 if (!buf->xdp) { 5195 ret = false; 5196 break; 5197 } 5198 } 5199 5200 if (priv->extend_desc) 5201 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 5202 else 5203 rx_desc = rx_q->dma_rx + entry; 5204 5205 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5206 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5207 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5208 stmmac_refill_desc3(priv, rx_q, rx_desc); 5209 5210 rx_q->rx_count_frames++; 5211 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5212 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5213 rx_q->rx_count_frames = 0; 5214 5215 use_rx_wd = !priv->rx_coal_frames[queue]; 5216 use_rx_wd |= rx_q->rx_count_frames > 0; 5217 if (!priv->use_riwt) 5218 use_rx_wd = false; 5219 5220 dma_wmb(); 5221 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5222 5223 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 5224 } 5225 5226 if (rx_desc) { 5227 rx_q->dirty_rx = entry; 5228 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 5229 (rx_q->dirty_rx * sizeof(struct dma_desc)); 5230 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 5231 } 5232 5233 return ret; 5234 } 5235 5236 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5237 { 5238 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5239 * to represent incoming packet, whereas cb field in the same structure 5240 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5241 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5242 */ 5243 return (struct stmmac_xdp_buff *)xdp; 5244 } 5245 5246 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5247 { 5248 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5249 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5250 unsigned int count = 0, error = 0, len = 0; 5251 int dirty = stmmac_rx_dirty(priv, queue); 5252 unsigned int next_entry = rx_q->cur_rx; 5253 u32 rx_errors = 0, rx_dropped = 0; 5254 unsigned int desc_size; 5255 struct bpf_prog *prog; 5256 bool failure = false; 5257 int xdp_status = 0; 5258 int status = 0; 5259 5260 if (netif_msg_rx_status(priv)) { 5261 void *rx_head; 5262 5263 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5264 if (priv->extend_desc) { 5265 rx_head = (void *)rx_q->dma_erx; 5266 desc_size = sizeof(struct dma_extended_desc); 5267 } else { 5268 rx_head = (void *)rx_q->dma_rx; 5269 desc_size = sizeof(struct dma_desc); 5270 } 5271 5272 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5273 rx_q->dma_rx_phy, desc_size); 5274 } 5275 while (count < limit) { 5276 struct stmmac_rx_buffer *buf; 5277 struct stmmac_xdp_buff *ctx; 5278 unsigned int buf1_len = 0; 5279 struct dma_desc *np, *p; 5280 int entry; 5281 int res; 5282 5283 if (!count && rx_q->state_saved) { 5284 error = rx_q->state.error; 5285 len = rx_q->state.len; 5286 } else { 5287 rx_q->state_saved = false; 5288 error = 0; 5289 len = 0; 5290 } 5291 5292 if (count >= limit) 5293 break; 5294 5295 read_again: 5296 buf1_len = 0; 5297 entry = next_entry; 5298 buf = &rx_q->buf_pool[entry]; 5299 5300 if (dirty >= STMMAC_RX_FILL_BATCH) { 5301 failure = failure || 5302 !stmmac_rx_refill_zc(priv, queue, dirty); 5303 dirty = 0; 5304 } 5305 5306 if (priv->extend_desc) 5307 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5308 else 5309 p = rx_q->dma_rx + entry; 5310 5311 /* read the status of the incoming frame */ 5312 status = stmmac_rx_status(priv, &priv->xstats, p); 5313 /* check if managed by the DMA otherwise go ahead */ 5314 if (unlikely(status & dma_own)) 5315 break; 5316 5317 /* Prefetch the next RX descriptor */ 5318 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5319 priv->dma_conf.dma_rx_size); 5320 next_entry = rx_q->cur_rx; 5321 5322 if (priv->extend_desc) 5323 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5324 else 5325 np = rx_q->dma_rx + next_entry; 5326 5327 prefetch(np); 5328 5329 /* Ensure a valid XSK buffer before proceed */ 5330 if (!buf->xdp) 5331 break; 5332 5333 if (priv->extend_desc) 5334 stmmac_rx_extended_status(priv, &priv->xstats, 5335 rx_q->dma_erx + entry); 5336 if (unlikely(status == discard_frame)) { 5337 xsk_buff_free(buf->xdp); 5338 buf->xdp = NULL; 5339 dirty++; 5340 error = 1; 5341 if (!priv->hwts_rx_en) 5342 rx_errors++; 5343 } 5344 5345 if (unlikely(error && (status & rx_not_ls))) 5346 goto read_again; 5347 if (unlikely(error)) { 5348 count++; 5349 continue; 5350 } 5351 5352 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5353 if (likely(status & rx_not_ls)) { 5354 xsk_buff_free(buf->xdp); 5355 buf->xdp = NULL; 5356 dirty++; 5357 count++; 5358 goto read_again; 5359 } 5360 5361 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5362 ctx->priv = priv; 5363 ctx->desc = p; 5364 ctx->ndesc = np; 5365 5366 /* XDP ZC Frame only support primary buffers for now */ 5367 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5368 len += buf1_len; 5369 5370 /* ACS is disabled; strip manually. */ 5371 if (likely(!(status & rx_not_ls))) { 5372 buf1_len -= ETH_FCS_LEN; 5373 len -= ETH_FCS_LEN; 5374 } 5375 5376 /* RX buffer is good and fit into a XSK pool buffer */ 5377 buf->xdp->data_end = buf->xdp->data + buf1_len; 5378 xsk_buff_dma_sync_for_cpu(buf->xdp); 5379 5380 prog = READ_ONCE(priv->xdp_prog); 5381 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5382 5383 switch (res) { 5384 case STMMAC_XDP_PASS: 5385 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5386 xsk_buff_free(buf->xdp); 5387 break; 5388 case STMMAC_XDP_CONSUMED: 5389 xsk_buff_free(buf->xdp); 5390 rx_dropped++; 5391 break; 5392 case STMMAC_XDP_TX: 5393 case STMMAC_XDP_REDIRECT: 5394 xdp_status |= res; 5395 break; 5396 } 5397 5398 buf->xdp = NULL; 5399 dirty++; 5400 count++; 5401 } 5402 5403 if (status & rx_not_ls) { 5404 rx_q->state_saved = true; 5405 rx_q->state.error = error; 5406 rx_q->state.len = len; 5407 } 5408 5409 stmmac_finalize_xdp_rx(priv, xdp_status); 5410 5411 u64_stats_update_begin(&rxq_stats->napi_syncp); 5412 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5413 u64_stats_update_end(&rxq_stats->napi_syncp); 5414 5415 priv->xstats.rx_dropped += rx_dropped; 5416 priv->xstats.rx_errors += rx_errors; 5417 5418 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5419 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5420 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5421 else 5422 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5423 5424 return (int)count; 5425 } 5426 5427 return failure ? limit : (int)count; 5428 } 5429 5430 /** 5431 * stmmac_rx - manage the receive process 5432 * @priv: driver private structure 5433 * @limit: napi bugget 5434 * @queue: RX queue index. 5435 * Description : this the function called by the napi poll method. 5436 * It gets all the frames inside the ring. 5437 */ 5438 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5439 { 5440 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5441 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5442 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5443 struct stmmac_channel *ch = &priv->channel[queue]; 5444 unsigned int count = 0, error = 0, len = 0; 5445 int status = 0, coe = priv->hw->rx_csum; 5446 unsigned int next_entry = rx_q->cur_rx; 5447 enum dma_data_direction dma_dir; 5448 unsigned int desc_size; 5449 struct sk_buff *skb = NULL; 5450 struct stmmac_xdp_buff ctx; 5451 int xdp_status = 0; 5452 int buf_sz; 5453 5454 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5455 buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5456 limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); 5457 5458 if (netif_msg_rx_status(priv)) { 5459 void *rx_head; 5460 5461 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5462 if (priv->extend_desc) { 5463 rx_head = (void *)rx_q->dma_erx; 5464 desc_size = sizeof(struct dma_extended_desc); 5465 } else { 5466 rx_head = (void *)rx_q->dma_rx; 5467 desc_size = sizeof(struct dma_desc); 5468 } 5469 5470 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5471 rx_q->dma_rx_phy, desc_size); 5472 } 5473 while (count < limit) { 5474 unsigned int buf1_len = 0, buf2_len = 0; 5475 enum pkt_hash_types hash_type; 5476 struct stmmac_rx_buffer *buf; 5477 struct dma_desc *np, *p; 5478 int entry; 5479 u32 hash; 5480 5481 if (!count && rx_q->state_saved) { 5482 skb = rx_q->state.skb; 5483 error = rx_q->state.error; 5484 len = rx_q->state.len; 5485 } else { 5486 rx_q->state_saved = false; 5487 skb = NULL; 5488 error = 0; 5489 len = 0; 5490 } 5491 5492 read_again: 5493 if (count >= limit) 5494 break; 5495 5496 buf1_len = 0; 5497 buf2_len = 0; 5498 entry = next_entry; 5499 buf = &rx_q->buf_pool[entry]; 5500 5501 if (priv->extend_desc) 5502 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5503 else 5504 p = rx_q->dma_rx + entry; 5505 5506 /* read the status of the incoming frame */ 5507 status = stmmac_rx_status(priv, &priv->xstats, p); 5508 /* check if managed by the DMA otherwise go ahead */ 5509 if (unlikely(status & dma_own)) 5510 break; 5511 5512 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5513 priv->dma_conf.dma_rx_size); 5514 next_entry = rx_q->cur_rx; 5515 5516 if (priv->extend_desc) 5517 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5518 else 5519 np = rx_q->dma_rx + next_entry; 5520 5521 prefetch(np); 5522 5523 if (priv->extend_desc) 5524 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5525 if (unlikely(status == discard_frame)) { 5526 page_pool_put_page(rx_q->page_pool, buf->page, 0, true); 5527 buf->page = NULL; 5528 error = 1; 5529 if (!priv->hwts_rx_en) 5530 rx_errors++; 5531 } 5532 5533 if (unlikely(error && (status & rx_not_ls))) 5534 goto read_again; 5535 if (unlikely(error)) { 5536 dev_kfree_skb(skb); 5537 skb = NULL; 5538 count++; 5539 continue; 5540 } 5541 5542 /* Buffer is good. Go on. */ 5543 5544 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5545 len += buf1_len; 5546 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5547 len += buf2_len; 5548 5549 /* ACS is disabled; strip manually. */ 5550 if (likely(!(status & rx_not_ls))) { 5551 if (buf2_len) { 5552 buf2_len -= ETH_FCS_LEN; 5553 len -= ETH_FCS_LEN; 5554 } else if (buf1_len) { 5555 buf1_len -= ETH_FCS_LEN; 5556 len -= ETH_FCS_LEN; 5557 } 5558 } 5559 5560 if (!skb) { 5561 unsigned int pre_len, sync_len; 5562 5563 dma_sync_single_for_cpu(priv->device, buf->addr, 5564 buf1_len, dma_dir); 5565 net_prefetch(page_address(buf->page) + 5566 buf->page_offset); 5567 5568 xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq); 5569 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5570 buf->page_offset, buf1_len, true); 5571 5572 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5573 buf->page_offset; 5574 5575 ctx.priv = priv; 5576 ctx.desc = p; 5577 ctx.ndesc = np; 5578 5579 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5580 /* Due xdp_adjust_tail: DMA sync for_device 5581 * cover max len CPU touch 5582 */ 5583 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5584 buf->page_offset; 5585 sync_len = max(sync_len, pre_len); 5586 5587 /* For Not XDP_PASS verdict */ 5588 if (IS_ERR(skb)) { 5589 unsigned int xdp_res = -PTR_ERR(skb); 5590 5591 if (xdp_res & STMMAC_XDP_CONSUMED) { 5592 page_pool_put_page(rx_q->page_pool, 5593 virt_to_head_page(ctx.xdp.data), 5594 sync_len, true); 5595 buf->page = NULL; 5596 rx_dropped++; 5597 5598 /* Clear skb as it was set as 5599 * status by XDP program. 5600 */ 5601 skb = NULL; 5602 5603 if (unlikely((status & rx_not_ls))) 5604 goto read_again; 5605 5606 count++; 5607 continue; 5608 } else if (xdp_res & (STMMAC_XDP_TX | 5609 STMMAC_XDP_REDIRECT)) { 5610 xdp_status |= xdp_res; 5611 buf->page = NULL; 5612 skb = NULL; 5613 count++; 5614 continue; 5615 } 5616 } 5617 } 5618 5619 if (!skb) { 5620 unsigned int head_pad_len; 5621 5622 /* XDP program may expand or reduce tail */ 5623 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5624 5625 skb = napi_build_skb(page_address(buf->page), 5626 rx_q->napi_skb_frag_size); 5627 if (!skb) { 5628 page_pool_recycle_direct(rx_q->page_pool, 5629 buf->page); 5630 rx_dropped++; 5631 count++; 5632 goto drain_data; 5633 } 5634 5635 /* XDP program may adjust header */ 5636 head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; 5637 skb_reserve(skb, head_pad_len); 5638 skb_put(skb, buf1_len); 5639 skb_mark_for_recycle(skb); 5640 buf->page = NULL; 5641 } else if (buf1_len) { 5642 dma_sync_single_for_cpu(priv->device, buf->addr, 5643 buf1_len, dma_dir); 5644 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5645 buf->page, buf->page_offset, buf1_len, 5646 priv->dma_conf.dma_buf_sz); 5647 5648 /* Data payload appended into SKB */ 5649 skb_mark_for_recycle(skb); 5650 buf->page = NULL; 5651 } 5652 5653 if (buf2_len) { 5654 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5655 buf2_len, dma_dir); 5656 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5657 buf->sec_page, 0, buf2_len, 5658 priv->dma_conf.dma_buf_sz); 5659 5660 /* Data payload appended into SKB */ 5661 skb_mark_for_recycle(skb); 5662 buf->sec_page = NULL; 5663 } 5664 5665 drain_data: 5666 if (likely(status & rx_not_ls)) 5667 goto read_again; 5668 if (!skb) 5669 continue; 5670 5671 /* Got entire packet into SKB. Finish it. */ 5672 5673 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5674 5675 if (priv->hw->hw_vlan_en) 5676 /* MAC level stripping. */ 5677 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5678 else 5679 /* Driver level stripping. */ 5680 stmmac_rx_vlan(priv->dev, skb); 5681 5682 skb->protocol = eth_type_trans(skb, priv->dev); 5683 5684 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5685 skb_checksum_none_assert(skb); 5686 else 5687 skb->ip_summed = CHECKSUM_UNNECESSARY; 5688 5689 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5690 skb_set_hash(skb, hash, hash_type); 5691 5692 skb_record_rx_queue(skb, queue); 5693 napi_gro_receive(&ch->rx_napi, skb); 5694 skb = NULL; 5695 5696 rx_packets++; 5697 rx_bytes += len; 5698 count++; 5699 } 5700 5701 if (status & rx_not_ls || skb) { 5702 rx_q->state_saved = true; 5703 rx_q->state.skb = skb; 5704 rx_q->state.error = error; 5705 rx_q->state.len = len; 5706 } 5707 5708 stmmac_finalize_xdp_rx(priv, xdp_status); 5709 5710 stmmac_rx_refill(priv, queue); 5711 5712 u64_stats_update_begin(&rxq_stats->napi_syncp); 5713 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5714 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5715 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5716 u64_stats_update_end(&rxq_stats->napi_syncp); 5717 5718 priv->xstats.rx_dropped += rx_dropped; 5719 priv->xstats.rx_errors += rx_errors; 5720 5721 return count; 5722 } 5723 5724 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5725 { 5726 struct stmmac_channel *ch = 5727 container_of(napi, struct stmmac_channel, rx_napi); 5728 struct stmmac_priv *priv = ch->priv_data; 5729 struct stmmac_rxq_stats *rxq_stats; 5730 u32 chan = ch->index; 5731 int work_done; 5732 5733 rxq_stats = &priv->xstats.rxq_stats[chan]; 5734 u64_stats_update_begin(&rxq_stats->napi_syncp); 5735 u64_stats_inc(&rxq_stats->napi.poll); 5736 u64_stats_update_end(&rxq_stats->napi_syncp); 5737 5738 work_done = stmmac_rx(priv, budget, chan); 5739 if (work_done < budget && napi_complete_done(napi, work_done)) { 5740 unsigned long flags; 5741 5742 spin_lock_irqsave(&ch->lock, flags); 5743 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5744 spin_unlock_irqrestore(&ch->lock, flags); 5745 } 5746 5747 return work_done; 5748 } 5749 5750 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5751 { 5752 struct stmmac_channel *ch = 5753 container_of(napi, struct stmmac_channel, tx_napi); 5754 struct stmmac_priv *priv = ch->priv_data; 5755 struct stmmac_txq_stats *txq_stats; 5756 bool pending_packets = false; 5757 u32 chan = ch->index; 5758 int work_done; 5759 5760 txq_stats = &priv->xstats.txq_stats[chan]; 5761 u64_stats_update_begin(&txq_stats->napi_syncp); 5762 u64_stats_inc(&txq_stats->napi.poll); 5763 u64_stats_update_end(&txq_stats->napi_syncp); 5764 5765 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 5766 work_done = min(work_done, budget); 5767 5768 if (work_done < budget && napi_complete_done(napi, work_done)) { 5769 unsigned long flags; 5770 5771 spin_lock_irqsave(&ch->lock, flags); 5772 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5773 spin_unlock_irqrestore(&ch->lock, flags); 5774 } 5775 5776 /* TX still have packet to handle, check if we need to arm tx timer */ 5777 if (pending_packets) 5778 stmmac_tx_timer_arm(priv, chan); 5779 5780 return work_done; 5781 } 5782 5783 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5784 { 5785 struct stmmac_channel *ch = 5786 container_of(napi, struct stmmac_channel, rxtx_napi); 5787 struct stmmac_priv *priv = ch->priv_data; 5788 bool tx_pending_packets = false; 5789 int rx_done, tx_done, rxtx_done; 5790 struct stmmac_rxq_stats *rxq_stats; 5791 struct stmmac_txq_stats *txq_stats; 5792 u32 chan = ch->index; 5793 5794 rxq_stats = &priv->xstats.rxq_stats[chan]; 5795 u64_stats_update_begin(&rxq_stats->napi_syncp); 5796 u64_stats_inc(&rxq_stats->napi.poll); 5797 u64_stats_update_end(&rxq_stats->napi_syncp); 5798 5799 txq_stats = &priv->xstats.txq_stats[chan]; 5800 u64_stats_update_begin(&txq_stats->napi_syncp); 5801 u64_stats_inc(&txq_stats->napi.poll); 5802 u64_stats_update_end(&txq_stats->napi_syncp); 5803 5804 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 5805 tx_done = min(tx_done, budget); 5806 5807 rx_done = stmmac_rx_zc(priv, budget, chan); 5808 5809 rxtx_done = max(tx_done, rx_done); 5810 5811 /* If either TX or RX work is not complete, return budget 5812 * and keep pooling 5813 */ 5814 if (rxtx_done >= budget) 5815 return budget; 5816 5817 /* all work done, exit the polling mode */ 5818 if (napi_complete_done(napi, rxtx_done)) { 5819 unsigned long flags; 5820 5821 spin_lock_irqsave(&ch->lock, flags); 5822 /* Both RX and TX work done are compelte, 5823 * so enable both RX & TX IRQs. 5824 */ 5825 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5826 spin_unlock_irqrestore(&ch->lock, flags); 5827 } 5828 5829 /* TX still have packet to handle, check if we need to arm tx timer */ 5830 if (tx_pending_packets) 5831 stmmac_tx_timer_arm(priv, chan); 5832 5833 return min(rxtx_done, budget - 1); 5834 } 5835 5836 /** 5837 * stmmac_tx_timeout 5838 * @dev : Pointer to net device structure 5839 * @txqueue: the index of the hanging transmit queue 5840 * Description: this function is called when a packet transmission fails to 5841 * complete within a reasonable time. The driver will mark the error in the 5842 * netdev structure and arrange for the device to be reset to a sane state 5843 * in order to transmit a new packet. 5844 */ 5845 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5846 { 5847 struct stmmac_priv *priv = netdev_priv(dev); 5848 5849 stmmac_global_err(priv); 5850 } 5851 5852 /** 5853 * stmmac_set_rx_mode - entry point for multicast addressing 5854 * @dev : pointer to the device structure 5855 * Description: 5856 * This function is a driver entry point which gets called by the kernel 5857 * whenever multicast addresses must be enabled/disabled. 5858 * Return value: 5859 * void. 5860 */ 5861 static void stmmac_set_rx_mode(struct net_device *dev) 5862 { 5863 struct stmmac_priv *priv = netdev_priv(dev); 5864 5865 stmmac_set_filter(priv, priv->hw, dev); 5866 } 5867 5868 /** 5869 * stmmac_change_mtu - entry point to change MTU size for the device. 5870 * @dev : device pointer. 5871 * @new_mtu : the new MTU size for the device. 5872 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5873 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5874 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5875 * Return value: 5876 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5877 * file on failure. 5878 */ 5879 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5880 { 5881 struct stmmac_priv *priv = netdev_priv(dev); 5882 int txfifosz = priv->plat->tx_fifo_size; 5883 struct stmmac_dma_conf *dma_conf; 5884 const int mtu = new_mtu; 5885 int ret; 5886 5887 if (txfifosz == 0) 5888 txfifosz = priv->dma_cap.tx_fifo_size; 5889 5890 txfifosz /= priv->plat->tx_queues_to_use; 5891 5892 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5893 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 5894 return -EINVAL; 5895 } 5896 5897 new_mtu = STMMAC_ALIGN(new_mtu); 5898 5899 /* If condition true, FIFO is too small or MTU too large */ 5900 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 5901 return -EINVAL; 5902 5903 if (netif_running(dev)) { 5904 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 5905 /* Try to allocate the new DMA conf with the new mtu */ 5906 dma_conf = stmmac_setup_dma_desc(priv, mtu); 5907 if (IS_ERR(dma_conf)) { 5908 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 5909 mtu); 5910 return PTR_ERR(dma_conf); 5911 } 5912 5913 stmmac_release(dev); 5914 5915 ret = __stmmac_open(dev, dma_conf); 5916 if (ret) { 5917 free_dma_desc_resources(priv, dma_conf); 5918 kfree(dma_conf); 5919 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 5920 return ret; 5921 } 5922 5923 kfree(dma_conf); 5924 5925 stmmac_set_rx_mode(dev); 5926 } 5927 5928 WRITE_ONCE(dev->mtu, mtu); 5929 netdev_update_features(dev); 5930 5931 return 0; 5932 } 5933 5934 static netdev_features_t stmmac_fix_features(struct net_device *dev, 5935 netdev_features_t features) 5936 { 5937 struct stmmac_priv *priv = netdev_priv(dev); 5938 5939 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 5940 features &= ~NETIF_F_RXCSUM; 5941 5942 if (!priv->plat->tx_coe) 5943 features &= ~NETIF_F_CSUM_MASK; 5944 5945 /* Some GMAC devices have a bugged Jumbo frame support that 5946 * needs to have the Tx COE disabled for oversized frames 5947 * (due to limited buffer sizes). In this case we disable 5948 * the TX csum insertion in the TDES and not use SF. 5949 */ 5950 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 5951 features &= ~NETIF_F_CSUM_MASK; 5952 5953 /* Disable tso if asked by ethtool */ 5954 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 5955 if (features & NETIF_F_TSO) 5956 priv->tso = true; 5957 else 5958 priv->tso = false; 5959 } 5960 5961 return features; 5962 } 5963 5964 static int stmmac_set_features(struct net_device *netdev, 5965 netdev_features_t features) 5966 { 5967 struct stmmac_priv *priv = netdev_priv(netdev); 5968 5969 /* Keep the COE Type in case of csum is supporting */ 5970 if (features & NETIF_F_RXCSUM) 5971 priv->hw->rx_csum = priv->plat->rx_coe; 5972 else 5973 priv->hw->rx_csum = 0; 5974 /* No check needed because rx_coe has been set before and it will be 5975 * fixed in case of issue. 5976 */ 5977 stmmac_rx_ipc(priv, priv->hw); 5978 5979 if (priv->sph_cap) { 5980 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; 5981 u32 chan; 5982 5983 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 5984 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 5985 } 5986 5987 if (features & NETIF_F_HW_VLAN_CTAG_RX) 5988 priv->hw->hw_vlan_en = true; 5989 else 5990 priv->hw->hw_vlan_en = false; 5991 5992 stmmac_set_hw_vlan_mode(priv, priv->hw); 5993 5994 return 0; 5995 } 5996 5997 static void stmmac_common_interrupt(struct stmmac_priv *priv) 5998 { 5999 u32 rx_cnt = priv->plat->rx_queues_to_use; 6000 u32 tx_cnt = priv->plat->tx_queues_to_use; 6001 u32 queues_count; 6002 u32 queue; 6003 bool xmac; 6004 6005 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 6006 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 6007 6008 if (priv->irq_wake) 6009 pm_wakeup_event(priv->device, 0); 6010 6011 if (priv->dma_cap.estsel) 6012 stmmac_est_irq_status(priv, priv, priv->dev, 6013 &priv->xstats, tx_cnt); 6014 6015 if (stmmac_fpe_supported(priv)) 6016 stmmac_fpe_irq_status(priv); 6017 6018 /* To handle GMAC own interrupts */ 6019 if ((priv->plat->has_gmac) || xmac) { 6020 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 6021 6022 if (unlikely(status)) { 6023 /* For LPI we need to save the tx status */ 6024 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 6025 priv->tx_path_in_lpi_mode = true; 6026 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 6027 priv->tx_path_in_lpi_mode = false; 6028 } 6029 6030 for (queue = 0; queue < queues_count; queue++) 6031 stmmac_host_mtl_irq_status(priv, priv->hw, queue); 6032 6033 /* PCS link status */ 6034 if (priv->hw->pcs && 6035 !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { 6036 if (priv->xstats.pcs_link) 6037 netif_carrier_on(priv->dev); 6038 else 6039 netif_carrier_off(priv->dev); 6040 } 6041 6042 stmmac_timestamp_interrupt(priv, priv); 6043 } 6044 } 6045 6046 /** 6047 * stmmac_interrupt - main ISR 6048 * @irq: interrupt number. 6049 * @dev_id: to pass the net device pointer. 6050 * Description: this is the main driver interrupt service routine. 6051 * It can call: 6052 * o DMA service routine (to manage incoming frame reception and transmission 6053 * status) 6054 * o Core interrupts to manage: remote wake-up, management counter, LPI 6055 * interrupts. 6056 */ 6057 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6058 { 6059 struct net_device *dev = (struct net_device *)dev_id; 6060 struct stmmac_priv *priv = netdev_priv(dev); 6061 6062 /* Check if adapter is up */ 6063 if (test_bit(STMMAC_DOWN, &priv->state)) 6064 return IRQ_HANDLED; 6065 6066 /* Check ASP error if it isn't delivered via an individual IRQ */ 6067 if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) 6068 return IRQ_HANDLED; 6069 6070 /* To handle Common interrupts */ 6071 stmmac_common_interrupt(priv); 6072 6073 /* To handle DMA interrupts */ 6074 stmmac_dma_interrupt(priv); 6075 6076 return IRQ_HANDLED; 6077 } 6078 6079 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6080 { 6081 struct net_device *dev = (struct net_device *)dev_id; 6082 struct stmmac_priv *priv = netdev_priv(dev); 6083 6084 /* Check if adapter is up */ 6085 if (test_bit(STMMAC_DOWN, &priv->state)) 6086 return IRQ_HANDLED; 6087 6088 /* To handle Common interrupts */ 6089 stmmac_common_interrupt(priv); 6090 6091 return IRQ_HANDLED; 6092 } 6093 6094 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6095 { 6096 struct net_device *dev = (struct net_device *)dev_id; 6097 struct stmmac_priv *priv = netdev_priv(dev); 6098 6099 /* Check if adapter is up */ 6100 if (test_bit(STMMAC_DOWN, &priv->state)) 6101 return IRQ_HANDLED; 6102 6103 /* Check if a fatal error happened */ 6104 stmmac_safety_feat_interrupt(priv); 6105 6106 return IRQ_HANDLED; 6107 } 6108 6109 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6110 { 6111 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6112 struct stmmac_dma_conf *dma_conf; 6113 int chan = tx_q->queue_index; 6114 struct stmmac_priv *priv; 6115 int status; 6116 6117 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6118 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6119 6120 /* Check if adapter is up */ 6121 if (test_bit(STMMAC_DOWN, &priv->state)) 6122 return IRQ_HANDLED; 6123 6124 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6125 6126 if (unlikely(status & tx_hard_error_bump_tc)) { 6127 /* Try to bump up the dma threshold on this failure */ 6128 stmmac_bump_dma_threshold(priv, chan); 6129 } else if (unlikely(status == tx_hard_error)) { 6130 stmmac_tx_err(priv, chan); 6131 } 6132 6133 return IRQ_HANDLED; 6134 } 6135 6136 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6137 { 6138 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6139 struct stmmac_dma_conf *dma_conf; 6140 int chan = rx_q->queue_index; 6141 struct stmmac_priv *priv; 6142 6143 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6144 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6145 6146 /* Check if adapter is up */ 6147 if (test_bit(STMMAC_DOWN, &priv->state)) 6148 return IRQ_HANDLED; 6149 6150 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6151 6152 return IRQ_HANDLED; 6153 } 6154 6155 /** 6156 * stmmac_ioctl - Entry point for the Ioctl 6157 * @dev: Device pointer. 6158 * @rq: An IOCTL specefic structure, that can contain a pointer to 6159 * a proprietary structure used to pass information to the driver. 6160 * @cmd: IOCTL command 6161 * Description: 6162 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6163 */ 6164 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6165 { 6166 struct stmmac_priv *priv = netdev_priv (dev); 6167 int ret = -EOPNOTSUPP; 6168 6169 if (!netif_running(dev)) 6170 return -EINVAL; 6171 6172 switch (cmd) { 6173 case SIOCGMIIPHY: 6174 case SIOCGMIIREG: 6175 case SIOCSMIIREG: 6176 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6177 break; 6178 case SIOCSHWTSTAMP: 6179 ret = stmmac_hwtstamp_set(dev, rq); 6180 break; 6181 case SIOCGHWTSTAMP: 6182 ret = stmmac_hwtstamp_get(dev, rq); 6183 break; 6184 default: 6185 break; 6186 } 6187 6188 return ret; 6189 } 6190 6191 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6192 void *cb_priv) 6193 { 6194 struct stmmac_priv *priv = cb_priv; 6195 int ret = -EOPNOTSUPP; 6196 6197 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6198 return ret; 6199 6200 __stmmac_disable_all_queues(priv); 6201 6202 switch (type) { 6203 case TC_SETUP_CLSU32: 6204 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6205 break; 6206 case TC_SETUP_CLSFLOWER: 6207 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6208 break; 6209 default: 6210 break; 6211 } 6212 6213 stmmac_enable_all_queues(priv); 6214 return ret; 6215 } 6216 6217 static LIST_HEAD(stmmac_block_cb_list); 6218 6219 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6220 void *type_data) 6221 { 6222 struct stmmac_priv *priv = netdev_priv(ndev); 6223 6224 switch (type) { 6225 case TC_QUERY_CAPS: 6226 return stmmac_tc_query_caps(priv, priv, type_data); 6227 case TC_SETUP_QDISC_MQPRIO: 6228 return stmmac_tc_setup_mqprio(priv, priv, type_data); 6229 case TC_SETUP_BLOCK: 6230 return flow_block_cb_setup_simple(type_data, 6231 &stmmac_block_cb_list, 6232 stmmac_setup_tc_block_cb, 6233 priv, priv, true); 6234 case TC_SETUP_QDISC_CBS: 6235 return stmmac_tc_setup_cbs(priv, priv, type_data); 6236 case TC_SETUP_QDISC_TAPRIO: 6237 return stmmac_tc_setup_taprio(priv, priv, type_data); 6238 case TC_SETUP_QDISC_ETF: 6239 return stmmac_tc_setup_etf(priv, priv, type_data); 6240 default: 6241 return -EOPNOTSUPP; 6242 } 6243 } 6244 6245 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6246 struct net_device *sb_dev) 6247 { 6248 int gso = skb_shinfo(skb)->gso_type; 6249 6250 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6251 /* 6252 * There is no way to determine the number of TSO/USO 6253 * capable Queues. Let's use always the Queue 0 6254 * because if TSO/USO is supported then at least this 6255 * one will be capable. 6256 */ 6257 return 0; 6258 } 6259 6260 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6261 } 6262 6263 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6264 { 6265 struct stmmac_priv *priv = netdev_priv(ndev); 6266 int ret = 0; 6267 6268 ret = pm_runtime_resume_and_get(priv->device); 6269 if (ret < 0) 6270 return ret; 6271 6272 ret = eth_mac_addr(ndev, addr); 6273 if (ret) 6274 goto set_mac_error; 6275 6276 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6277 6278 set_mac_error: 6279 pm_runtime_put(priv->device); 6280 6281 return ret; 6282 } 6283 6284 #ifdef CONFIG_DEBUG_FS 6285 static struct dentry *stmmac_fs_dir; 6286 6287 static void sysfs_display_ring(void *head, int size, int extend_desc, 6288 struct seq_file *seq, dma_addr_t dma_phy_addr) 6289 { 6290 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6291 struct dma_desc *p = (struct dma_desc *)head; 6292 unsigned int desc_size; 6293 dma_addr_t dma_addr; 6294 int i; 6295 6296 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6297 for (i = 0; i < size; i++) { 6298 dma_addr = dma_phy_addr + i * desc_size; 6299 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6300 i, &dma_addr, 6301 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6302 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6303 if (extend_desc) 6304 p = &(++ep)->basic; 6305 else 6306 p++; 6307 } 6308 } 6309 6310 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6311 { 6312 struct net_device *dev = seq->private; 6313 struct stmmac_priv *priv = netdev_priv(dev); 6314 u32 rx_count = priv->plat->rx_queues_to_use; 6315 u32 tx_count = priv->plat->tx_queues_to_use; 6316 u32 queue; 6317 6318 if ((dev->flags & IFF_UP) == 0) 6319 return 0; 6320 6321 for (queue = 0; queue < rx_count; queue++) { 6322 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6323 6324 seq_printf(seq, "RX Queue %d:\n", queue); 6325 6326 if (priv->extend_desc) { 6327 seq_printf(seq, "Extended descriptor ring:\n"); 6328 sysfs_display_ring((void *)rx_q->dma_erx, 6329 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6330 } else { 6331 seq_printf(seq, "Descriptor ring:\n"); 6332 sysfs_display_ring((void *)rx_q->dma_rx, 6333 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6334 } 6335 } 6336 6337 for (queue = 0; queue < tx_count; queue++) { 6338 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6339 6340 seq_printf(seq, "TX Queue %d:\n", queue); 6341 6342 if (priv->extend_desc) { 6343 seq_printf(seq, "Extended descriptor ring:\n"); 6344 sysfs_display_ring((void *)tx_q->dma_etx, 6345 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6346 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6347 seq_printf(seq, "Descriptor ring:\n"); 6348 sysfs_display_ring((void *)tx_q->dma_tx, 6349 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6350 } 6351 } 6352 6353 return 0; 6354 } 6355 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6356 6357 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6358 { 6359 static const char * const dwxgmac_timestamp_source[] = { 6360 "None", 6361 "Internal", 6362 "External", 6363 "Both", 6364 }; 6365 static const char * const dwxgmac_safety_feature_desc[] = { 6366 "No", 6367 "All Safety Features with ECC and Parity", 6368 "All Safety Features without ECC or Parity", 6369 "All Safety Features with Parity Only", 6370 "ECC Only", 6371 "UNDEFINED", 6372 "UNDEFINED", 6373 "UNDEFINED", 6374 }; 6375 struct net_device *dev = seq->private; 6376 struct stmmac_priv *priv = netdev_priv(dev); 6377 6378 if (!priv->hw_cap_support) { 6379 seq_printf(seq, "DMA HW features not supported\n"); 6380 return 0; 6381 } 6382 6383 seq_printf(seq, "==============================\n"); 6384 seq_printf(seq, "\tDMA HW features\n"); 6385 seq_printf(seq, "==============================\n"); 6386 6387 seq_printf(seq, "\t10/100 Mbps: %s\n", 6388 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6389 seq_printf(seq, "\t1000 Mbps: %s\n", 6390 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6391 seq_printf(seq, "\tHalf duplex: %s\n", 6392 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6393 if (priv->plat->has_xgmac) { 6394 seq_printf(seq, 6395 "\tNumber of Additional MAC address registers: %d\n", 6396 priv->dma_cap.multi_addr); 6397 } else { 6398 seq_printf(seq, "\tHash Filter: %s\n", 6399 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6400 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6401 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6402 } 6403 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6404 (priv->dma_cap.pcs) ? "Y" : "N"); 6405 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6406 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6407 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6408 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6409 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6410 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6411 seq_printf(seq, "\tRMON module: %s\n", 6412 (priv->dma_cap.rmon) ? "Y" : "N"); 6413 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6414 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6415 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6416 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6417 if (priv->plat->has_xgmac) 6418 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6419 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6420 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6421 (priv->dma_cap.eee) ? "Y" : "N"); 6422 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6423 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6424 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6425 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6426 priv->plat->has_xgmac) { 6427 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6428 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6429 } else { 6430 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6431 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6432 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6433 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6434 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6435 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6436 } 6437 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6438 priv->dma_cap.number_rx_channel); 6439 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6440 priv->dma_cap.number_tx_channel); 6441 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6442 priv->dma_cap.number_rx_queues); 6443 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6444 priv->dma_cap.number_tx_queues); 6445 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6446 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6447 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6448 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6449 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6450 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6451 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6452 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6453 priv->dma_cap.pps_out_num); 6454 seq_printf(seq, "\tSafety Features: %s\n", 6455 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6456 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6457 priv->dma_cap.frpsel ? "Y" : "N"); 6458 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6459 priv->dma_cap.host_dma_width); 6460 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6461 priv->dma_cap.rssen ? "Y" : "N"); 6462 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6463 priv->dma_cap.vlhash ? "Y" : "N"); 6464 seq_printf(seq, "\tSplit Header: %s\n", 6465 priv->dma_cap.sphen ? "Y" : "N"); 6466 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6467 priv->dma_cap.vlins ? "Y" : "N"); 6468 seq_printf(seq, "\tDouble VLAN: %s\n", 6469 priv->dma_cap.dvlan ? "Y" : "N"); 6470 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6471 priv->dma_cap.l3l4fnum); 6472 seq_printf(seq, "\tARP Offloading: %s\n", 6473 priv->dma_cap.arpoffsel ? "Y" : "N"); 6474 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6475 priv->dma_cap.estsel ? "Y" : "N"); 6476 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6477 priv->dma_cap.fpesel ? "Y" : "N"); 6478 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6479 priv->dma_cap.tbssel ? "Y" : "N"); 6480 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6481 priv->dma_cap.tbs_ch_num); 6482 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6483 priv->dma_cap.sgfsel ? "Y" : "N"); 6484 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6485 BIT(priv->dma_cap.ttsfd) >> 1); 6486 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6487 priv->dma_cap.numtc); 6488 seq_printf(seq, "\tDCB Feature: %s\n", 6489 priv->dma_cap.dcben ? "Y" : "N"); 6490 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6491 priv->dma_cap.advthword ? "Y" : "N"); 6492 seq_printf(seq, "\tPTP Offload: %s\n", 6493 priv->dma_cap.ptoen ? "Y" : "N"); 6494 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6495 priv->dma_cap.osten ? "Y" : "N"); 6496 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6497 priv->dma_cap.pfcen ? "Y" : "N"); 6498 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6499 BIT(priv->dma_cap.frpes) << 6); 6500 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6501 BIT(priv->dma_cap.frpbs) << 6); 6502 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6503 priv->dma_cap.frppipe_num); 6504 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6505 priv->dma_cap.nrvf_num ? 6506 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6507 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6508 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6509 seq_printf(seq, "\tDepth of GCL: %lu\n", 6510 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6511 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6512 priv->dma_cap.cbtisel ? "Y" : "N"); 6513 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6514 priv->dma_cap.aux_snapshot_n); 6515 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6516 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6517 seq_printf(seq, "\tEnhanced DMA: %s\n", 6518 priv->dma_cap.edma ? "Y" : "N"); 6519 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6520 priv->dma_cap.ediffc ? "Y" : "N"); 6521 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6522 priv->dma_cap.vxn ? "Y" : "N"); 6523 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6524 priv->dma_cap.dbgmem ? "Y" : "N"); 6525 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6526 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6527 return 0; 6528 } 6529 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6530 6531 /* Use network device events to rename debugfs file entries. 6532 */ 6533 static int stmmac_device_event(struct notifier_block *unused, 6534 unsigned long event, void *ptr) 6535 { 6536 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6537 struct stmmac_priv *priv = netdev_priv(dev); 6538 6539 if (dev->netdev_ops != &stmmac_netdev_ops) 6540 goto done; 6541 6542 switch (event) { 6543 case NETDEV_CHANGENAME: 6544 if (priv->dbgfs_dir) 6545 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, 6546 priv->dbgfs_dir, 6547 stmmac_fs_dir, 6548 dev->name); 6549 break; 6550 } 6551 done: 6552 return NOTIFY_DONE; 6553 } 6554 6555 static struct notifier_block stmmac_notifier = { 6556 .notifier_call = stmmac_device_event, 6557 }; 6558 6559 static void stmmac_init_fs(struct net_device *dev) 6560 { 6561 struct stmmac_priv *priv = netdev_priv(dev); 6562 6563 rtnl_lock(); 6564 6565 /* Create per netdev entries */ 6566 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6567 6568 /* Entry to report DMA RX/TX rings */ 6569 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6570 &stmmac_rings_status_fops); 6571 6572 /* Entry to report the DMA HW features */ 6573 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6574 &stmmac_dma_cap_fops); 6575 6576 rtnl_unlock(); 6577 } 6578 6579 static void stmmac_exit_fs(struct net_device *dev) 6580 { 6581 struct stmmac_priv *priv = netdev_priv(dev); 6582 6583 debugfs_remove_recursive(priv->dbgfs_dir); 6584 } 6585 #endif /* CONFIG_DEBUG_FS */ 6586 6587 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6588 { 6589 unsigned char *data = (unsigned char *)&vid_le; 6590 unsigned char data_byte = 0; 6591 u32 crc = ~0x0; 6592 u32 temp = 0; 6593 int i, bits; 6594 6595 bits = get_bitmask_order(VLAN_VID_MASK); 6596 for (i = 0; i < bits; i++) { 6597 if ((i % 8) == 0) 6598 data_byte = data[i / 8]; 6599 6600 temp = ((crc & 1) ^ data_byte) & 1; 6601 crc >>= 1; 6602 data_byte >>= 1; 6603 6604 if (temp) 6605 crc ^= 0xedb88320; 6606 } 6607 6608 return crc; 6609 } 6610 6611 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6612 { 6613 u32 crc, hash = 0; 6614 u16 pmatch = 0; 6615 int count = 0; 6616 u16 vid = 0; 6617 6618 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6619 __le16 vid_le = cpu_to_le16(vid); 6620 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6621 hash |= (1 << crc); 6622 count++; 6623 } 6624 6625 if (!priv->dma_cap.vlhash) { 6626 if (count > 2) /* VID = 0 always passes filter */ 6627 return -EOPNOTSUPP; 6628 6629 pmatch = vid; 6630 hash = 0; 6631 } 6632 6633 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6634 } 6635 6636 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6637 { 6638 struct stmmac_priv *priv = netdev_priv(ndev); 6639 bool is_double = false; 6640 int ret; 6641 6642 ret = pm_runtime_resume_and_get(priv->device); 6643 if (ret < 0) 6644 return ret; 6645 6646 if (be16_to_cpu(proto) == ETH_P_8021AD) 6647 is_double = true; 6648 6649 set_bit(vid, priv->active_vlans); 6650 ret = stmmac_vlan_update(priv, is_double); 6651 if (ret) { 6652 clear_bit(vid, priv->active_vlans); 6653 goto err_pm_put; 6654 } 6655 6656 if (priv->hw->num_vlan) { 6657 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6658 if (ret) 6659 goto err_pm_put; 6660 } 6661 err_pm_put: 6662 pm_runtime_put(priv->device); 6663 6664 return ret; 6665 } 6666 6667 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6668 { 6669 struct stmmac_priv *priv = netdev_priv(ndev); 6670 bool is_double = false; 6671 int ret; 6672 6673 ret = pm_runtime_resume_and_get(priv->device); 6674 if (ret < 0) 6675 return ret; 6676 6677 if (be16_to_cpu(proto) == ETH_P_8021AD) 6678 is_double = true; 6679 6680 clear_bit(vid, priv->active_vlans); 6681 6682 if (priv->hw->num_vlan) { 6683 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6684 if (ret) 6685 goto del_vlan_error; 6686 } 6687 6688 ret = stmmac_vlan_update(priv, is_double); 6689 6690 del_vlan_error: 6691 pm_runtime_put(priv->device); 6692 6693 return ret; 6694 } 6695 6696 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6697 { 6698 struct stmmac_priv *priv = netdev_priv(dev); 6699 6700 switch (bpf->command) { 6701 case XDP_SETUP_PROG: 6702 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6703 case XDP_SETUP_XSK_POOL: 6704 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6705 bpf->xsk.queue_id); 6706 default: 6707 return -EOPNOTSUPP; 6708 } 6709 } 6710 6711 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6712 struct xdp_frame **frames, u32 flags) 6713 { 6714 struct stmmac_priv *priv = netdev_priv(dev); 6715 int cpu = smp_processor_id(); 6716 struct netdev_queue *nq; 6717 int i, nxmit = 0; 6718 int queue; 6719 6720 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6721 return -ENETDOWN; 6722 6723 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6724 return -EINVAL; 6725 6726 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6727 nq = netdev_get_tx_queue(priv->dev, queue); 6728 6729 __netif_tx_lock(nq, cpu); 6730 /* Avoids TX time-out as we are sharing with slow path */ 6731 txq_trans_cond_update(nq); 6732 6733 for (i = 0; i < num_frames; i++) { 6734 int res; 6735 6736 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6737 if (res == STMMAC_XDP_CONSUMED) 6738 break; 6739 6740 nxmit++; 6741 } 6742 6743 if (flags & XDP_XMIT_FLUSH) { 6744 stmmac_flush_tx_descriptors(priv, queue); 6745 stmmac_tx_timer_arm(priv, queue); 6746 } 6747 6748 __netif_tx_unlock(nq); 6749 6750 return nxmit; 6751 } 6752 6753 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6754 { 6755 struct stmmac_channel *ch = &priv->channel[queue]; 6756 unsigned long flags; 6757 6758 spin_lock_irqsave(&ch->lock, flags); 6759 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6760 spin_unlock_irqrestore(&ch->lock, flags); 6761 6762 stmmac_stop_rx_dma(priv, queue); 6763 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6764 } 6765 6766 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6767 { 6768 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6769 struct stmmac_channel *ch = &priv->channel[queue]; 6770 unsigned long flags; 6771 u32 buf_size; 6772 int ret; 6773 6774 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6775 if (ret) { 6776 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6777 return; 6778 } 6779 6780 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 6781 if (ret) { 6782 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6783 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6784 return; 6785 } 6786 6787 stmmac_reset_rx_queue(priv, queue); 6788 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 6789 6790 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6791 rx_q->dma_rx_phy, rx_q->queue_index); 6792 6793 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6794 sizeof(struct dma_desc)); 6795 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6796 rx_q->rx_tail_addr, rx_q->queue_index); 6797 6798 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6799 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6800 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6801 buf_size, 6802 rx_q->queue_index); 6803 } else { 6804 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6805 priv->dma_conf.dma_buf_sz, 6806 rx_q->queue_index); 6807 } 6808 6809 stmmac_start_rx_dma(priv, queue); 6810 6811 spin_lock_irqsave(&ch->lock, flags); 6812 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6813 spin_unlock_irqrestore(&ch->lock, flags); 6814 } 6815 6816 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6817 { 6818 struct stmmac_channel *ch = &priv->channel[queue]; 6819 unsigned long flags; 6820 6821 spin_lock_irqsave(&ch->lock, flags); 6822 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6823 spin_unlock_irqrestore(&ch->lock, flags); 6824 6825 stmmac_stop_tx_dma(priv, queue); 6826 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6827 } 6828 6829 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6830 { 6831 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6832 struct stmmac_channel *ch = &priv->channel[queue]; 6833 unsigned long flags; 6834 int ret; 6835 6836 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6837 if (ret) { 6838 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6839 return; 6840 } 6841 6842 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 6843 if (ret) { 6844 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6845 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6846 return; 6847 } 6848 6849 stmmac_reset_tx_queue(priv, queue); 6850 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 6851 6852 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6853 tx_q->dma_tx_phy, tx_q->queue_index); 6854 6855 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6856 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6857 6858 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6859 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6860 tx_q->tx_tail_addr, tx_q->queue_index); 6861 6862 stmmac_start_tx_dma(priv, queue); 6863 6864 spin_lock_irqsave(&ch->lock, flags); 6865 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6866 spin_unlock_irqrestore(&ch->lock, flags); 6867 } 6868 6869 void stmmac_xdp_release(struct net_device *dev) 6870 { 6871 struct stmmac_priv *priv = netdev_priv(dev); 6872 u32 chan; 6873 6874 /* Ensure tx function is not running */ 6875 netif_tx_disable(dev); 6876 6877 /* Disable NAPI process */ 6878 stmmac_disable_all_queues(priv); 6879 6880 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6881 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6882 6883 /* Free the IRQ lines */ 6884 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 6885 6886 /* Stop TX/RX DMA channels */ 6887 stmmac_stop_all_dma(priv); 6888 6889 /* Release and free the Rx/Tx resources */ 6890 free_dma_desc_resources(priv, &priv->dma_conf); 6891 6892 /* Disable the MAC Rx/Tx */ 6893 stmmac_mac_set(priv, priv->ioaddr, false); 6894 6895 /* set trans_start so we don't get spurious 6896 * watchdogs during reset 6897 */ 6898 netif_trans_update(dev); 6899 netif_carrier_off(dev); 6900 } 6901 6902 int stmmac_xdp_open(struct net_device *dev) 6903 { 6904 struct stmmac_priv *priv = netdev_priv(dev); 6905 u32 rx_cnt = priv->plat->rx_queues_to_use; 6906 u32 tx_cnt = priv->plat->tx_queues_to_use; 6907 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 6908 struct stmmac_rx_queue *rx_q; 6909 struct stmmac_tx_queue *tx_q; 6910 u32 buf_size; 6911 bool sph_en; 6912 u32 chan; 6913 int ret; 6914 6915 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 6916 if (ret < 0) { 6917 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 6918 __func__); 6919 goto dma_desc_error; 6920 } 6921 6922 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 6923 if (ret < 0) { 6924 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 6925 __func__); 6926 goto init_error; 6927 } 6928 6929 stmmac_reset_queues_param(priv); 6930 6931 /* DMA CSR Channel configuration */ 6932 for (chan = 0; chan < dma_csr_ch; chan++) { 6933 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 6934 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6935 } 6936 6937 /* Adjust Split header */ 6938 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 6939 6940 /* DMA RX Channel Configuration */ 6941 for (chan = 0; chan < rx_cnt; chan++) { 6942 rx_q = &priv->dma_conf.rx_queue[chan]; 6943 6944 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6945 rx_q->dma_rx_phy, chan); 6946 6947 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 6948 (rx_q->buf_alloc_num * 6949 sizeof(struct dma_desc)); 6950 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6951 rx_q->rx_tail_addr, chan); 6952 6953 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6954 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6955 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6956 buf_size, 6957 rx_q->queue_index); 6958 } else { 6959 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6960 priv->dma_conf.dma_buf_sz, 6961 rx_q->queue_index); 6962 } 6963 6964 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6965 } 6966 6967 /* DMA TX Channel Configuration */ 6968 for (chan = 0; chan < tx_cnt; chan++) { 6969 tx_q = &priv->dma_conf.tx_queue[chan]; 6970 6971 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6972 tx_q->dma_tx_phy, chan); 6973 6974 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6975 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6976 tx_q->tx_tail_addr, chan); 6977 6978 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6979 tx_q->txtimer.function = stmmac_tx_timer; 6980 } 6981 6982 /* Enable the MAC Rx/Tx */ 6983 stmmac_mac_set(priv, priv->ioaddr, true); 6984 6985 /* Start Rx & Tx DMA Channels */ 6986 stmmac_start_all_dma(priv); 6987 6988 ret = stmmac_request_irq(dev); 6989 if (ret) 6990 goto irq_error; 6991 6992 /* Enable NAPI process*/ 6993 stmmac_enable_all_queues(priv); 6994 netif_carrier_on(dev); 6995 netif_tx_start_all_queues(dev); 6996 stmmac_enable_all_dma_irq(priv); 6997 6998 return 0; 6999 7000 irq_error: 7001 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7002 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7003 7004 stmmac_hw_teardown(dev); 7005 init_error: 7006 free_dma_desc_resources(priv, &priv->dma_conf); 7007 dma_desc_error: 7008 return ret; 7009 } 7010 7011 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 7012 { 7013 struct stmmac_priv *priv = netdev_priv(dev); 7014 struct stmmac_rx_queue *rx_q; 7015 struct stmmac_tx_queue *tx_q; 7016 struct stmmac_channel *ch; 7017 7018 if (test_bit(STMMAC_DOWN, &priv->state) || 7019 !netif_carrier_ok(priv->dev)) 7020 return -ENETDOWN; 7021 7022 if (!stmmac_xdp_is_enabled(priv)) 7023 return -EINVAL; 7024 7025 if (queue >= priv->plat->rx_queues_to_use || 7026 queue >= priv->plat->tx_queues_to_use) 7027 return -EINVAL; 7028 7029 rx_q = &priv->dma_conf.rx_queue[queue]; 7030 tx_q = &priv->dma_conf.tx_queue[queue]; 7031 ch = &priv->channel[queue]; 7032 7033 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 7034 return -EINVAL; 7035 7036 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 7037 /* EQoS does not have per-DMA channel SW interrupt, 7038 * so we schedule RX Napi straight-away. 7039 */ 7040 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 7041 __napi_schedule(&ch->rxtx_napi); 7042 } 7043 7044 return 0; 7045 } 7046 7047 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7048 { 7049 struct stmmac_priv *priv = netdev_priv(dev); 7050 u32 tx_cnt = priv->plat->tx_queues_to_use; 7051 u32 rx_cnt = priv->plat->rx_queues_to_use; 7052 unsigned int start; 7053 int q; 7054 7055 for (q = 0; q < tx_cnt; q++) { 7056 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7057 u64 tx_packets; 7058 u64 tx_bytes; 7059 7060 do { 7061 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 7062 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 7063 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 7064 do { 7065 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 7066 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 7067 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 7068 7069 stats->tx_packets += tx_packets; 7070 stats->tx_bytes += tx_bytes; 7071 } 7072 7073 for (q = 0; q < rx_cnt; q++) { 7074 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7075 u64 rx_packets; 7076 u64 rx_bytes; 7077 7078 do { 7079 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 7080 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 7081 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 7082 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 7083 7084 stats->rx_packets += rx_packets; 7085 stats->rx_bytes += rx_bytes; 7086 } 7087 7088 stats->rx_dropped = priv->xstats.rx_dropped; 7089 stats->rx_errors = priv->xstats.rx_errors; 7090 stats->tx_dropped = priv->xstats.tx_dropped; 7091 stats->tx_errors = priv->xstats.tx_errors; 7092 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7093 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7094 stats->rx_length_errors = priv->xstats.rx_length; 7095 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7096 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7097 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7098 } 7099 7100 static const struct net_device_ops stmmac_netdev_ops = { 7101 .ndo_open = stmmac_open, 7102 .ndo_start_xmit = stmmac_xmit, 7103 .ndo_stop = stmmac_release, 7104 .ndo_change_mtu = stmmac_change_mtu, 7105 .ndo_fix_features = stmmac_fix_features, 7106 .ndo_set_features = stmmac_set_features, 7107 .ndo_set_rx_mode = stmmac_set_rx_mode, 7108 .ndo_tx_timeout = stmmac_tx_timeout, 7109 .ndo_eth_ioctl = stmmac_ioctl, 7110 .ndo_get_stats64 = stmmac_get_stats64, 7111 .ndo_setup_tc = stmmac_setup_tc, 7112 .ndo_select_queue = stmmac_select_queue, 7113 .ndo_set_mac_address = stmmac_set_mac_address, 7114 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7115 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7116 .ndo_bpf = stmmac_bpf, 7117 .ndo_xdp_xmit = stmmac_xdp_xmit, 7118 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7119 }; 7120 7121 static void stmmac_reset_subtask(struct stmmac_priv *priv) 7122 { 7123 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7124 return; 7125 if (test_bit(STMMAC_DOWN, &priv->state)) 7126 return; 7127 7128 netdev_err(priv->dev, "Reset adapter.\n"); 7129 7130 rtnl_lock(); 7131 netif_trans_update(priv->dev); 7132 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7133 usleep_range(1000, 2000); 7134 7135 set_bit(STMMAC_DOWN, &priv->state); 7136 dev_close(priv->dev); 7137 dev_open(priv->dev, NULL); 7138 clear_bit(STMMAC_DOWN, &priv->state); 7139 clear_bit(STMMAC_RESETING, &priv->state); 7140 rtnl_unlock(); 7141 } 7142 7143 static void stmmac_service_task(struct work_struct *work) 7144 { 7145 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7146 service_task); 7147 7148 stmmac_reset_subtask(priv); 7149 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7150 } 7151 7152 /** 7153 * stmmac_hw_init - Init the MAC device 7154 * @priv: driver private structure 7155 * Description: this function is to configure the MAC device according to 7156 * some platform parameters or the HW capability register. It prepares the 7157 * driver to use either ring or chain modes and to setup either enhanced or 7158 * normal descriptors. 7159 */ 7160 static int stmmac_hw_init(struct stmmac_priv *priv) 7161 { 7162 int ret; 7163 7164 /* dwmac-sun8i only work in chain mode */ 7165 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7166 chain_mode = 1; 7167 priv->chain_mode = chain_mode; 7168 7169 /* Initialize HW Interface */ 7170 ret = stmmac_hwif_init(priv); 7171 if (ret) 7172 return ret; 7173 7174 /* Get the HW capability (new GMAC newer than 3.50a) */ 7175 priv->hw_cap_support = stmmac_get_hw_features(priv); 7176 if (priv->hw_cap_support) { 7177 dev_info(priv->device, "DMA HW capability register supported\n"); 7178 7179 /* We can override some gmac/dma configuration fields: e.g. 7180 * enh_desc, tx_coe (e.g. that are passed through the 7181 * platform) with the values from the HW capability 7182 * register (if supported). 7183 */ 7184 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7185 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7186 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7187 priv->hw->pmt = priv->plat->pmt; 7188 if (priv->dma_cap.hash_tb_sz) { 7189 priv->hw->multicast_filter_bins = 7190 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7191 priv->hw->mcast_bits_log2 = 7192 ilog2(priv->hw->multicast_filter_bins); 7193 } 7194 7195 /* TXCOE doesn't work in thresh DMA mode */ 7196 if (priv->plat->force_thresh_dma_mode) 7197 priv->plat->tx_coe = 0; 7198 else 7199 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7200 7201 /* In case of GMAC4 rx_coe is from HW cap register. */ 7202 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7203 7204 if (priv->dma_cap.rx_coe_type2) 7205 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7206 else if (priv->dma_cap.rx_coe_type1) 7207 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7208 7209 } else { 7210 dev_info(priv->device, "No HW DMA feature register supported\n"); 7211 } 7212 7213 if (priv->plat->rx_coe) { 7214 priv->hw->rx_csum = priv->plat->rx_coe; 7215 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7216 if (priv->synopsys_id < DWMAC_CORE_4_00) 7217 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7218 } 7219 if (priv->plat->tx_coe) 7220 dev_info(priv->device, "TX Checksum insertion supported\n"); 7221 7222 if (priv->plat->pmt) { 7223 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7224 device_set_wakeup_capable(priv->device, 1); 7225 } 7226 7227 if (priv->dma_cap.tsoen) 7228 dev_info(priv->device, "TSO supported\n"); 7229 7230 priv->hw->vlan_fail_q_en = 7231 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7232 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7233 7234 /* Run HW quirks, if any */ 7235 if (priv->hwif_quirks) { 7236 ret = priv->hwif_quirks(priv); 7237 if (ret) 7238 return ret; 7239 } 7240 7241 /* Rx Watchdog is available in the COREs newer than the 3.40. 7242 * In some case, for example on bugged HW this feature 7243 * has to be disable and this can be done by passing the 7244 * riwt_off field from the platform. 7245 */ 7246 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 7247 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 7248 priv->use_riwt = 1; 7249 dev_info(priv->device, 7250 "Enable RX Mitigation via HW Watchdog Timer\n"); 7251 } 7252 7253 return 0; 7254 } 7255 7256 static void stmmac_napi_add(struct net_device *dev) 7257 { 7258 struct stmmac_priv *priv = netdev_priv(dev); 7259 u32 queue, maxq; 7260 7261 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7262 7263 for (queue = 0; queue < maxq; queue++) { 7264 struct stmmac_channel *ch = &priv->channel[queue]; 7265 7266 ch->priv_data = priv; 7267 ch->index = queue; 7268 spin_lock_init(&ch->lock); 7269 7270 if (queue < priv->plat->rx_queues_to_use) { 7271 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7272 } 7273 if (queue < priv->plat->tx_queues_to_use) { 7274 netif_napi_add_tx(dev, &ch->tx_napi, 7275 stmmac_napi_poll_tx); 7276 } 7277 if (queue < priv->plat->rx_queues_to_use && 7278 queue < priv->plat->tx_queues_to_use) { 7279 netif_napi_add(dev, &ch->rxtx_napi, 7280 stmmac_napi_poll_rxtx); 7281 } 7282 } 7283 } 7284 7285 static void stmmac_napi_del(struct net_device *dev) 7286 { 7287 struct stmmac_priv *priv = netdev_priv(dev); 7288 u32 queue, maxq; 7289 7290 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7291 7292 for (queue = 0; queue < maxq; queue++) { 7293 struct stmmac_channel *ch = &priv->channel[queue]; 7294 7295 if (queue < priv->plat->rx_queues_to_use) 7296 netif_napi_del(&ch->rx_napi); 7297 if (queue < priv->plat->tx_queues_to_use) 7298 netif_napi_del(&ch->tx_napi); 7299 if (queue < priv->plat->rx_queues_to_use && 7300 queue < priv->plat->tx_queues_to_use) { 7301 netif_napi_del(&ch->rxtx_napi); 7302 } 7303 } 7304 } 7305 7306 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 7307 { 7308 struct stmmac_priv *priv = netdev_priv(dev); 7309 int ret = 0, i; 7310 7311 if (netif_running(dev)) 7312 stmmac_release(dev); 7313 7314 stmmac_napi_del(dev); 7315 7316 priv->plat->rx_queues_to_use = rx_cnt; 7317 priv->plat->tx_queues_to_use = tx_cnt; 7318 if (!netif_is_rxfh_configured(dev)) 7319 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7320 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7321 rx_cnt); 7322 7323 stmmac_napi_add(dev); 7324 7325 if (netif_running(dev)) 7326 ret = stmmac_open(dev); 7327 7328 return ret; 7329 } 7330 7331 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7332 { 7333 struct stmmac_priv *priv = netdev_priv(dev); 7334 int ret = 0; 7335 7336 if (netif_running(dev)) 7337 stmmac_release(dev); 7338 7339 priv->dma_conf.dma_rx_size = rx_size; 7340 priv->dma_conf.dma_tx_size = tx_size; 7341 7342 if (netif_running(dev)) 7343 ret = stmmac_open(dev); 7344 7345 return ret; 7346 } 7347 7348 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7349 { 7350 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7351 struct dma_desc *desc_contains_ts = ctx->desc; 7352 struct stmmac_priv *priv = ctx->priv; 7353 struct dma_desc *ndesc = ctx->ndesc; 7354 struct dma_desc *desc = ctx->desc; 7355 u64 ns = 0; 7356 7357 if (!priv->hwts_rx_en) 7358 return -ENODATA; 7359 7360 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7361 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 7362 desc_contains_ts = ndesc; 7363 7364 /* Check if timestamp is available */ 7365 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7366 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7367 ns -= priv->plat->cdc_error_adj; 7368 *timestamp = ns_to_ktime(ns); 7369 return 0; 7370 } 7371 7372 return -ENODATA; 7373 } 7374 7375 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7376 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7377 }; 7378 7379 /** 7380 * stmmac_dvr_probe 7381 * @device: device pointer 7382 * @plat_dat: platform data pointer 7383 * @res: stmmac resource pointer 7384 * Description: this is the main probe function used to 7385 * call the alloc_etherdev, allocate the priv structure. 7386 * Return: 7387 * returns 0 on success, otherwise errno. 7388 */ 7389 int stmmac_dvr_probe(struct device *device, 7390 struct plat_stmmacenet_data *plat_dat, 7391 struct stmmac_resources *res) 7392 { 7393 struct net_device *ndev = NULL; 7394 struct stmmac_priv *priv; 7395 u32 rxq; 7396 int i, ret = 0; 7397 7398 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7399 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7400 if (!ndev) 7401 return -ENOMEM; 7402 7403 SET_NETDEV_DEV(ndev, device); 7404 7405 priv = netdev_priv(ndev); 7406 priv->device = device; 7407 priv->dev = ndev; 7408 7409 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7410 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7411 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7412 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7413 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7414 } 7415 7416 priv->xstats.pcpu_stats = 7417 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7418 if (!priv->xstats.pcpu_stats) 7419 return -ENOMEM; 7420 7421 stmmac_set_ethtool_ops(ndev); 7422 priv->pause = pause; 7423 priv->plat = plat_dat; 7424 priv->ioaddr = res->addr; 7425 priv->dev->base_addr = (unsigned long)res->addr; 7426 priv->plat->dma_cfg->multi_msi_en = 7427 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7428 7429 priv->dev->irq = res->irq; 7430 priv->wol_irq = res->wol_irq; 7431 priv->lpi_irq = res->lpi_irq; 7432 priv->sfty_irq = res->sfty_irq; 7433 priv->sfty_ce_irq = res->sfty_ce_irq; 7434 priv->sfty_ue_irq = res->sfty_ue_irq; 7435 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7436 priv->rx_irq[i] = res->rx_irq[i]; 7437 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7438 priv->tx_irq[i] = res->tx_irq[i]; 7439 7440 if (!is_zero_ether_addr(res->mac)) 7441 eth_hw_addr_set(priv->dev, res->mac); 7442 7443 dev_set_drvdata(device, priv->dev); 7444 7445 /* Verify driver arguments */ 7446 stmmac_verify_args(); 7447 7448 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7449 if (!priv->af_xdp_zc_qps) 7450 return -ENOMEM; 7451 7452 /* Allocate workqueue */ 7453 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7454 if (!priv->wq) { 7455 dev_err(priv->device, "failed to create workqueue\n"); 7456 ret = -ENOMEM; 7457 goto error_wq_init; 7458 } 7459 7460 INIT_WORK(&priv->service_task, stmmac_service_task); 7461 7462 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 7463 7464 /* Override with kernel parameters if supplied XXX CRS XXX 7465 * this needs to have multiple instances 7466 */ 7467 if ((phyaddr >= 0) && (phyaddr <= 31)) 7468 priv->plat->phy_addr = phyaddr; 7469 7470 if (priv->plat->stmmac_rst) { 7471 ret = reset_control_assert(priv->plat->stmmac_rst); 7472 reset_control_deassert(priv->plat->stmmac_rst); 7473 /* Some reset controllers have only reset callback instead of 7474 * assert + deassert callbacks pair. 7475 */ 7476 if (ret == -ENOTSUPP) 7477 reset_control_reset(priv->plat->stmmac_rst); 7478 } 7479 7480 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7481 if (ret == -ENOTSUPP) 7482 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7483 ERR_PTR(ret)); 7484 7485 /* Wait a bit for the reset to take effect */ 7486 udelay(10); 7487 7488 /* Init MAC and get the capabilities */ 7489 ret = stmmac_hw_init(priv); 7490 if (ret) 7491 goto error_hw_init; 7492 7493 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7494 */ 7495 if (priv->synopsys_id < DWMAC_CORE_5_20) 7496 priv->plat->dma_cfg->dche = false; 7497 7498 stmmac_check_ether_addr(priv); 7499 7500 ndev->netdev_ops = &stmmac_netdev_ops; 7501 7502 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7503 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7504 7505 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7506 NETIF_F_RXCSUM; 7507 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7508 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7509 7510 ret = stmmac_tc_init(priv, priv); 7511 if (!ret) { 7512 ndev->hw_features |= NETIF_F_HW_TC; 7513 } 7514 7515 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 7516 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7517 if (priv->plat->has_gmac4) 7518 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7519 priv->tso = true; 7520 dev_info(priv->device, "TSO feature enabled\n"); 7521 } 7522 7523 if (priv->dma_cap.sphen && 7524 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7525 ndev->hw_features |= NETIF_F_GRO; 7526 priv->sph_cap = true; 7527 priv->sph = priv->sph_cap; 7528 dev_info(priv->device, "SPH feature enabled\n"); 7529 } 7530 7531 /* Ideally our host DMA address width is the same as for the 7532 * device. However, it may differ and then we have to use our 7533 * host DMA width for allocation and the device DMA width for 7534 * register handling. 7535 */ 7536 if (priv->plat->host_dma_width) 7537 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7538 else 7539 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7540 7541 if (priv->dma_cap.host_dma_width) { 7542 ret = dma_set_mask_and_coherent(device, 7543 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7544 if (!ret) { 7545 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7546 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7547 7548 /* 7549 * If more than 32 bits can be addressed, make sure to 7550 * enable enhanced addressing mode. 7551 */ 7552 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7553 priv->plat->dma_cfg->eame = true; 7554 } else { 7555 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7556 if (ret) { 7557 dev_err(priv->device, "Failed to set DMA Mask\n"); 7558 goto error_hw_init; 7559 } 7560 7561 priv->dma_cap.host_dma_width = 32; 7562 } 7563 } 7564 7565 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7566 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7567 #ifdef STMMAC_VLAN_TAG_USED 7568 /* Both mac100 and gmac support receive VLAN tag detection */ 7569 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7570 if (priv->plat->has_gmac4) { 7571 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 7572 priv->hw->hw_vlan_en = true; 7573 } 7574 if (priv->dma_cap.vlhash) { 7575 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7576 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7577 } 7578 if (priv->dma_cap.vlins) { 7579 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7580 if (priv->dma_cap.dvlan) 7581 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; 7582 } 7583 #endif 7584 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7585 7586 priv->xstats.threshold = tc; 7587 7588 /* Initialize RSS */ 7589 rxq = priv->plat->rx_queues_to_use; 7590 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7591 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7592 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7593 7594 if (priv->dma_cap.rssen && priv->plat->rss_en) 7595 ndev->features |= NETIF_F_RXHASH; 7596 7597 ndev->vlan_features |= ndev->features; 7598 7599 /* MTU range: 46 - hw-specific max */ 7600 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7601 if (priv->plat->has_xgmac) 7602 ndev->max_mtu = XGMAC_JUMBO_LEN; 7603 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 7604 ndev->max_mtu = JUMBO_LEN; 7605 else 7606 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7607 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 7608 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 7609 */ 7610 if ((priv->plat->maxmtu < ndev->max_mtu) && 7611 (priv->plat->maxmtu >= ndev->min_mtu)) 7612 ndev->max_mtu = priv->plat->maxmtu; 7613 else if (priv->plat->maxmtu < ndev->min_mtu) 7614 dev_warn(priv->device, 7615 "%s: warning: maxmtu having invalid value (%d)\n", 7616 __func__, priv->plat->maxmtu); 7617 7618 if (flow_ctrl) 7619 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 7620 7621 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 7622 7623 /* Setup channels NAPI */ 7624 stmmac_napi_add(ndev); 7625 7626 mutex_init(&priv->lock); 7627 7628 stmmac_fpe_init(priv); 7629 7630 /* If a specific clk_csr value is passed from the platform 7631 * this means that the CSR Clock Range selection cannot be 7632 * changed at run-time and it is fixed. Viceversa the driver'll try to 7633 * set the MDC clock dynamically according to the csr actual 7634 * clock input. 7635 */ 7636 if (priv->plat->clk_csr >= 0) 7637 priv->clk_csr = priv->plat->clk_csr; 7638 else 7639 stmmac_clk_csr_set(priv); 7640 7641 stmmac_check_pcs_mode(priv); 7642 7643 pm_runtime_get_noresume(device); 7644 pm_runtime_set_active(device); 7645 if (!pm_runtime_enabled(device)) 7646 pm_runtime_enable(device); 7647 7648 ret = stmmac_mdio_register(ndev); 7649 if (ret < 0) { 7650 dev_err_probe(priv->device, ret, 7651 "MDIO bus (id: %d) registration failed\n", 7652 priv->plat->bus_id); 7653 goto error_mdio_register; 7654 } 7655 7656 if (priv->plat->speed_mode_2500) 7657 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); 7658 7659 ret = stmmac_pcs_setup(ndev); 7660 if (ret) 7661 goto error_pcs_setup; 7662 7663 ret = stmmac_phy_setup(priv); 7664 if (ret) { 7665 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7666 goto error_phy_setup; 7667 } 7668 7669 ret = register_netdev(ndev); 7670 if (ret) { 7671 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7672 __func__, ret); 7673 goto error_netdev_register; 7674 } 7675 7676 #ifdef CONFIG_DEBUG_FS 7677 stmmac_init_fs(ndev); 7678 #endif 7679 7680 if (priv->plat->dump_debug_regs) 7681 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7682 7683 /* Let pm_runtime_put() disable the clocks. 7684 * If CONFIG_PM is not enabled, the clocks will stay powered. 7685 */ 7686 pm_runtime_put(device); 7687 7688 return ret; 7689 7690 error_netdev_register: 7691 phylink_destroy(priv->phylink); 7692 error_phy_setup: 7693 stmmac_pcs_clean(ndev); 7694 error_pcs_setup: 7695 stmmac_mdio_unregister(ndev); 7696 error_mdio_register: 7697 stmmac_napi_del(ndev); 7698 error_hw_init: 7699 destroy_workqueue(priv->wq); 7700 error_wq_init: 7701 bitmap_free(priv->af_xdp_zc_qps); 7702 7703 return ret; 7704 } 7705 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 7706 7707 /** 7708 * stmmac_dvr_remove 7709 * @dev: device pointer 7710 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 7711 * changes the link status, releases the DMA descriptor rings. 7712 */ 7713 void stmmac_dvr_remove(struct device *dev) 7714 { 7715 struct net_device *ndev = dev_get_drvdata(dev); 7716 struct stmmac_priv *priv = netdev_priv(ndev); 7717 7718 netdev_info(priv->dev, "%s: removing driver", __func__); 7719 7720 pm_runtime_get_sync(dev); 7721 7722 stmmac_stop_all_dma(priv); 7723 stmmac_mac_set(priv, priv->ioaddr, false); 7724 unregister_netdev(ndev); 7725 7726 #ifdef CONFIG_DEBUG_FS 7727 stmmac_exit_fs(ndev); 7728 #endif 7729 phylink_destroy(priv->phylink); 7730 if (priv->plat->stmmac_rst) 7731 reset_control_assert(priv->plat->stmmac_rst); 7732 reset_control_assert(priv->plat->stmmac_ahb_rst); 7733 7734 stmmac_pcs_clean(ndev); 7735 stmmac_mdio_unregister(ndev); 7736 7737 destroy_workqueue(priv->wq); 7738 mutex_destroy(&priv->lock); 7739 bitmap_free(priv->af_xdp_zc_qps); 7740 7741 pm_runtime_disable(dev); 7742 pm_runtime_put_noidle(dev); 7743 } 7744 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 7745 7746 /** 7747 * stmmac_suspend - suspend callback 7748 * @dev: device pointer 7749 * Description: this is the function to suspend the device and it is called 7750 * by the platform driver to stop the network queue, release the resources, 7751 * program the PMT register (for WoL), clean and release driver resources. 7752 */ 7753 int stmmac_suspend(struct device *dev) 7754 { 7755 struct net_device *ndev = dev_get_drvdata(dev); 7756 struct stmmac_priv *priv = netdev_priv(ndev); 7757 u32 chan; 7758 7759 if (!ndev || !netif_running(ndev)) 7760 return 0; 7761 7762 mutex_lock(&priv->lock); 7763 7764 netif_device_detach(ndev); 7765 7766 stmmac_disable_all_queues(priv); 7767 7768 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7769 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7770 7771 if (priv->eee_sw_timer_en) { 7772 priv->tx_path_in_lpi_mode = false; 7773 del_timer_sync(&priv->eee_ctrl_timer); 7774 } 7775 7776 /* Stop TX/RX DMA */ 7777 stmmac_stop_all_dma(priv); 7778 7779 if (priv->plat->serdes_powerdown) 7780 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7781 7782 /* Enable Power down mode by programming the PMT regs */ 7783 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7784 stmmac_pmt(priv, priv->hw, priv->wolopts); 7785 priv->irq_wake = 1; 7786 } else { 7787 stmmac_mac_set(priv, priv->ioaddr, false); 7788 pinctrl_pm_select_sleep_state(priv->device); 7789 } 7790 7791 mutex_unlock(&priv->lock); 7792 7793 rtnl_lock(); 7794 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7795 phylink_suspend(priv->phylink, true); 7796 } else { 7797 if (device_may_wakeup(priv->device)) 7798 phylink_speed_down(priv->phylink, false); 7799 phylink_suspend(priv->phylink, false); 7800 } 7801 rtnl_unlock(); 7802 7803 if (stmmac_fpe_supported(priv)) 7804 timer_shutdown_sync(&priv->fpe_cfg.verify_timer); 7805 7806 priv->speed = SPEED_UNKNOWN; 7807 return 0; 7808 } 7809 EXPORT_SYMBOL_GPL(stmmac_suspend); 7810 7811 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 7812 { 7813 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7814 7815 rx_q->cur_rx = 0; 7816 rx_q->dirty_rx = 0; 7817 } 7818 7819 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 7820 { 7821 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7822 7823 tx_q->cur_tx = 0; 7824 tx_q->dirty_tx = 0; 7825 tx_q->mss = 0; 7826 7827 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 7828 } 7829 7830 /** 7831 * stmmac_reset_queues_param - reset queue parameters 7832 * @priv: device pointer 7833 */ 7834 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 7835 { 7836 u32 rx_cnt = priv->plat->rx_queues_to_use; 7837 u32 tx_cnt = priv->plat->tx_queues_to_use; 7838 u32 queue; 7839 7840 for (queue = 0; queue < rx_cnt; queue++) 7841 stmmac_reset_rx_queue(priv, queue); 7842 7843 for (queue = 0; queue < tx_cnt; queue++) 7844 stmmac_reset_tx_queue(priv, queue); 7845 } 7846 7847 /** 7848 * stmmac_resume - resume callback 7849 * @dev: device pointer 7850 * Description: when resume this function is invoked to setup the DMA and CORE 7851 * in a usable state. 7852 */ 7853 int stmmac_resume(struct device *dev) 7854 { 7855 struct net_device *ndev = dev_get_drvdata(dev); 7856 struct stmmac_priv *priv = netdev_priv(ndev); 7857 int ret; 7858 7859 if (!netif_running(ndev)) 7860 return 0; 7861 7862 /* Power Down bit, into the PM register, is cleared 7863 * automatically as soon as a magic packet or a Wake-up frame 7864 * is received. Anyway, it's better to manually clear 7865 * this bit because it can generate problems while resuming 7866 * from another devices (e.g. serial console). 7867 */ 7868 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7869 mutex_lock(&priv->lock); 7870 stmmac_pmt(priv, priv->hw, 0); 7871 mutex_unlock(&priv->lock); 7872 priv->irq_wake = 0; 7873 } else { 7874 pinctrl_pm_select_default_state(priv->device); 7875 /* reset the phy so that it's ready */ 7876 if (priv->mii) 7877 stmmac_mdio_reset(priv->mii); 7878 } 7879 7880 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 7881 priv->plat->serdes_powerup) { 7882 ret = priv->plat->serdes_powerup(ndev, 7883 priv->plat->bsp_priv); 7884 7885 if (ret < 0) 7886 return ret; 7887 } 7888 7889 rtnl_lock(); 7890 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7891 phylink_resume(priv->phylink); 7892 } else { 7893 phylink_resume(priv->phylink); 7894 if (device_may_wakeup(priv->device)) 7895 phylink_speed_up(priv->phylink); 7896 } 7897 rtnl_unlock(); 7898 7899 rtnl_lock(); 7900 mutex_lock(&priv->lock); 7901 7902 stmmac_reset_queues_param(priv); 7903 7904 stmmac_free_tx_skbufs(priv); 7905 stmmac_clear_descriptors(priv, &priv->dma_conf); 7906 7907 stmmac_hw_setup(ndev, false); 7908 stmmac_init_coalesce(priv); 7909 stmmac_set_rx_mode(ndev); 7910 7911 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); 7912 7913 stmmac_enable_all_queues(priv); 7914 stmmac_enable_all_dma_irq(priv); 7915 7916 mutex_unlock(&priv->lock); 7917 rtnl_unlock(); 7918 7919 netif_device_attach(ndev); 7920 7921 return 0; 7922 } 7923 EXPORT_SYMBOL_GPL(stmmac_resume); 7924 7925 #ifndef MODULE 7926 static int __init stmmac_cmdline_opt(char *str) 7927 { 7928 char *opt; 7929 7930 if (!str || !*str) 7931 return 1; 7932 while ((opt = strsep(&str, ",")) != NULL) { 7933 if (!strncmp(opt, "debug:", 6)) { 7934 if (kstrtoint(opt + 6, 0, &debug)) 7935 goto err; 7936 } else if (!strncmp(opt, "phyaddr:", 8)) { 7937 if (kstrtoint(opt + 8, 0, &phyaddr)) 7938 goto err; 7939 } else if (!strncmp(opt, "buf_sz:", 7)) { 7940 if (kstrtoint(opt + 7, 0, &buf_sz)) 7941 goto err; 7942 } else if (!strncmp(opt, "tc:", 3)) { 7943 if (kstrtoint(opt + 3, 0, &tc)) 7944 goto err; 7945 } else if (!strncmp(opt, "watchdog:", 9)) { 7946 if (kstrtoint(opt + 9, 0, &watchdog)) 7947 goto err; 7948 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 7949 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 7950 goto err; 7951 } else if (!strncmp(opt, "pause:", 6)) { 7952 if (kstrtoint(opt + 6, 0, &pause)) 7953 goto err; 7954 } else if (!strncmp(opt, "eee_timer:", 10)) { 7955 if (kstrtoint(opt + 10, 0, &eee_timer)) 7956 goto err; 7957 } else if (!strncmp(opt, "chain_mode:", 11)) { 7958 if (kstrtoint(opt + 11, 0, &chain_mode)) 7959 goto err; 7960 } 7961 } 7962 return 1; 7963 7964 err: 7965 pr_err("%s: ERROR broken module parameter conversion", __func__); 7966 return 1; 7967 } 7968 7969 __setup("stmmaceth=", stmmac_cmdline_opt); 7970 #endif /* MODULE */ 7971 7972 static int __init stmmac_init(void) 7973 { 7974 #ifdef CONFIG_DEBUG_FS 7975 /* Create debugfs main directory if it doesn't exist yet */ 7976 if (!stmmac_fs_dir) 7977 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 7978 register_netdevice_notifier(&stmmac_notifier); 7979 #endif 7980 7981 return 0; 7982 } 7983 7984 static void __exit stmmac_exit(void) 7985 { 7986 #ifdef CONFIG_DEBUG_FS 7987 unregister_netdevice_notifier(&stmmac_notifier); 7988 debugfs_remove_recursive(stmmac_fs_dir); 7989 #endif 7990 } 7991 7992 module_init(stmmac_init) 7993 module_exit(stmmac_exit) 7994 7995 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 7996 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 7997 MODULE_LICENSE("GPL"); 7998