1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/clk.h> 18 #include <linux/kernel.h> 19 #include <linux/interrupt.h> 20 #include <linux/ip.h> 21 #include <linux/tcp.h> 22 #include <linux/skbuff.h> 23 #include <linux/ethtool.h> 24 #include <linux/if_ether.h> 25 #include <linux/crc32.h> 26 #include <linux/mii.h> 27 #include <linux/if.h> 28 #include <linux/if_vlan.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/slab.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/prefetch.h> 33 #include <linux/pinctrl/consumer.h> 34 #ifdef CONFIG_DEBUG_FS 35 #include <linux/debugfs.h> 36 #include <linux/seq_file.h> 37 #endif /* CONFIG_DEBUG_FS */ 38 #include <linux/net_tstamp.h> 39 #include <linux/phylink.h> 40 #include <linux/udp.h> 41 #include <linux/bpf_trace.h> 42 #include <net/page_pool/helpers.h> 43 #include <net/pkt_cls.h> 44 #include <net/xdp_sock_drv.h> 45 #include "stmmac_ptp.h" 46 #include "stmmac.h" 47 #include "stmmac_xdp.h" 48 #include <linux/reset.h> 49 #include <linux/of_mdio.h> 50 #include "dwmac1000.h" 51 #include "dwxgmac2.h" 52 #include "hwif.h" 53 54 /* As long as the interface is active, we keep the timestamping counter enabled 55 * with fine resolution and binary rollover. This avoid non-monotonic behavior 56 * (clock jumps) when changing timestamping settings at runtime. 57 */ 58 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ 59 PTP_TCR_TSCTRLSSR) 60 61 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 62 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 63 64 /* Module parameters */ 65 #define TX_TIMEO 5000 66 static int watchdog = TX_TIMEO; 67 module_param(watchdog, int, 0644); 68 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 69 70 static int debug = -1; 71 module_param(debug, int, 0644); 72 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 73 74 static int phyaddr = -1; 75 module_param(phyaddr, int, 0444); 76 MODULE_PARM_DESC(phyaddr, "Physical device address"); 77 78 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 79 #define STMMAC_RX_THRESH(x) ((x)->dma_conf.dma_rx_size / 4) 80 81 /* Limit to make sure XDP TX and slow path can coexist */ 82 #define STMMAC_XSK_TX_BUDGET_MAX 256 83 #define STMMAC_TX_XSK_AVAIL 16 84 #define STMMAC_RX_FILL_BATCH 16 85 86 #define STMMAC_XDP_PASS 0 87 #define STMMAC_XDP_CONSUMED BIT(0) 88 #define STMMAC_XDP_TX BIT(1) 89 #define STMMAC_XDP_REDIRECT BIT(2) 90 91 static int flow_ctrl = FLOW_AUTO; 92 module_param(flow_ctrl, int, 0644); 93 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 94 95 static int pause = PAUSE_TIME; 96 module_param(pause, int, 0644); 97 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 98 99 #define TC_DEFAULT 64 100 static int tc = TC_DEFAULT; 101 module_param(tc, int, 0644); 102 MODULE_PARM_DESC(tc, "DMA threshold control value"); 103 104 #define DEFAULT_BUFSIZE 1536 105 static int buf_sz = DEFAULT_BUFSIZE; 106 module_param(buf_sz, int, 0644); 107 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 108 109 #define STMMAC_RX_COPYBREAK 256 110 111 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 112 NETIF_MSG_LINK | NETIF_MSG_IFUP | 113 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 114 115 #define STMMAC_DEFAULT_LPI_TIMER 1000 116 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 117 module_param(eee_timer, int, 0644); 118 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 119 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 120 121 /* By default the driver will use the ring mode to manage tx and rx descriptors, 122 * but allow user to force to use the chain instead of the ring 123 */ 124 static unsigned int chain_mode; 125 module_param(chain_mode, int, 0444); 126 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 127 128 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 129 /* For MSI interrupts handling */ 130 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 131 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 132 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 133 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 134 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 135 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 136 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 137 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 138 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 139 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 140 u32 rxmode, u32 chan); 141 142 #ifdef CONFIG_DEBUG_FS 143 static const struct net_device_ops stmmac_netdev_ops; 144 static void stmmac_init_fs(struct net_device *dev); 145 static void stmmac_exit_fs(struct net_device *dev); 146 #endif 147 148 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 149 150 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) 151 { 152 int ret = 0; 153 154 if (enabled) { 155 ret = clk_prepare_enable(priv->plat->stmmac_clk); 156 if (ret) 157 return ret; 158 ret = clk_prepare_enable(priv->plat->pclk); 159 if (ret) { 160 clk_disable_unprepare(priv->plat->stmmac_clk); 161 return ret; 162 } 163 if (priv->plat->clks_config) { 164 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); 165 if (ret) { 166 clk_disable_unprepare(priv->plat->stmmac_clk); 167 clk_disable_unprepare(priv->plat->pclk); 168 return ret; 169 } 170 } 171 } else { 172 clk_disable_unprepare(priv->plat->stmmac_clk); 173 clk_disable_unprepare(priv->plat->pclk); 174 if (priv->plat->clks_config) 175 priv->plat->clks_config(priv->plat->bsp_priv, enabled); 176 } 177 178 return ret; 179 } 180 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); 181 182 /** 183 * stmmac_verify_args - verify the driver parameters. 184 * Description: it checks the driver parameters and set a default in case of 185 * errors. 186 */ 187 static void stmmac_verify_args(void) 188 { 189 if (unlikely(watchdog < 0)) 190 watchdog = TX_TIMEO; 191 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 192 buf_sz = DEFAULT_BUFSIZE; 193 if (unlikely(flow_ctrl > 1)) 194 flow_ctrl = FLOW_AUTO; 195 else if (likely(flow_ctrl < 0)) 196 flow_ctrl = FLOW_OFF; 197 if (unlikely((pause < 0) || (pause > 0xffff))) 198 pause = PAUSE_TIME; 199 if (eee_timer < 0) 200 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 201 } 202 203 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 204 { 205 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 206 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 207 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 208 u32 queue; 209 210 for (queue = 0; queue < maxq; queue++) { 211 struct stmmac_channel *ch = &priv->channel[queue]; 212 213 if (stmmac_xdp_is_enabled(priv) && 214 test_bit(queue, priv->af_xdp_zc_qps)) { 215 napi_disable(&ch->rxtx_napi); 216 continue; 217 } 218 219 if (queue < rx_queues_cnt) 220 napi_disable(&ch->rx_napi); 221 if (queue < tx_queues_cnt) 222 napi_disable(&ch->tx_napi); 223 } 224 } 225 226 /** 227 * stmmac_disable_all_queues - Disable all queues 228 * @priv: driver private structure 229 */ 230 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 231 { 232 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 233 struct stmmac_rx_queue *rx_q; 234 u32 queue; 235 236 /* synchronize_rcu() needed for pending XDP buffers to drain */ 237 for (queue = 0; queue < rx_queues_cnt; queue++) { 238 rx_q = &priv->dma_conf.rx_queue[queue]; 239 if (rx_q->xsk_pool) { 240 synchronize_rcu(); 241 break; 242 } 243 } 244 245 __stmmac_disable_all_queues(priv); 246 } 247 248 /** 249 * stmmac_enable_all_queues - Enable all queues 250 * @priv: driver private structure 251 */ 252 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 253 { 254 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 255 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 256 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 257 u32 queue; 258 259 for (queue = 0; queue < maxq; queue++) { 260 struct stmmac_channel *ch = &priv->channel[queue]; 261 262 if (stmmac_xdp_is_enabled(priv) && 263 test_bit(queue, priv->af_xdp_zc_qps)) { 264 napi_enable(&ch->rxtx_napi); 265 continue; 266 } 267 268 if (queue < rx_queues_cnt) 269 napi_enable(&ch->rx_napi); 270 if (queue < tx_queues_cnt) 271 napi_enable(&ch->tx_napi); 272 } 273 } 274 275 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 276 { 277 if (!test_bit(STMMAC_DOWN, &priv->state) && 278 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 279 queue_work(priv->wq, &priv->service_task); 280 } 281 282 static void stmmac_global_err(struct stmmac_priv *priv) 283 { 284 netif_carrier_off(priv->dev); 285 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 286 stmmac_service_event_schedule(priv); 287 } 288 289 /** 290 * stmmac_clk_csr_set - dynamically set the MDC clock 291 * @priv: driver private structure 292 * Description: this is to dynamically set the MDC clock according to the csr 293 * clock input. 294 * Note: 295 * If a specific clk_csr value is passed from the platform 296 * this means that the CSR Clock Range selection cannot be 297 * changed at run-time and it is fixed (as reported in the driver 298 * documentation). Viceversa the driver will try to set the MDC 299 * clock dynamically according to the actual clock input. 300 */ 301 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 302 { 303 u32 clk_rate; 304 305 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 306 307 /* Platform provided default clk_csr would be assumed valid 308 * for all other cases except for the below mentioned ones. 309 * For values higher than the IEEE 802.3 specified frequency 310 * we can not estimate the proper divider as it is not known 311 * the frequency of clk_csr_i. So we do not change the default 312 * divider. 313 */ 314 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 315 if (clk_rate < CSR_F_35M) 316 priv->clk_csr = STMMAC_CSR_20_35M; 317 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 318 priv->clk_csr = STMMAC_CSR_35_60M; 319 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 320 priv->clk_csr = STMMAC_CSR_60_100M; 321 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 322 priv->clk_csr = STMMAC_CSR_100_150M; 323 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 324 priv->clk_csr = STMMAC_CSR_150_250M; 325 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) 326 priv->clk_csr = STMMAC_CSR_250_300M; 327 } 328 329 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { 330 if (clk_rate > 160000000) 331 priv->clk_csr = 0x03; 332 else if (clk_rate > 80000000) 333 priv->clk_csr = 0x02; 334 else if (clk_rate > 40000000) 335 priv->clk_csr = 0x01; 336 else 337 priv->clk_csr = 0; 338 } 339 340 if (priv->plat->has_xgmac) { 341 if (clk_rate > 400000000) 342 priv->clk_csr = 0x5; 343 else if (clk_rate > 350000000) 344 priv->clk_csr = 0x4; 345 else if (clk_rate > 300000000) 346 priv->clk_csr = 0x3; 347 else if (clk_rate > 250000000) 348 priv->clk_csr = 0x2; 349 else if (clk_rate > 150000000) 350 priv->clk_csr = 0x1; 351 else 352 priv->clk_csr = 0x0; 353 } 354 } 355 356 static void print_pkt(unsigned char *buf, int len) 357 { 358 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 359 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 360 } 361 362 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 363 { 364 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 365 u32 avail; 366 367 if (tx_q->dirty_tx > tx_q->cur_tx) 368 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 369 else 370 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 371 372 return avail; 373 } 374 375 /** 376 * stmmac_rx_dirty - Get RX queue dirty 377 * @priv: driver private structure 378 * @queue: RX queue index 379 */ 380 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 381 { 382 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 383 u32 dirty; 384 385 if (rx_q->dirty_rx <= rx_q->cur_rx) 386 dirty = rx_q->cur_rx - rx_q->dirty_rx; 387 else 388 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 389 390 return dirty; 391 } 392 393 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) 394 { 395 int tx_lpi_timer; 396 397 /* Clear/set the SW EEE timer flag based on LPI ET enablement */ 398 priv->eee_sw_timer_en = en ? 0 : 1; 399 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; 400 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); 401 } 402 403 /** 404 * stmmac_enable_eee_mode - check and enter in LPI mode 405 * @priv: driver private structure 406 * Description: this function is to verify and enter in LPI mode in case of 407 * EEE. 408 */ 409 static int stmmac_enable_eee_mode(struct stmmac_priv *priv) 410 { 411 u32 tx_cnt = priv->plat->tx_queues_to_use; 412 u32 queue; 413 414 /* check if all TX queues have the work finished */ 415 for (queue = 0; queue < tx_cnt; queue++) { 416 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 417 418 if (tx_q->dirty_tx != tx_q->cur_tx) 419 return -EBUSY; /* still unfinished work */ 420 } 421 422 /* Check and enter in LPI mode */ 423 if (!priv->tx_path_in_lpi_mode) 424 stmmac_set_eee_mode(priv, priv->hw, 425 priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); 426 return 0; 427 } 428 429 /** 430 * stmmac_disable_eee_mode - disable and exit from LPI mode 431 * @priv: driver private structure 432 * Description: this function is to exit and disable EEE in case of 433 * LPI state is true. This is called by the xmit. 434 */ 435 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 436 { 437 if (!priv->eee_sw_timer_en) { 438 stmmac_lpi_entry_timer_config(priv, 0); 439 return; 440 } 441 442 stmmac_reset_eee_mode(priv, priv->hw); 443 del_timer_sync(&priv->eee_ctrl_timer); 444 priv->tx_path_in_lpi_mode = false; 445 } 446 447 /** 448 * stmmac_eee_ctrl_timer - EEE TX SW timer. 449 * @t: timer_list struct containing private info 450 * Description: 451 * if there is no data transfer and if we are not in LPI state, 452 * then MAC Transmitter can be moved to LPI state. 453 */ 454 static void stmmac_eee_ctrl_timer(struct timer_list *t) 455 { 456 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 457 458 if (stmmac_enable_eee_mode(priv)) 459 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 460 } 461 462 /** 463 * stmmac_eee_init - init EEE 464 * @priv: driver private structure 465 * Description: 466 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 467 * can also manage EEE, this function enable the LPI state and start related 468 * timer. 469 */ 470 bool stmmac_eee_init(struct stmmac_priv *priv) 471 { 472 int eee_tw_timer = priv->eee_tw_timer; 473 474 /* Using PCS we cannot dial with the phy registers at this stage 475 * so we do not support extra feature like EEE. 476 */ 477 if (priv->hw->pcs == STMMAC_PCS_TBI || 478 priv->hw->pcs == STMMAC_PCS_RTBI) 479 return false; 480 481 /* Check if MAC core supports the EEE feature. */ 482 if (!priv->dma_cap.eee) 483 return false; 484 485 mutex_lock(&priv->lock); 486 487 /* Check if it needs to be deactivated */ 488 if (!priv->eee_active) { 489 if (priv->eee_enabled) { 490 netdev_dbg(priv->dev, "disable EEE\n"); 491 stmmac_lpi_entry_timer_config(priv, 0); 492 del_timer_sync(&priv->eee_ctrl_timer); 493 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); 494 if (priv->hw->xpcs) 495 xpcs_config_eee(priv->hw->xpcs, 496 priv->plat->mult_fact_100ns, 497 false); 498 } 499 mutex_unlock(&priv->lock); 500 return false; 501 } 502 503 if (priv->eee_active && !priv->eee_enabled) { 504 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 505 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 506 eee_tw_timer); 507 if (priv->hw->xpcs) 508 xpcs_config_eee(priv->hw->xpcs, 509 priv->plat->mult_fact_100ns, 510 true); 511 } 512 513 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { 514 del_timer_sync(&priv->eee_ctrl_timer); 515 priv->tx_path_in_lpi_mode = false; 516 stmmac_lpi_entry_timer_config(priv, 1); 517 } else { 518 stmmac_lpi_entry_timer_config(priv, 0); 519 mod_timer(&priv->eee_ctrl_timer, 520 STMMAC_LPI_T(priv->tx_lpi_timer)); 521 } 522 523 mutex_unlock(&priv->lock); 524 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 525 return true; 526 } 527 528 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 529 * @priv: driver private structure 530 * @p : descriptor pointer 531 * @skb : the socket buffer 532 * Description : 533 * This function will read timestamp from the descriptor & pass it to stack. 534 * and also perform some sanity checks. 535 */ 536 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 537 struct dma_desc *p, struct sk_buff *skb) 538 { 539 struct skb_shared_hwtstamps shhwtstamp; 540 bool found = false; 541 u64 ns = 0; 542 543 if (!priv->hwts_tx_en) 544 return; 545 546 /* exit if skb doesn't support hw tstamp */ 547 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 548 return; 549 550 /* check tx tstamp status */ 551 if (stmmac_get_tx_timestamp_status(priv, p)) { 552 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 553 found = true; 554 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 555 found = true; 556 } 557 558 if (found) { 559 ns -= priv->plat->cdc_error_adj; 560 561 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 562 shhwtstamp.hwtstamp = ns_to_ktime(ns); 563 564 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 565 /* pass tstamp to stack */ 566 skb_tstamp_tx(skb, &shhwtstamp); 567 } 568 } 569 570 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 571 * @priv: driver private structure 572 * @p : descriptor pointer 573 * @np : next descriptor pointer 574 * @skb : the socket buffer 575 * Description : 576 * This function will read received packet's timestamp from the descriptor 577 * and pass it to stack. It also perform some sanity checks. 578 */ 579 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 580 struct dma_desc *np, struct sk_buff *skb) 581 { 582 struct skb_shared_hwtstamps *shhwtstamp = NULL; 583 struct dma_desc *desc = p; 584 u64 ns = 0; 585 586 if (!priv->hwts_rx_en) 587 return; 588 /* For GMAC4, the valid timestamp is from CTX next desc. */ 589 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 590 desc = np; 591 592 /* Check if timestamp is available */ 593 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 594 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 595 596 ns -= priv->plat->cdc_error_adj; 597 598 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 599 shhwtstamp = skb_hwtstamps(skb); 600 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 601 shhwtstamp->hwtstamp = ns_to_ktime(ns); 602 } else { 603 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 604 } 605 } 606 607 /** 608 * stmmac_hwtstamp_set - control hardware timestamping. 609 * @dev: device pointer. 610 * @ifr: An IOCTL specific structure, that can contain a pointer to 611 * a proprietary structure used to pass information to the driver. 612 * Description: 613 * This function configures the MAC to enable/disable both outgoing(TX) 614 * and incoming(RX) packets time stamping based on user input. 615 * Return Value: 616 * 0 on success and an appropriate -ve integer on failure. 617 */ 618 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 619 { 620 struct stmmac_priv *priv = netdev_priv(dev); 621 struct hwtstamp_config config; 622 u32 ptp_v2 = 0; 623 u32 tstamp_all = 0; 624 u32 ptp_over_ipv4_udp = 0; 625 u32 ptp_over_ipv6_udp = 0; 626 u32 ptp_over_ethernet = 0; 627 u32 snap_type_sel = 0; 628 u32 ts_master_en = 0; 629 u32 ts_event_en = 0; 630 631 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 632 netdev_alert(priv->dev, "No support for HW time stamping\n"); 633 priv->hwts_tx_en = 0; 634 priv->hwts_rx_en = 0; 635 636 return -EOPNOTSUPP; 637 } 638 639 if (copy_from_user(&config, ifr->ifr_data, 640 sizeof(config))) 641 return -EFAULT; 642 643 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 644 __func__, config.flags, config.tx_type, config.rx_filter); 645 646 if (config.tx_type != HWTSTAMP_TX_OFF && 647 config.tx_type != HWTSTAMP_TX_ON) 648 return -ERANGE; 649 650 if (priv->adv_ts) { 651 switch (config.rx_filter) { 652 case HWTSTAMP_FILTER_NONE: 653 /* time stamp no incoming packet at all */ 654 config.rx_filter = HWTSTAMP_FILTER_NONE; 655 break; 656 657 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 658 /* PTP v1, UDP, any kind of event packet */ 659 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 660 /* 'xmac' hardware can support Sync, Pdelay_Req and 661 * Pdelay_resp by setting bit14 and bits17/16 to 01 662 * This leaves Delay_Req timestamps out. 663 * Enable all events *and* general purpose message 664 * timestamping 665 */ 666 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 667 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 668 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 669 break; 670 671 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 672 /* PTP v1, UDP, Sync packet */ 673 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 674 /* take time stamp for SYNC messages only */ 675 ts_event_en = PTP_TCR_TSEVNTENA; 676 677 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 678 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 679 break; 680 681 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 682 /* PTP v1, UDP, Delay_req packet */ 683 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 684 /* take time stamp for Delay_Req messages only */ 685 ts_master_en = PTP_TCR_TSMSTRENA; 686 ts_event_en = PTP_TCR_TSEVNTENA; 687 688 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 689 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 690 break; 691 692 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 693 /* PTP v2, UDP, any kind of event packet */ 694 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 695 ptp_v2 = PTP_TCR_TSVER2ENA; 696 /* take time stamp for all event messages */ 697 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 698 699 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 700 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 701 break; 702 703 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 704 /* PTP v2, UDP, Sync packet */ 705 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 706 ptp_v2 = PTP_TCR_TSVER2ENA; 707 /* take time stamp for SYNC messages only */ 708 ts_event_en = PTP_TCR_TSEVNTENA; 709 710 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 711 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 712 break; 713 714 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 715 /* PTP v2, UDP, Delay_req packet */ 716 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 717 ptp_v2 = PTP_TCR_TSVER2ENA; 718 /* take time stamp for Delay_Req messages only */ 719 ts_master_en = PTP_TCR_TSMSTRENA; 720 ts_event_en = PTP_TCR_TSEVNTENA; 721 722 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 723 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 724 break; 725 726 case HWTSTAMP_FILTER_PTP_V2_EVENT: 727 /* PTP v2/802.AS1 any layer, any kind of event packet */ 728 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 729 ptp_v2 = PTP_TCR_TSVER2ENA; 730 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 731 if (priv->synopsys_id < DWMAC_CORE_4_10) 732 ts_event_en = PTP_TCR_TSEVNTENA; 733 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 734 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 735 ptp_over_ethernet = PTP_TCR_TSIPENA; 736 break; 737 738 case HWTSTAMP_FILTER_PTP_V2_SYNC: 739 /* PTP v2/802.AS1, any layer, Sync packet */ 740 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 741 ptp_v2 = PTP_TCR_TSVER2ENA; 742 /* take time stamp for SYNC messages only */ 743 ts_event_en = PTP_TCR_TSEVNTENA; 744 745 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 746 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 747 ptp_over_ethernet = PTP_TCR_TSIPENA; 748 break; 749 750 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 751 /* PTP v2/802.AS1, any layer, Delay_req packet */ 752 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 753 ptp_v2 = PTP_TCR_TSVER2ENA; 754 /* take time stamp for Delay_Req messages only */ 755 ts_master_en = PTP_TCR_TSMSTRENA; 756 ts_event_en = PTP_TCR_TSEVNTENA; 757 758 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 759 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 760 ptp_over_ethernet = PTP_TCR_TSIPENA; 761 break; 762 763 case HWTSTAMP_FILTER_NTP_ALL: 764 case HWTSTAMP_FILTER_ALL: 765 /* time stamp any incoming packet */ 766 config.rx_filter = HWTSTAMP_FILTER_ALL; 767 tstamp_all = PTP_TCR_TSENALL; 768 break; 769 770 default: 771 return -ERANGE; 772 } 773 } else { 774 switch (config.rx_filter) { 775 case HWTSTAMP_FILTER_NONE: 776 config.rx_filter = HWTSTAMP_FILTER_NONE; 777 break; 778 default: 779 /* PTP v1, UDP, any kind of event packet */ 780 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 781 break; 782 } 783 } 784 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 785 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 786 787 priv->systime_flags = STMMAC_HWTS_ACTIVE; 788 789 if (priv->hwts_tx_en || priv->hwts_rx_en) { 790 priv->systime_flags |= tstamp_all | ptp_v2 | 791 ptp_over_ethernet | ptp_over_ipv6_udp | 792 ptp_over_ipv4_udp | ts_event_en | 793 ts_master_en | snap_type_sel; 794 } 795 796 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 797 798 memcpy(&priv->tstamp_config, &config, sizeof(config)); 799 800 return copy_to_user(ifr->ifr_data, &config, 801 sizeof(config)) ? -EFAULT : 0; 802 } 803 804 /** 805 * stmmac_hwtstamp_get - read hardware timestamping. 806 * @dev: device pointer. 807 * @ifr: An IOCTL specific structure, that can contain a pointer to 808 * a proprietary structure used to pass information to the driver. 809 * Description: 810 * This function obtain the current hardware timestamping settings 811 * as requested. 812 */ 813 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 814 { 815 struct stmmac_priv *priv = netdev_priv(dev); 816 struct hwtstamp_config *config = &priv->tstamp_config; 817 818 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 819 return -EOPNOTSUPP; 820 821 return copy_to_user(ifr->ifr_data, config, 822 sizeof(*config)) ? -EFAULT : 0; 823 } 824 825 /** 826 * stmmac_init_tstamp_counter - init hardware timestamping counter 827 * @priv: driver private structure 828 * @systime_flags: timestamping flags 829 * Description: 830 * Initialize hardware counter for packet timestamping. 831 * This is valid as long as the interface is open and not suspended. 832 * Will be rerun after resuming from suspend, case in which the timestamping 833 * flags updated by stmmac_hwtstamp_set() also need to be restored. 834 */ 835 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) 836 { 837 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 838 struct timespec64 now; 839 u32 sec_inc = 0; 840 u64 temp = 0; 841 842 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 843 return -EOPNOTSUPP; 844 845 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 846 priv->systime_flags = systime_flags; 847 848 /* program Sub Second Increment reg */ 849 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 850 priv->plat->clk_ptp_rate, 851 xmac, &sec_inc); 852 temp = div_u64(1000000000ULL, sec_inc); 853 854 /* Store sub second increment for later use */ 855 priv->sub_second_inc = sec_inc; 856 857 /* calculate default added value: 858 * formula is : 859 * addend = (2^32)/freq_div_ratio; 860 * where, freq_div_ratio = 1e9ns/sec_inc 861 */ 862 temp = (u64)(temp << 32); 863 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 864 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 865 866 /* initialize system time */ 867 ktime_get_real_ts64(&now); 868 869 /* lower 32 bits of tv_sec are safe until y2106 */ 870 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 871 872 return 0; 873 } 874 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); 875 876 /** 877 * stmmac_init_ptp - init PTP 878 * @priv: driver private structure 879 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 880 * This is done by looking at the HW cap. register. 881 * This function also registers the ptp driver. 882 */ 883 static int stmmac_init_ptp(struct stmmac_priv *priv) 884 { 885 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 886 int ret; 887 888 if (priv->plat->ptp_clk_freq_config) 889 priv->plat->ptp_clk_freq_config(priv); 890 891 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); 892 if (ret) 893 return ret; 894 895 priv->adv_ts = 0; 896 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 897 if (xmac && priv->dma_cap.atime_stamp) 898 priv->adv_ts = 1; 899 /* Dwmac 3.x core with extend_desc can support adv_ts */ 900 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 901 priv->adv_ts = 1; 902 903 if (priv->dma_cap.time_stamp) 904 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 905 906 if (priv->adv_ts) 907 netdev_info(priv->dev, 908 "IEEE 1588-2008 Advanced Timestamp supported\n"); 909 910 priv->hwts_tx_en = 0; 911 priv->hwts_rx_en = 0; 912 913 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 914 stmmac_hwtstamp_correct_latency(priv, priv); 915 916 return 0; 917 } 918 919 static void stmmac_release_ptp(struct stmmac_priv *priv) 920 { 921 clk_disable_unprepare(priv->plat->clk_ptp_ref); 922 stmmac_ptp_unregister(priv); 923 } 924 925 /** 926 * stmmac_mac_flow_ctrl - Configure flow control in all queues 927 * @priv: driver private structure 928 * @duplex: duplex passed to the next function 929 * Description: It is used for configuring the flow control in all queues 930 */ 931 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 932 { 933 u32 tx_cnt = priv->plat->tx_queues_to_use; 934 935 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 936 priv->pause, tx_cnt); 937 } 938 939 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 940 phy_interface_t interface) 941 { 942 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 943 944 if (priv->hw->xpcs) 945 return &priv->hw->xpcs->pcs; 946 947 if (priv->hw->lynx_pcs) 948 return priv->hw->lynx_pcs; 949 950 return NULL; 951 } 952 953 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 954 const struct phylink_link_state *state) 955 { 956 /* Nothing to do, xpcs_config() handles everything */ 957 } 958 959 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) 960 { 961 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 962 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 963 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 964 bool *hs_enable = &fpe_cfg->hs_enable; 965 966 if (is_up && *hs_enable) { 967 stmmac_fpe_send_mpacket(priv, priv->ioaddr, fpe_cfg, 968 MPACKET_VERIFY); 969 } else { 970 *lo_state = FPE_STATE_OFF; 971 *lp_state = FPE_STATE_OFF; 972 } 973 } 974 975 static void stmmac_mac_link_down(struct phylink_config *config, 976 unsigned int mode, phy_interface_t interface) 977 { 978 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 979 980 stmmac_mac_set(priv, priv->ioaddr, false); 981 priv->eee_active = false; 982 priv->tx_lpi_enabled = false; 983 priv->eee_enabled = stmmac_eee_init(priv); 984 stmmac_set_eee_pls(priv, priv->hw, false); 985 986 if (priv->dma_cap.fpesel) 987 stmmac_fpe_link_state_handle(priv, false); 988 } 989 990 static void stmmac_mac_link_up(struct phylink_config *config, 991 struct phy_device *phy, 992 unsigned int mode, phy_interface_t interface, 993 int speed, int duplex, 994 bool tx_pause, bool rx_pause) 995 { 996 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 997 u32 old_ctrl, ctrl; 998 999 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 1000 priv->plat->serdes_powerup) 1001 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 1002 1003 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1004 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1005 1006 if (interface == PHY_INTERFACE_MODE_USXGMII) { 1007 switch (speed) { 1008 case SPEED_10000: 1009 ctrl |= priv->hw->link.xgmii.speed10000; 1010 break; 1011 case SPEED_5000: 1012 ctrl |= priv->hw->link.xgmii.speed5000; 1013 break; 1014 case SPEED_2500: 1015 ctrl |= priv->hw->link.xgmii.speed2500; 1016 break; 1017 default: 1018 return; 1019 } 1020 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1021 switch (speed) { 1022 case SPEED_100000: 1023 ctrl |= priv->hw->link.xlgmii.speed100000; 1024 break; 1025 case SPEED_50000: 1026 ctrl |= priv->hw->link.xlgmii.speed50000; 1027 break; 1028 case SPEED_40000: 1029 ctrl |= priv->hw->link.xlgmii.speed40000; 1030 break; 1031 case SPEED_25000: 1032 ctrl |= priv->hw->link.xlgmii.speed25000; 1033 break; 1034 case SPEED_10000: 1035 ctrl |= priv->hw->link.xgmii.speed10000; 1036 break; 1037 case SPEED_2500: 1038 ctrl |= priv->hw->link.speed2500; 1039 break; 1040 case SPEED_1000: 1041 ctrl |= priv->hw->link.speed1000; 1042 break; 1043 default: 1044 return; 1045 } 1046 } else { 1047 switch (speed) { 1048 case SPEED_2500: 1049 ctrl |= priv->hw->link.speed2500; 1050 break; 1051 case SPEED_1000: 1052 ctrl |= priv->hw->link.speed1000; 1053 break; 1054 case SPEED_100: 1055 ctrl |= priv->hw->link.speed100; 1056 break; 1057 case SPEED_10: 1058 ctrl |= priv->hw->link.speed10; 1059 break; 1060 default: 1061 return; 1062 } 1063 } 1064 1065 priv->speed = speed; 1066 1067 if (priv->plat->fix_mac_speed) 1068 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); 1069 1070 if (!duplex) 1071 ctrl &= ~priv->hw->link.duplex; 1072 else 1073 ctrl |= priv->hw->link.duplex; 1074 1075 /* Flow Control operation */ 1076 if (rx_pause && tx_pause) 1077 priv->flow_ctrl = FLOW_AUTO; 1078 else if (rx_pause && !tx_pause) 1079 priv->flow_ctrl = FLOW_RX; 1080 else if (!rx_pause && tx_pause) 1081 priv->flow_ctrl = FLOW_TX; 1082 else 1083 priv->flow_ctrl = FLOW_OFF; 1084 1085 stmmac_mac_flow_ctrl(priv, duplex); 1086 1087 if (ctrl != old_ctrl) 1088 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1089 1090 stmmac_mac_set(priv, priv->ioaddr, true); 1091 if (phy && priv->dma_cap.eee) { 1092 priv->eee_active = 1093 phy_init_eee(phy, !(priv->plat->flags & 1094 STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) >= 0; 1095 priv->eee_enabled = stmmac_eee_init(priv); 1096 priv->tx_lpi_enabled = priv->eee_enabled; 1097 stmmac_set_eee_pls(priv, priv->hw, true); 1098 } 1099 1100 if (priv->dma_cap.fpesel) 1101 stmmac_fpe_link_state_handle(priv, true); 1102 1103 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1104 stmmac_hwtstamp_correct_latency(priv, priv); 1105 } 1106 1107 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1108 .mac_select_pcs = stmmac_mac_select_pcs, 1109 .mac_config = stmmac_mac_config, 1110 .mac_link_down = stmmac_mac_link_down, 1111 .mac_link_up = stmmac_mac_link_up, 1112 }; 1113 1114 /** 1115 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1116 * @priv: driver private structure 1117 * Description: this is to verify if the HW supports the PCS. 1118 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1119 * configured for the TBI, RTBI, or SGMII PHY interface. 1120 */ 1121 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1122 { 1123 int interface = priv->plat->mac_interface; 1124 1125 if (priv->dma_cap.pcs) { 1126 if ((interface == PHY_INTERFACE_MODE_RGMII) || 1127 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 1128 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 1129 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 1130 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 1131 priv->hw->pcs = STMMAC_PCS_RGMII; 1132 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 1133 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1134 priv->hw->pcs = STMMAC_PCS_SGMII; 1135 } 1136 } 1137 } 1138 1139 /** 1140 * stmmac_init_phy - PHY initialization 1141 * @dev: net device structure 1142 * Description: it initializes the driver's PHY state, and attaches the PHY 1143 * to the mac driver. 1144 * Return value: 1145 * 0 on success 1146 */ 1147 static int stmmac_init_phy(struct net_device *dev) 1148 { 1149 struct stmmac_priv *priv = netdev_priv(dev); 1150 struct fwnode_handle *phy_fwnode; 1151 struct fwnode_handle *fwnode; 1152 int ret; 1153 1154 if (!phylink_expects_phy(priv->phylink)) 1155 return 0; 1156 1157 fwnode = priv->plat->port_node; 1158 if (!fwnode) 1159 fwnode = dev_fwnode(priv->device); 1160 1161 if (fwnode) 1162 phy_fwnode = fwnode_get_phy_node(fwnode); 1163 else 1164 phy_fwnode = NULL; 1165 1166 /* Some DT bindings do not set-up the PHY handle. Let's try to 1167 * manually parse it 1168 */ 1169 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1170 int addr = priv->plat->phy_addr; 1171 struct phy_device *phydev; 1172 1173 if (addr < 0) { 1174 netdev_err(priv->dev, "no phy found\n"); 1175 return -ENODEV; 1176 } 1177 1178 phydev = mdiobus_get_phy(priv->mii, addr); 1179 if (!phydev) { 1180 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1181 return -ENODEV; 1182 } 1183 1184 ret = phylink_connect_phy(priv->phylink, phydev); 1185 } else { 1186 fwnode_handle_put(phy_fwnode); 1187 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); 1188 } 1189 1190 if (!priv->plat->pmt) { 1191 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 1192 1193 phylink_ethtool_get_wol(priv->phylink, &wol); 1194 device_set_wakeup_capable(priv->device, !!wol.supported); 1195 device_set_wakeup_enable(priv->device, !!wol.wolopts); 1196 } 1197 1198 return ret; 1199 } 1200 1201 static void stmmac_set_half_duplex(struct stmmac_priv *priv) 1202 { 1203 /* Half-Duplex can only work with single tx queue */ 1204 if (priv->plat->tx_queues_to_use > 1) 1205 priv->phylink_config.mac_capabilities &= 1206 ~(MAC_10HD | MAC_100HD | MAC_1000HD); 1207 else 1208 priv->phylink_config.mac_capabilities |= 1209 (MAC_10HD | MAC_100HD | MAC_1000HD); 1210 } 1211 1212 static int stmmac_phy_setup(struct stmmac_priv *priv) 1213 { 1214 struct stmmac_mdio_bus_data *mdio_bus_data; 1215 int mode = priv->plat->phy_interface; 1216 struct fwnode_handle *fwnode; 1217 struct phylink *phylink; 1218 int max_speed; 1219 1220 priv->phylink_config.dev = &priv->dev->dev; 1221 priv->phylink_config.type = PHYLINK_NETDEV; 1222 priv->phylink_config.mac_managed_pm = true; 1223 1224 mdio_bus_data = priv->plat->mdio_bus_data; 1225 if (mdio_bus_data) 1226 priv->phylink_config.ovr_an_inband = 1227 mdio_bus_data->xpcs_an_inband; 1228 1229 /* Set the platform/firmware specified interface mode. Note, phylink 1230 * deals with the PHY interface mode, not the MAC interface mode. 1231 */ 1232 __set_bit(mode, priv->phylink_config.supported_interfaces); 1233 1234 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1235 if (priv->hw->xpcs) 1236 xpcs_get_interfaces(priv->hw->xpcs, 1237 priv->phylink_config.supported_interfaces); 1238 1239 priv->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | 1240 MAC_10FD | MAC_100FD | 1241 MAC_1000FD; 1242 1243 stmmac_set_half_duplex(priv); 1244 1245 /* Get the MAC specific capabilities */ 1246 stmmac_mac_phylink_get_caps(priv); 1247 1248 max_speed = priv->plat->max_speed; 1249 if (max_speed) 1250 phylink_limit_mac_speed(&priv->phylink_config, max_speed); 1251 1252 fwnode = priv->plat->port_node; 1253 if (!fwnode) 1254 fwnode = dev_fwnode(priv->device); 1255 1256 phylink = phylink_create(&priv->phylink_config, fwnode, 1257 mode, &stmmac_phylink_mac_ops); 1258 if (IS_ERR(phylink)) 1259 return PTR_ERR(phylink); 1260 1261 priv->phylink = phylink; 1262 return 0; 1263 } 1264 1265 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1266 struct stmmac_dma_conf *dma_conf) 1267 { 1268 u32 rx_cnt = priv->plat->rx_queues_to_use; 1269 unsigned int desc_size; 1270 void *head_rx; 1271 u32 queue; 1272 1273 /* Display RX rings */ 1274 for (queue = 0; queue < rx_cnt; queue++) { 1275 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1276 1277 pr_info("\tRX Queue %u rings\n", queue); 1278 1279 if (priv->extend_desc) { 1280 head_rx = (void *)rx_q->dma_erx; 1281 desc_size = sizeof(struct dma_extended_desc); 1282 } else { 1283 head_rx = (void *)rx_q->dma_rx; 1284 desc_size = sizeof(struct dma_desc); 1285 } 1286 1287 /* Display RX ring */ 1288 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1289 rx_q->dma_rx_phy, desc_size); 1290 } 1291 } 1292 1293 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1294 struct stmmac_dma_conf *dma_conf) 1295 { 1296 u32 tx_cnt = priv->plat->tx_queues_to_use; 1297 unsigned int desc_size; 1298 void *head_tx; 1299 u32 queue; 1300 1301 /* Display TX rings */ 1302 for (queue = 0; queue < tx_cnt; queue++) { 1303 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1304 1305 pr_info("\tTX Queue %d rings\n", queue); 1306 1307 if (priv->extend_desc) { 1308 head_tx = (void *)tx_q->dma_etx; 1309 desc_size = sizeof(struct dma_extended_desc); 1310 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1311 head_tx = (void *)tx_q->dma_entx; 1312 desc_size = sizeof(struct dma_edesc); 1313 } else { 1314 head_tx = (void *)tx_q->dma_tx; 1315 desc_size = sizeof(struct dma_desc); 1316 } 1317 1318 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1319 tx_q->dma_tx_phy, desc_size); 1320 } 1321 } 1322 1323 static void stmmac_display_rings(struct stmmac_priv *priv, 1324 struct stmmac_dma_conf *dma_conf) 1325 { 1326 /* Display RX ring */ 1327 stmmac_display_rx_rings(priv, dma_conf); 1328 1329 /* Display TX ring */ 1330 stmmac_display_tx_rings(priv, dma_conf); 1331 } 1332 1333 static int stmmac_set_bfsize(int mtu, int bufsize) 1334 { 1335 int ret = bufsize; 1336 1337 if (mtu >= BUF_SIZE_8KiB) 1338 ret = BUF_SIZE_16KiB; 1339 else if (mtu >= BUF_SIZE_4KiB) 1340 ret = BUF_SIZE_8KiB; 1341 else if (mtu >= BUF_SIZE_2KiB) 1342 ret = BUF_SIZE_4KiB; 1343 else if (mtu > DEFAULT_BUFSIZE) 1344 ret = BUF_SIZE_2KiB; 1345 else 1346 ret = DEFAULT_BUFSIZE; 1347 1348 return ret; 1349 } 1350 1351 /** 1352 * stmmac_clear_rx_descriptors - clear RX descriptors 1353 * @priv: driver private structure 1354 * @dma_conf: structure to take the dma data 1355 * @queue: RX queue index 1356 * Description: this function is called to clear the RX descriptors 1357 * in case of both basic and extended descriptors are used. 1358 */ 1359 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1360 struct stmmac_dma_conf *dma_conf, 1361 u32 queue) 1362 { 1363 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1364 int i; 1365 1366 /* Clear the RX descriptors */ 1367 for (i = 0; i < dma_conf->dma_rx_size; i++) 1368 if (priv->extend_desc) 1369 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1370 priv->use_riwt, priv->mode, 1371 (i == dma_conf->dma_rx_size - 1), 1372 dma_conf->dma_buf_sz); 1373 else 1374 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1375 priv->use_riwt, priv->mode, 1376 (i == dma_conf->dma_rx_size - 1), 1377 dma_conf->dma_buf_sz); 1378 } 1379 1380 /** 1381 * stmmac_clear_tx_descriptors - clear tx descriptors 1382 * @priv: driver private structure 1383 * @dma_conf: structure to take the dma data 1384 * @queue: TX queue index. 1385 * Description: this function is called to clear the TX descriptors 1386 * in case of both basic and extended descriptors are used. 1387 */ 1388 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1389 struct stmmac_dma_conf *dma_conf, 1390 u32 queue) 1391 { 1392 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1393 int i; 1394 1395 /* Clear the TX descriptors */ 1396 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1397 int last = (i == (dma_conf->dma_tx_size - 1)); 1398 struct dma_desc *p; 1399 1400 if (priv->extend_desc) 1401 p = &tx_q->dma_etx[i].basic; 1402 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1403 p = &tx_q->dma_entx[i].basic; 1404 else 1405 p = &tx_q->dma_tx[i]; 1406 1407 stmmac_init_tx_desc(priv, p, priv->mode, last); 1408 } 1409 } 1410 1411 /** 1412 * stmmac_clear_descriptors - clear descriptors 1413 * @priv: driver private structure 1414 * @dma_conf: structure to take the dma data 1415 * Description: this function is called to clear the TX and RX descriptors 1416 * in case of both basic and extended descriptors are used. 1417 */ 1418 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1419 struct stmmac_dma_conf *dma_conf) 1420 { 1421 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1422 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1423 u32 queue; 1424 1425 /* Clear the RX descriptors */ 1426 for (queue = 0; queue < rx_queue_cnt; queue++) 1427 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1428 1429 /* Clear the TX descriptors */ 1430 for (queue = 0; queue < tx_queue_cnt; queue++) 1431 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1432 } 1433 1434 /** 1435 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1436 * @priv: driver private structure 1437 * @dma_conf: structure to take the dma data 1438 * @p: descriptor pointer 1439 * @i: descriptor index 1440 * @flags: gfp flag 1441 * @queue: RX queue index 1442 * Description: this function is called to allocate a receive buffer, perform 1443 * the DMA mapping and init the descriptor. 1444 */ 1445 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1446 struct stmmac_dma_conf *dma_conf, 1447 struct dma_desc *p, 1448 int i, gfp_t flags, u32 queue) 1449 { 1450 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1451 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1452 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1453 1454 if (priv->dma_cap.host_dma_width <= 32) 1455 gfp |= GFP_DMA32; 1456 1457 if (!buf->page) { 1458 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1459 if (!buf->page) 1460 return -ENOMEM; 1461 buf->page_offset = stmmac_rx_offset(priv); 1462 } 1463 1464 if (priv->sph && !buf->sec_page) { 1465 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1466 if (!buf->sec_page) 1467 return -ENOMEM; 1468 1469 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1470 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1471 } else { 1472 buf->sec_page = NULL; 1473 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1474 } 1475 1476 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1477 1478 stmmac_set_desc_addr(priv, p, buf->addr); 1479 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1480 stmmac_init_desc3(priv, p); 1481 1482 return 0; 1483 } 1484 1485 /** 1486 * stmmac_free_rx_buffer - free RX dma buffers 1487 * @priv: private structure 1488 * @rx_q: RX queue 1489 * @i: buffer index. 1490 */ 1491 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1492 struct stmmac_rx_queue *rx_q, 1493 int i) 1494 { 1495 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1496 1497 if (buf->page) 1498 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1499 buf->page = NULL; 1500 1501 if (buf->sec_page) 1502 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1503 buf->sec_page = NULL; 1504 } 1505 1506 /** 1507 * stmmac_free_tx_buffer - free RX dma buffers 1508 * @priv: private structure 1509 * @dma_conf: structure to take the dma data 1510 * @queue: RX queue index 1511 * @i: buffer index. 1512 */ 1513 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1514 struct stmmac_dma_conf *dma_conf, 1515 u32 queue, int i) 1516 { 1517 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1518 1519 if (tx_q->tx_skbuff_dma[i].buf && 1520 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1521 if (tx_q->tx_skbuff_dma[i].map_as_page) 1522 dma_unmap_page(priv->device, 1523 tx_q->tx_skbuff_dma[i].buf, 1524 tx_q->tx_skbuff_dma[i].len, 1525 DMA_TO_DEVICE); 1526 else 1527 dma_unmap_single(priv->device, 1528 tx_q->tx_skbuff_dma[i].buf, 1529 tx_q->tx_skbuff_dma[i].len, 1530 DMA_TO_DEVICE); 1531 } 1532 1533 if (tx_q->xdpf[i] && 1534 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1535 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1536 xdp_return_frame(tx_q->xdpf[i]); 1537 tx_q->xdpf[i] = NULL; 1538 } 1539 1540 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1541 tx_q->xsk_frames_done++; 1542 1543 if (tx_q->tx_skbuff[i] && 1544 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1545 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1546 tx_q->tx_skbuff[i] = NULL; 1547 } 1548 1549 tx_q->tx_skbuff_dma[i].buf = 0; 1550 tx_q->tx_skbuff_dma[i].map_as_page = false; 1551 } 1552 1553 /** 1554 * dma_free_rx_skbufs - free RX dma buffers 1555 * @priv: private structure 1556 * @dma_conf: structure to take the dma data 1557 * @queue: RX queue index 1558 */ 1559 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1560 struct stmmac_dma_conf *dma_conf, 1561 u32 queue) 1562 { 1563 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1564 int i; 1565 1566 for (i = 0; i < dma_conf->dma_rx_size; i++) 1567 stmmac_free_rx_buffer(priv, rx_q, i); 1568 } 1569 1570 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1571 struct stmmac_dma_conf *dma_conf, 1572 u32 queue, gfp_t flags) 1573 { 1574 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1575 int i; 1576 1577 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1578 struct dma_desc *p; 1579 int ret; 1580 1581 if (priv->extend_desc) 1582 p = &((rx_q->dma_erx + i)->basic); 1583 else 1584 p = rx_q->dma_rx + i; 1585 1586 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1587 queue); 1588 if (ret) 1589 return ret; 1590 1591 rx_q->buf_alloc_num++; 1592 } 1593 1594 return 0; 1595 } 1596 1597 /** 1598 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1599 * @priv: private structure 1600 * @dma_conf: structure to take the dma data 1601 * @queue: RX queue index 1602 */ 1603 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1604 struct stmmac_dma_conf *dma_conf, 1605 u32 queue) 1606 { 1607 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1608 int i; 1609 1610 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1611 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1612 1613 if (!buf->xdp) 1614 continue; 1615 1616 xsk_buff_free(buf->xdp); 1617 buf->xdp = NULL; 1618 } 1619 } 1620 1621 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1622 struct stmmac_dma_conf *dma_conf, 1623 u32 queue) 1624 { 1625 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1626 int i; 1627 1628 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1629 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1630 * use this macro to make sure no size violations. 1631 */ 1632 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1633 1634 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1635 struct stmmac_rx_buffer *buf; 1636 dma_addr_t dma_addr; 1637 struct dma_desc *p; 1638 1639 if (priv->extend_desc) 1640 p = (struct dma_desc *)(rx_q->dma_erx + i); 1641 else 1642 p = rx_q->dma_rx + i; 1643 1644 buf = &rx_q->buf_pool[i]; 1645 1646 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1647 if (!buf->xdp) 1648 return -ENOMEM; 1649 1650 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1651 stmmac_set_desc_addr(priv, p, dma_addr); 1652 rx_q->buf_alloc_num++; 1653 } 1654 1655 return 0; 1656 } 1657 1658 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1659 { 1660 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1661 return NULL; 1662 1663 return xsk_get_pool_from_qid(priv->dev, queue); 1664 } 1665 1666 /** 1667 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1668 * @priv: driver private structure 1669 * @dma_conf: structure to take the dma data 1670 * @queue: RX queue index 1671 * @flags: gfp flag. 1672 * Description: this function initializes the DMA RX descriptors 1673 * and allocates the socket buffers. It supports the chained and ring 1674 * modes. 1675 */ 1676 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1677 struct stmmac_dma_conf *dma_conf, 1678 u32 queue, gfp_t flags) 1679 { 1680 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1681 int ret; 1682 1683 netif_dbg(priv, probe, priv->dev, 1684 "(%s) dma_rx_phy=0x%08x\n", __func__, 1685 (u32)rx_q->dma_rx_phy); 1686 1687 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1688 1689 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1690 1691 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1692 1693 if (rx_q->xsk_pool) { 1694 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1695 MEM_TYPE_XSK_BUFF_POOL, 1696 NULL)); 1697 netdev_info(priv->dev, 1698 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1699 rx_q->queue_index); 1700 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1701 } else { 1702 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1703 MEM_TYPE_PAGE_POOL, 1704 rx_q->page_pool)); 1705 netdev_info(priv->dev, 1706 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1707 rx_q->queue_index); 1708 } 1709 1710 if (rx_q->xsk_pool) { 1711 /* RX XDP ZC buffer pool may not be populated, e.g. 1712 * xdpsock TX-only. 1713 */ 1714 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1715 } else { 1716 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1717 if (ret < 0) 1718 return -ENOMEM; 1719 } 1720 1721 /* Setup the chained descriptor addresses */ 1722 if (priv->mode == STMMAC_CHAIN_MODE) { 1723 if (priv->extend_desc) 1724 stmmac_mode_init(priv, rx_q->dma_erx, 1725 rx_q->dma_rx_phy, 1726 dma_conf->dma_rx_size, 1); 1727 else 1728 stmmac_mode_init(priv, rx_q->dma_rx, 1729 rx_q->dma_rx_phy, 1730 dma_conf->dma_rx_size, 0); 1731 } 1732 1733 return 0; 1734 } 1735 1736 static int init_dma_rx_desc_rings(struct net_device *dev, 1737 struct stmmac_dma_conf *dma_conf, 1738 gfp_t flags) 1739 { 1740 struct stmmac_priv *priv = netdev_priv(dev); 1741 u32 rx_count = priv->plat->rx_queues_to_use; 1742 int queue; 1743 int ret; 1744 1745 /* RX INITIALIZATION */ 1746 netif_dbg(priv, probe, priv->dev, 1747 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1748 1749 for (queue = 0; queue < rx_count; queue++) { 1750 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1751 if (ret) 1752 goto err_init_rx_buffers; 1753 } 1754 1755 return 0; 1756 1757 err_init_rx_buffers: 1758 while (queue >= 0) { 1759 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1760 1761 if (rx_q->xsk_pool) 1762 dma_free_rx_xskbufs(priv, dma_conf, queue); 1763 else 1764 dma_free_rx_skbufs(priv, dma_conf, queue); 1765 1766 rx_q->buf_alloc_num = 0; 1767 rx_q->xsk_pool = NULL; 1768 1769 queue--; 1770 } 1771 1772 return ret; 1773 } 1774 1775 /** 1776 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1777 * @priv: driver private structure 1778 * @dma_conf: structure to take the dma data 1779 * @queue: TX queue index 1780 * Description: this function initializes the DMA TX descriptors 1781 * and allocates the socket buffers. It supports the chained and ring 1782 * modes. 1783 */ 1784 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 1785 struct stmmac_dma_conf *dma_conf, 1786 u32 queue) 1787 { 1788 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1789 int i; 1790 1791 netif_dbg(priv, probe, priv->dev, 1792 "(%s) dma_tx_phy=0x%08x\n", __func__, 1793 (u32)tx_q->dma_tx_phy); 1794 1795 /* Setup the chained descriptor addresses */ 1796 if (priv->mode == STMMAC_CHAIN_MODE) { 1797 if (priv->extend_desc) 1798 stmmac_mode_init(priv, tx_q->dma_etx, 1799 tx_q->dma_tx_phy, 1800 dma_conf->dma_tx_size, 1); 1801 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1802 stmmac_mode_init(priv, tx_q->dma_tx, 1803 tx_q->dma_tx_phy, 1804 dma_conf->dma_tx_size, 0); 1805 } 1806 1807 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1808 1809 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1810 struct dma_desc *p; 1811 1812 if (priv->extend_desc) 1813 p = &((tx_q->dma_etx + i)->basic); 1814 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1815 p = &((tx_q->dma_entx + i)->basic); 1816 else 1817 p = tx_q->dma_tx + i; 1818 1819 stmmac_clear_desc(priv, p); 1820 1821 tx_q->tx_skbuff_dma[i].buf = 0; 1822 tx_q->tx_skbuff_dma[i].map_as_page = false; 1823 tx_q->tx_skbuff_dma[i].len = 0; 1824 tx_q->tx_skbuff_dma[i].last_segment = false; 1825 tx_q->tx_skbuff[i] = NULL; 1826 } 1827 1828 return 0; 1829 } 1830 1831 static int init_dma_tx_desc_rings(struct net_device *dev, 1832 struct stmmac_dma_conf *dma_conf) 1833 { 1834 struct stmmac_priv *priv = netdev_priv(dev); 1835 u32 tx_queue_cnt; 1836 u32 queue; 1837 1838 tx_queue_cnt = priv->plat->tx_queues_to_use; 1839 1840 for (queue = 0; queue < tx_queue_cnt; queue++) 1841 __init_dma_tx_desc_rings(priv, dma_conf, queue); 1842 1843 return 0; 1844 } 1845 1846 /** 1847 * init_dma_desc_rings - init the RX/TX descriptor rings 1848 * @dev: net device structure 1849 * @dma_conf: structure to take the dma data 1850 * @flags: gfp flag. 1851 * Description: this function initializes the DMA RX/TX descriptors 1852 * and allocates the socket buffers. It supports the chained and ring 1853 * modes. 1854 */ 1855 static int init_dma_desc_rings(struct net_device *dev, 1856 struct stmmac_dma_conf *dma_conf, 1857 gfp_t flags) 1858 { 1859 struct stmmac_priv *priv = netdev_priv(dev); 1860 int ret; 1861 1862 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 1863 if (ret) 1864 return ret; 1865 1866 ret = init_dma_tx_desc_rings(dev, dma_conf); 1867 1868 stmmac_clear_descriptors(priv, dma_conf); 1869 1870 if (netif_msg_hw(priv)) 1871 stmmac_display_rings(priv, dma_conf); 1872 1873 return ret; 1874 } 1875 1876 /** 1877 * dma_free_tx_skbufs - free TX dma buffers 1878 * @priv: private structure 1879 * @dma_conf: structure to take the dma data 1880 * @queue: TX queue index 1881 */ 1882 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 1883 struct stmmac_dma_conf *dma_conf, 1884 u32 queue) 1885 { 1886 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1887 int i; 1888 1889 tx_q->xsk_frames_done = 0; 1890 1891 for (i = 0; i < dma_conf->dma_tx_size; i++) 1892 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 1893 1894 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 1895 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 1896 tx_q->xsk_frames_done = 0; 1897 tx_q->xsk_pool = NULL; 1898 } 1899 } 1900 1901 /** 1902 * stmmac_free_tx_skbufs - free TX skb buffers 1903 * @priv: private structure 1904 */ 1905 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 1906 { 1907 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1908 u32 queue; 1909 1910 for (queue = 0; queue < tx_queue_cnt; queue++) 1911 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 1912 } 1913 1914 /** 1915 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 1916 * @priv: private structure 1917 * @dma_conf: structure to take the dma data 1918 * @queue: RX queue index 1919 */ 1920 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 1921 struct stmmac_dma_conf *dma_conf, 1922 u32 queue) 1923 { 1924 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1925 1926 /* Release the DMA RX socket buffers */ 1927 if (rx_q->xsk_pool) 1928 dma_free_rx_xskbufs(priv, dma_conf, queue); 1929 else 1930 dma_free_rx_skbufs(priv, dma_conf, queue); 1931 1932 rx_q->buf_alloc_num = 0; 1933 rx_q->xsk_pool = NULL; 1934 1935 /* Free DMA regions of consistent memory previously allocated */ 1936 if (!priv->extend_desc) 1937 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1938 sizeof(struct dma_desc), 1939 rx_q->dma_rx, rx_q->dma_rx_phy); 1940 else 1941 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1942 sizeof(struct dma_extended_desc), 1943 rx_q->dma_erx, rx_q->dma_rx_phy); 1944 1945 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 1946 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 1947 1948 kfree(rx_q->buf_pool); 1949 if (rx_q->page_pool) 1950 page_pool_destroy(rx_q->page_pool); 1951 } 1952 1953 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 1954 struct stmmac_dma_conf *dma_conf) 1955 { 1956 u32 rx_count = priv->plat->rx_queues_to_use; 1957 u32 queue; 1958 1959 /* Free RX queue resources */ 1960 for (queue = 0; queue < rx_count; queue++) 1961 __free_dma_rx_desc_resources(priv, dma_conf, queue); 1962 } 1963 1964 /** 1965 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 1966 * @priv: private structure 1967 * @dma_conf: structure to take the dma data 1968 * @queue: TX queue index 1969 */ 1970 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 1971 struct stmmac_dma_conf *dma_conf, 1972 u32 queue) 1973 { 1974 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1975 size_t size; 1976 void *addr; 1977 1978 /* Release the DMA TX socket buffers */ 1979 dma_free_tx_skbufs(priv, dma_conf, queue); 1980 1981 if (priv->extend_desc) { 1982 size = sizeof(struct dma_extended_desc); 1983 addr = tx_q->dma_etx; 1984 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1985 size = sizeof(struct dma_edesc); 1986 addr = tx_q->dma_entx; 1987 } else { 1988 size = sizeof(struct dma_desc); 1989 addr = tx_q->dma_tx; 1990 } 1991 1992 size *= dma_conf->dma_tx_size; 1993 1994 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 1995 1996 kfree(tx_q->tx_skbuff_dma); 1997 kfree(tx_q->tx_skbuff); 1998 } 1999 2000 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2001 struct stmmac_dma_conf *dma_conf) 2002 { 2003 u32 tx_count = priv->plat->tx_queues_to_use; 2004 u32 queue; 2005 2006 /* Free TX queue resources */ 2007 for (queue = 0; queue < tx_count; queue++) 2008 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2009 } 2010 2011 /** 2012 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2013 * @priv: private structure 2014 * @dma_conf: structure to take the dma data 2015 * @queue: RX queue index 2016 * Description: according to which descriptor can be used (extend or basic) 2017 * this function allocates the resources for TX and RX paths. In case of 2018 * reception, for example, it pre-allocated the RX socket buffer in order to 2019 * allow zero-copy mechanism. 2020 */ 2021 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2022 struct stmmac_dma_conf *dma_conf, 2023 u32 queue) 2024 { 2025 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2026 struct stmmac_channel *ch = &priv->channel[queue]; 2027 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2028 struct page_pool_params pp_params = { 0 }; 2029 unsigned int num_pages; 2030 unsigned int napi_id; 2031 int ret; 2032 2033 rx_q->queue_index = queue; 2034 rx_q->priv_data = priv; 2035 2036 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2037 pp_params.pool_size = dma_conf->dma_rx_size; 2038 num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); 2039 pp_params.order = ilog2(num_pages); 2040 pp_params.nid = dev_to_node(priv->device); 2041 pp_params.dev = priv->device; 2042 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2043 pp_params.offset = stmmac_rx_offset(priv); 2044 pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages); 2045 2046 rx_q->page_pool = page_pool_create(&pp_params); 2047 if (IS_ERR(rx_q->page_pool)) { 2048 ret = PTR_ERR(rx_q->page_pool); 2049 rx_q->page_pool = NULL; 2050 return ret; 2051 } 2052 2053 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, 2054 sizeof(*rx_q->buf_pool), 2055 GFP_KERNEL); 2056 if (!rx_q->buf_pool) 2057 return -ENOMEM; 2058 2059 if (priv->extend_desc) { 2060 rx_q->dma_erx = dma_alloc_coherent(priv->device, 2061 dma_conf->dma_rx_size * 2062 sizeof(struct dma_extended_desc), 2063 &rx_q->dma_rx_phy, 2064 GFP_KERNEL); 2065 if (!rx_q->dma_erx) 2066 return -ENOMEM; 2067 2068 } else { 2069 rx_q->dma_rx = dma_alloc_coherent(priv->device, 2070 dma_conf->dma_rx_size * 2071 sizeof(struct dma_desc), 2072 &rx_q->dma_rx_phy, 2073 GFP_KERNEL); 2074 if (!rx_q->dma_rx) 2075 return -ENOMEM; 2076 } 2077 2078 if (stmmac_xdp_is_enabled(priv) && 2079 test_bit(queue, priv->af_xdp_zc_qps)) 2080 napi_id = ch->rxtx_napi.napi_id; 2081 else 2082 napi_id = ch->rx_napi.napi_id; 2083 2084 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 2085 rx_q->queue_index, 2086 napi_id); 2087 if (ret) { 2088 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2089 return -EINVAL; 2090 } 2091 2092 return 0; 2093 } 2094 2095 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2096 struct stmmac_dma_conf *dma_conf) 2097 { 2098 u32 rx_count = priv->plat->rx_queues_to_use; 2099 u32 queue; 2100 int ret; 2101 2102 /* RX queues buffers and DMA */ 2103 for (queue = 0; queue < rx_count; queue++) { 2104 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2105 if (ret) 2106 goto err_dma; 2107 } 2108 2109 return 0; 2110 2111 err_dma: 2112 free_dma_rx_desc_resources(priv, dma_conf); 2113 2114 return ret; 2115 } 2116 2117 /** 2118 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2119 * @priv: private structure 2120 * @dma_conf: structure to take the dma data 2121 * @queue: TX queue index 2122 * Description: according to which descriptor can be used (extend or basic) 2123 * this function allocates the resources for TX and RX paths. In case of 2124 * reception, for example, it pre-allocated the RX socket buffer in order to 2125 * allow zero-copy mechanism. 2126 */ 2127 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2128 struct stmmac_dma_conf *dma_conf, 2129 u32 queue) 2130 { 2131 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2132 size_t size; 2133 void *addr; 2134 2135 tx_q->queue_index = queue; 2136 tx_q->priv_data = priv; 2137 2138 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, 2139 sizeof(*tx_q->tx_skbuff_dma), 2140 GFP_KERNEL); 2141 if (!tx_q->tx_skbuff_dma) 2142 return -ENOMEM; 2143 2144 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, 2145 sizeof(struct sk_buff *), 2146 GFP_KERNEL); 2147 if (!tx_q->tx_skbuff) 2148 return -ENOMEM; 2149 2150 if (priv->extend_desc) 2151 size = sizeof(struct dma_extended_desc); 2152 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2153 size = sizeof(struct dma_edesc); 2154 else 2155 size = sizeof(struct dma_desc); 2156 2157 size *= dma_conf->dma_tx_size; 2158 2159 addr = dma_alloc_coherent(priv->device, size, 2160 &tx_q->dma_tx_phy, GFP_KERNEL); 2161 if (!addr) 2162 return -ENOMEM; 2163 2164 if (priv->extend_desc) 2165 tx_q->dma_etx = addr; 2166 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2167 tx_q->dma_entx = addr; 2168 else 2169 tx_q->dma_tx = addr; 2170 2171 return 0; 2172 } 2173 2174 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2175 struct stmmac_dma_conf *dma_conf) 2176 { 2177 u32 tx_count = priv->plat->tx_queues_to_use; 2178 u32 queue; 2179 int ret; 2180 2181 /* TX queues buffers and DMA */ 2182 for (queue = 0; queue < tx_count; queue++) { 2183 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2184 if (ret) 2185 goto err_dma; 2186 } 2187 2188 return 0; 2189 2190 err_dma: 2191 free_dma_tx_desc_resources(priv, dma_conf); 2192 return ret; 2193 } 2194 2195 /** 2196 * alloc_dma_desc_resources - alloc TX/RX resources. 2197 * @priv: private structure 2198 * @dma_conf: structure to take the dma data 2199 * Description: according to which descriptor can be used (extend or basic) 2200 * this function allocates the resources for TX and RX paths. In case of 2201 * reception, for example, it pre-allocated the RX socket buffer in order to 2202 * allow zero-copy mechanism. 2203 */ 2204 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2205 struct stmmac_dma_conf *dma_conf) 2206 { 2207 /* RX Allocation */ 2208 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2209 2210 if (ret) 2211 return ret; 2212 2213 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2214 2215 return ret; 2216 } 2217 2218 /** 2219 * free_dma_desc_resources - free dma desc resources 2220 * @priv: private structure 2221 * @dma_conf: structure to take the dma data 2222 */ 2223 static void free_dma_desc_resources(struct stmmac_priv *priv, 2224 struct stmmac_dma_conf *dma_conf) 2225 { 2226 /* Release the DMA TX socket buffers */ 2227 free_dma_tx_desc_resources(priv, dma_conf); 2228 2229 /* Release the DMA RX socket buffers later 2230 * to ensure all pending XDP_TX buffers are returned. 2231 */ 2232 free_dma_rx_desc_resources(priv, dma_conf); 2233 } 2234 2235 /** 2236 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2237 * @priv: driver private structure 2238 * Description: It is used for enabling the rx queues in the MAC 2239 */ 2240 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2241 { 2242 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2243 int queue; 2244 u8 mode; 2245 2246 for (queue = 0; queue < rx_queues_count; queue++) { 2247 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2248 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2249 } 2250 } 2251 2252 /** 2253 * stmmac_start_rx_dma - start RX DMA channel 2254 * @priv: driver private structure 2255 * @chan: RX channel index 2256 * Description: 2257 * This starts a RX DMA channel 2258 */ 2259 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2260 { 2261 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2262 stmmac_start_rx(priv, priv->ioaddr, chan); 2263 } 2264 2265 /** 2266 * stmmac_start_tx_dma - start TX DMA channel 2267 * @priv: driver private structure 2268 * @chan: TX channel index 2269 * Description: 2270 * This starts a TX DMA channel 2271 */ 2272 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2273 { 2274 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2275 stmmac_start_tx(priv, priv->ioaddr, chan); 2276 } 2277 2278 /** 2279 * stmmac_stop_rx_dma - stop RX DMA channel 2280 * @priv: driver private structure 2281 * @chan: RX channel index 2282 * Description: 2283 * This stops a RX DMA channel 2284 */ 2285 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2286 { 2287 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2288 stmmac_stop_rx(priv, priv->ioaddr, chan); 2289 } 2290 2291 /** 2292 * stmmac_stop_tx_dma - stop TX DMA channel 2293 * @priv: driver private structure 2294 * @chan: TX channel index 2295 * Description: 2296 * This stops a TX DMA channel 2297 */ 2298 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2299 { 2300 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2301 stmmac_stop_tx(priv, priv->ioaddr, chan); 2302 } 2303 2304 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2305 { 2306 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2307 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2308 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2309 u32 chan; 2310 2311 for (chan = 0; chan < dma_csr_ch; chan++) { 2312 struct stmmac_channel *ch = &priv->channel[chan]; 2313 unsigned long flags; 2314 2315 spin_lock_irqsave(&ch->lock, flags); 2316 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2317 spin_unlock_irqrestore(&ch->lock, flags); 2318 } 2319 } 2320 2321 /** 2322 * stmmac_start_all_dma - start all RX and TX DMA channels 2323 * @priv: driver private structure 2324 * Description: 2325 * This starts all the RX and TX DMA channels 2326 */ 2327 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2328 { 2329 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2330 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2331 u32 chan = 0; 2332 2333 for (chan = 0; chan < rx_channels_count; chan++) 2334 stmmac_start_rx_dma(priv, chan); 2335 2336 for (chan = 0; chan < tx_channels_count; chan++) 2337 stmmac_start_tx_dma(priv, chan); 2338 } 2339 2340 /** 2341 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2342 * @priv: driver private structure 2343 * Description: 2344 * This stops the RX and TX DMA channels 2345 */ 2346 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2347 { 2348 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2349 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2350 u32 chan = 0; 2351 2352 for (chan = 0; chan < rx_channels_count; chan++) 2353 stmmac_stop_rx_dma(priv, chan); 2354 2355 for (chan = 0; chan < tx_channels_count; chan++) 2356 stmmac_stop_tx_dma(priv, chan); 2357 } 2358 2359 /** 2360 * stmmac_dma_operation_mode - HW DMA operation mode 2361 * @priv: driver private structure 2362 * Description: it is used for configuring the DMA operation mode register in 2363 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2364 */ 2365 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2366 { 2367 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2368 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2369 int rxfifosz = priv->plat->rx_fifo_size; 2370 int txfifosz = priv->plat->tx_fifo_size; 2371 u32 txmode = 0; 2372 u32 rxmode = 0; 2373 u32 chan = 0; 2374 u8 qmode = 0; 2375 2376 if (rxfifosz == 0) 2377 rxfifosz = priv->dma_cap.rx_fifo_size; 2378 if (txfifosz == 0) 2379 txfifosz = priv->dma_cap.tx_fifo_size; 2380 2381 /* Adjust for real per queue fifo size */ 2382 rxfifosz /= rx_channels_count; 2383 txfifosz /= tx_channels_count; 2384 2385 if (priv->plat->force_thresh_dma_mode) { 2386 txmode = tc; 2387 rxmode = tc; 2388 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2389 /* 2390 * In case of GMAC, SF mode can be enabled 2391 * to perform the TX COE in HW. This depends on: 2392 * 1) TX COE if actually supported 2393 * 2) There is no bugged Jumbo frame support 2394 * that needs to not insert csum in the TDES. 2395 */ 2396 txmode = SF_DMA_MODE; 2397 rxmode = SF_DMA_MODE; 2398 priv->xstats.threshold = SF_DMA_MODE; 2399 } else { 2400 txmode = tc; 2401 rxmode = SF_DMA_MODE; 2402 } 2403 2404 /* configure all channels */ 2405 for (chan = 0; chan < rx_channels_count; chan++) { 2406 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2407 u32 buf_size; 2408 2409 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2410 2411 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2412 rxfifosz, qmode); 2413 2414 if (rx_q->xsk_pool) { 2415 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2416 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2417 buf_size, 2418 chan); 2419 } else { 2420 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2421 priv->dma_conf.dma_buf_sz, 2422 chan); 2423 } 2424 } 2425 2426 for (chan = 0; chan < tx_channels_count; chan++) { 2427 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2428 2429 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2430 txfifosz, qmode); 2431 } 2432 } 2433 2434 static void stmmac_xsk_request_timestamp(void *_priv) 2435 { 2436 struct stmmac_metadata_request *meta_req = _priv; 2437 2438 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2439 *meta_req->set_ic = true; 2440 } 2441 2442 static u64 stmmac_xsk_fill_timestamp(void *_priv) 2443 { 2444 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2445 struct stmmac_priv *priv = tx_compl->priv; 2446 struct dma_desc *desc = tx_compl->desc; 2447 bool found = false; 2448 u64 ns = 0; 2449 2450 if (!priv->hwts_tx_en) 2451 return 0; 2452 2453 /* check tx tstamp status */ 2454 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2455 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2456 found = true; 2457 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2458 found = true; 2459 } 2460 2461 if (found) { 2462 ns -= priv->plat->cdc_error_adj; 2463 return ns_to_ktime(ns); 2464 } 2465 2466 return 0; 2467 } 2468 2469 static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2470 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2471 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2472 }; 2473 2474 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2475 { 2476 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2477 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2478 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2479 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2480 unsigned int entry = tx_q->cur_tx; 2481 struct dma_desc *tx_desc = NULL; 2482 struct xdp_desc xdp_desc; 2483 bool work_done = true; 2484 u32 tx_set_ic_bit = 0; 2485 unsigned long flags; 2486 2487 /* Avoids TX time-out as we are sharing with slow path */ 2488 txq_trans_cond_update(nq); 2489 2490 budget = min(budget, stmmac_tx_avail(priv, queue)); 2491 2492 while (budget-- > 0) { 2493 struct stmmac_metadata_request meta_req; 2494 struct xsk_tx_metadata *meta = NULL; 2495 dma_addr_t dma_addr; 2496 bool set_ic; 2497 2498 /* We are sharing with slow path and stop XSK TX desc submission when 2499 * available TX ring is less than threshold. 2500 */ 2501 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2502 !netif_carrier_ok(priv->dev)) { 2503 work_done = false; 2504 break; 2505 } 2506 2507 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2508 break; 2509 2510 if (likely(priv->extend_desc)) 2511 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2512 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2513 tx_desc = &tx_q->dma_entx[entry].basic; 2514 else 2515 tx_desc = tx_q->dma_tx + entry; 2516 2517 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2518 meta = xsk_buff_get_metadata(pool, xdp_desc.addr); 2519 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2520 2521 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2522 2523 /* To return XDP buffer to XSK pool, we simple call 2524 * xsk_tx_completed(), so we don't need to fill up 2525 * 'buf' and 'xdpf'. 2526 */ 2527 tx_q->tx_skbuff_dma[entry].buf = 0; 2528 tx_q->xdpf[entry] = NULL; 2529 2530 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2531 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2532 tx_q->tx_skbuff_dma[entry].last_segment = true; 2533 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2534 2535 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2536 2537 tx_q->tx_count_frames++; 2538 2539 if (!priv->tx_coal_frames[queue]) 2540 set_ic = false; 2541 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2542 set_ic = true; 2543 else 2544 set_ic = false; 2545 2546 meta_req.priv = priv; 2547 meta_req.tx_desc = tx_desc; 2548 meta_req.set_ic = &set_ic; 2549 xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops, 2550 &meta_req); 2551 if (set_ic) { 2552 tx_q->tx_count_frames = 0; 2553 stmmac_set_tx_ic(priv, tx_desc); 2554 tx_set_ic_bit++; 2555 } 2556 2557 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2558 true, priv->mode, true, true, 2559 xdp_desc.len); 2560 2561 stmmac_enable_dma_transmission(priv, priv->ioaddr); 2562 2563 xsk_tx_metadata_to_compl(meta, 2564 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2565 2566 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2567 entry = tx_q->cur_tx; 2568 } 2569 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 2570 txq_stats->tx_set_ic_bit += tx_set_ic_bit; 2571 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 2572 2573 if (tx_desc) { 2574 stmmac_flush_tx_descriptors(priv, queue); 2575 xsk_tx_release(pool); 2576 } 2577 2578 /* Return true if all of the 3 conditions are met 2579 * a) TX Budget is still available 2580 * b) work_done = true when XSK TX desc peek is empty (no more 2581 * pending XSK TX for transmission) 2582 */ 2583 return !!budget && work_done; 2584 } 2585 2586 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2587 { 2588 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2589 tc += 64; 2590 2591 if (priv->plat->force_thresh_dma_mode) 2592 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2593 else 2594 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2595 chan); 2596 2597 priv->xstats.threshold = tc; 2598 } 2599 } 2600 2601 /** 2602 * stmmac_tx_clean - to manage the transmission completion 2603 * @priv: driver private structure 2604 * @budget: napi budget limiting this functions packet handling 2605 * @queue: TX queue index 2606 * @pending_packets: signal to arm the TX coal timer 2607 * Description: it reclaims the transmit resources after transmission completes. 2608 * If some packets still needs to be handled, due to TX coalesce, set 2609 * pending_packets to true to make NAPI arm the TX coal timer. 2610 */ 2611 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2612 bool *pending_packets) 2613 { 2614 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2615 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2616 unsigned int bytes_compl = 0, pkts_compl = 0; 2617 unsigned int entry, xmits = 0, count = 0; 2618 u32 tx_packets = 0, tx_errors = 0; 2619 unsigned long flags; 2620 2621 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2622 2623 tx_q->xsk_frames_done = 0; 2624 2625 entry = tx_q->dirty_tx; 2626 2627 /* Try to clean all TX complete frame in 1 shot */ 2628 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2629 struct xdp_frame *xdpf; 2630 struct sk_buff *skb; 2631 struct dma_desc *p; 2632 int status; 2633 2634 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2635 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2636 xdpf = tx_q->xdpf[entry]; 2637 skb = NULL; 2638 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2639 xdpf = NULL; 2640 skb = tx_q->tx_skbuff[entry]; 2641 } else { 2642 xdpf = NULL; 2643 skb = NULL; 2644 } 2645 2646 if (priv->extend_desc) 2647 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2648 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2649 p = &tx_q->dma_entx[entry].basic; 2650 else 2651 p = tx_q->dma_tx + entry; 2652 2653 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2654 /* Check if the descriptor is owned by the DMA */ 2655 if (unlikely(status & tx_dma_own)) 2656 break; 2657 2658 count++; 2659 2660 /* Make sure descriptor fields are read after reading 2661 * the own bit. 2662 */ 2663 dma_rmb(); 2664 2665 /* Just consider the last segment and ...*/ 2666 if (likely(!(status & tx_not_ls))) { 2667 /* ... verify the status error condition */ 2668 if (unlikely(status & tx_err)) { 2669 tx_errors++; 2670 if (unlikely(status & tx_err_bump_tc)) 2671 stmmac_bump_dma_threshold(priv, queue); 2672 } else { 2673 tx_packets++; 2674 } 2675 if (skb) { 2676 stmmac_get_tx_hwtstamp(priv, p, skb); 2677 } else { 2678 struct stmmac_xsk_tx_complete tx_compl = { 2679 .priv = priv, 2680 .desc = p, 2681 }; 2682 2683 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2684 &stmmac_xsk_tx_metadata_ops, 2685 &tx_compl); 2686 } 2687 } 2688 2689 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2690 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2691 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2692 dma_unmap_page(priv->device, 2693 tx_q->tx_skbuff_dma[entry].buf, 2694 tx_q->tx_skbuff_dma[entry].len, 2695 DMA_TO_DEVICE); 2696 else 2697 dma_unmap_single(priv->device, 2698 tx_q->tx_skbuff_dma[entry].buf, 2699 tx_q->tx_skbuff_dma[entry].len, 2700 DMA_TO_DEVICE); 2701 tx_q->tx_skbuff_dma[entry].buf = 0; 2702 tx_q->tx_skbuff_dma[entry].len = 0; 2703 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2704 } 2705 2706 stmmac_clean_desc3(priv, tx_q, p); 2707 2708 tx_q->tx_skbuff_dma[entry].last_segment = false; 2709 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2710 2711 if (xdpf && 2712 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2713 xdp_return_frame_rx_napi(xdpf); 2714 tx_q->xdpf[entry] = NULL; 2715 } 2716 2717 if (xdpf && 2718 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2719 xdp_return_frame(xdpf); 2720 tx_q->xdpf[entry] = NULL; 2721 } 2722 2723 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2724 tx_q->xsk_frames_done++; 2725 2726 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2727 if (likely(skb)) { 2728 pkts_compl++; 2729 bytes_compl += skb->len; 2730 dev_consume_skb_any(skb); 2731 tx_q->tx_skbuff[entry] = NULL; 2732 } 2733 } 2734 2735 stmmac_release_tx_desc(priv, p, priv->mode); 2736 2737 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 2738 } 2739 tx_q->dirty_tx = entry; 2740 2741 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2742 pkts_compl, bytes_compl); 2743 2744 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2745 queue))) && 2746 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2747 2748 netif_dbg(priv, tx_done, priv->dev, 2749 "%s: restart transmit\n", __func__); 2750 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2751 } 2752 2753 if (tx_q->xsk_pool) { 2754 bool work_done; 2755 2756 if (tx_q->xsk_frames_done) 2757 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2758 2759 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2760 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2761 2762 /* For XSK TX, we try to send as many as possible. 2763 * If XSK work done (XSK TX desc empty and budget still 2764 * available), return "budget - 1" to reenable TX IRQ. 2765 * Else, return "budget" to make NAPI continue polling. 2766 */ 2767 work_done = stmmac_xdp_xmit_zc(priv, queue, 2768 STMMAC_XSK_TX_BUDGET_MAX); 2769 if (work_done) 2770 xmits = budget - 1; 2771 else 2772 xmits = budget; 2773 } 2774 2775 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && 2776 priv->eee_sw_timer_en) { 2777 if (stmmac_enable_eee_mode(priv)) 2778 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 2779 } 2780 2781 /* We still have pending packets, let's call for a new scheduling */ 2782 if (tx_q->dirty_tx != tx_q->cur_tx) 2783 *pending_packets = true; 2784 2785 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 2786 txq_stats->tx_packets += tx_packets; 2787 txq_stats->tx_pkt_n += tx_packets; 2788 txq_stats->tx_clean++; 2789 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 2790 2791 priv->xstats.tx_errors += tx_errors; 2792 2793 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2794 2795 /* Combine decisions from TX clean and XSK TX */ 2796 return max(count, xmits); 2797 } 2798 2799 /** 2800 * stmmac_tx_err - to manage the tx error 2801 * @priv: driver private structure 2802 * @chan: channel index 2803 * Description: it cleans the descriptors and restarts the transmission 2804 * in case of transmission errors. 2805 */ 2806 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2807 { 2808 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2809 2810 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2811 2812 stmmac_stop_tx_dma(priv, chan); 2813 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 2814 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 2815 stmmac_reset_tx_queue(priv, chan); 2816 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2817 tx_q->dma_tx_phy, chan); 2818 stmmac_start_tx_dma(priv, chan); 2819 2820 priv->xstats.tx_errors++; 2821 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2822 } 2823 2824 /** 2825 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2826 * @priv: driver private structure 2827 * @txmode: TX operating mode 2828 * @rxmode: RX operating mode 2829 * @chan: channel index 2830 * Description: it is used for configuring of the DMA operation mode in 2831 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2832 * mode. 2833 */ 2834 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2835 u32 rxmode, u32 chan) 2836 { 2837 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2838 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2839 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2840 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2841 int rxfifosz = priv->plat->rx_fifo_size; 2842 int txfifosz = priv->plat->tx_fifo_size; 2843 2844 if (rxfifosz == 0) 2845 rxfifosz = priv->dma_cap.rx_fifo_size; 2846 if (txfifosz == 0) 2847 txfifosz = priv->dma_cap.tx_fifo_size; 2848 2849 /* Adjust for real per queue fifo size */ 2850 rxfifosz /= rx_channels_count; 2851 txfifosz /= tx_channels_count; 2852 2853 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2854 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2855 } 2856 2857 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2858 { 2859 int ret; 2860 2861 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2862 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2863 if (ret && (ret != -EINVAL)) { 2864 stmmac_global_err(priv); 2865 return true; 2866 } 2867 2868 return false; 2869 } 2870 2871 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 2872 { 2873 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2874 &priv->xstats, chan, dir); 2875 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2876 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2877 struct stmmac_channel *ch = &priv->channel[chan]; 2878 struct napi_struct *rx_napi; 2879 struct napi_struct *tx_napi; 2880 unsigned long flags; 2881 2882 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 2883 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2884 2885 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 2886 if (napi_schedule_prep(rx_napi)) { 2887 spin_lock_irqsave(&ch->lock, flags); 2888 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 2889 spin_unlock_irqrestore(&ch->lock, flags); 2890 __napi_schedule(rx_napi); 2891 } 2892 } 2893 2894 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 2895 if (napi_schedule_prep(tx_napi)) { 2896 spin_lock_irqsave(&ch->lock, flags); 2897 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 2898 spin_unlock_irqrestore(&ch->lock, flags); 2899 __napi_schedule(tx_napi); 2900 } 2901 } 2902 2903 return status; 2904 } 2905 2906 /** 2907 * stmmac_dma_interrupt - DMA ISR 2908 * @priv: driver private structure 2909 * Description: this is the DMA ISR. It is called by the main ISR. 2910 * It calls the dwmac dma routine and schedule poll method in case of some 2911 * work can be done. 2912 */ 2913 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2914 { 2915 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2916 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2917 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2918 tx_channel_count : rx_channel_count; 2919 u32 chan; 2920 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2921 2922 /* Make sure we never check beyond our status buffer. */ 2923 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2924 channels_to_check = ARRAY_SIZE(status); 2925 2926 for (chan = 0; chan < channels_to_check; chan++) 2927 status[chan] = stmmac_napi_check(priv, chan, 2928 DMA_DIR_RXTX); 2929 2930 for (chan = 0; chan < tx_channel_count; chan++) { 2931 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2932 /* Try to bump up the dma threshold on this failure */ 2933 stmmac_bump_dma_threshold(priv, chan); 2934 } else if (unlikely(status[chan] == tx_hard_error)) { 2935 stmmac_tx_err(priv, chan); 2936 } 2937 } 2938 } 2939 2940 /** 2941 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2942 * @priv: driver private structure 2943 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2944 */ 2945 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2946 { 2947 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2948 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2949 2950 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 2951 2952 if (priv->dma_cap.rmon) { 2953 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 2954 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2955 } else 2956 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2957 } 2958 2959 /** 2960 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2961 * @priv: driver private structure 2962 * Description: 2963 * new GMAC chip generations have a new register to indicate the 2964 * presence of the optional feature/functions. 2965 * This can be also used to override the value passed through the 2966 * platform and necessary for old MAC10/100 and GMAC chips. 2967 */ 2968 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2969 { 2970 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2971 } 2972 2973 /** 2974 * stmmac_check_ether_addr - check if the MAC addr is valid 2975 * @priv: driver private structure 2976 * Description: 2977 * it is to verify if the MAC address is valid, in case of failures it 2978 * generates a random MAC address 2979 */ 2980 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2981 { 2982 u8 addr[ETH_ALEN]; 2983 2984 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2985 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 2986 if (is_valid_ether_addr(addr)) 2987 eth_hw_addr_set(priv->dev, addr); 2988 else 2989 eth_hw_addr_random(priv->dev); 2990 dev_info(priv->device, "device MAC address %pM\n", 2991 priv->dev->dev_addr); 2992 } 2993 } 2994 2995 /** 2996 * stmmac_init_dma_engine - DMA init. 2997 * @priv: driver private structure 2998 * Description: 2999 * It inits the DMA invoking the specific MAC/GMAC callback. 3000 * Some DMA parameters can be passed from the platform; 3001 * in case of these are not passed a default is kept for the MAC or GMAC. 3002 */ 3003 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 3004 { 3005 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3006 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3007 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 3008 struct stmmac_rx_queue *rx_q; 3009 struct stmmac_tx_queue *tx_q; 3010 u32 chan = 0; 3011 int atds = 0; 3012 int ret = 0; 3013 3014 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 3015 dev_err(priv->device, "Invalid DMA configuration\n"); 3016 return -EINVAL; 3017 } 3018 3019 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 3020 atds = 1; 3021 3022 ret = stmmac_reset(priv, priv->ioaddr); 3023 if (ret) { 3024 dev_err(priv->device, "Failed to reset the dma\n"); 3025 return ret; 3026 } 3027 3028 /* DMA Configuration */ 3029 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); 3030 3031 if (priv->plat->axi) 3032 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3033 3034 /* DMA CSR Channel configuration */ 3035 for (chan = 0; chan < dma_csr_ch; chan++) { 3036 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3037 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3038 } 3039 3040 /* DMA RX Channel Configuration */ 3041 for (chan = 0; chan < rx_channels_count; chan++) { 3042 rx_q = &priv->dma_conf.rx_queue[chan]; 3043 3044 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3045 rx_q->dma_rx_phy, chan); 3046 3047 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 3048 (rx_q->buf_alloc_num * 3049 sizeof(struct dma_desc)); 3050 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 3051 rx_q->rx_tail_addr, chan); 3052 } 3053 3054 /* DMA TX Channel Configuration */ 3055 for (chan = 0; chan < tx_channels_count; chan++) { 3056 tx_q = &priv->dma_conf.tx_queue[chan]; 3057 3058 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3059 tx_q->dma_tx_phy, chan); 3060 3061 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 3062 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 3063 tx_q->tx_tail_addr, chan); 3064 } 3065 3066 return ret; 3067 } 3068 3069 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3070 { 3071 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3072 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3073 struct stmmac_channel *ch; 3074 struct napi_struct *napi; 3075 3076 if (!tx_coal_timer) 3077 return; 3078 3079 ch = &priv->channel[tx_q->queue_index]; 3080 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3081 3082 /* Arm timer only if napi is not already scheduled. 3083 * Try to cancel any timer if napi is scheduled, timer will be armed 3084 * again in the next scheduled napi. 3085 */ 3086 if (unlikely(!napi_is_scheduled(napi))) 3087 hrtimer_start(&tx_q->txtimer, 3088 STMMAC_COAL_TIMER(tx_coal_timer), 3089 HRTIMER_MODE_REL); 3090 else 3091 hrtimer_try_to_cancel(&tx_q->txtimer); 3092 } 3093 3094 /** 3095 * stmmac_tx_timer - mitigation sw timer for tx. 3096 * @t: data pointer 3097 * Description: 3098 * This is the timer handler to directly invoke the stmmac_tx_clean. 3099 */ 3100 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3101 { 3102 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3103 struct stmmac_priv *priv = tx_q->priv_data; 3104 struct stmmac_channel *ch; 3105 struct napi_struct *napi; 3106 3107 ch = &priv->channel[tx_q->queue_index]; 3108 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3109 3110 if (likely(napi_schedule_prep(napi))) { 3111 unsigned long flags; 3112 3113 spin_lock_irqsave(&ch->lock, flags); 3114 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3115 spin_unlock_irqrestore(&ch->lock, flags); 3116 __napi_schedule(napi); 3117 } 3118 3119 return HRTIMER_NORESTART; 3120 } 3121 3122 /** 3123 * stmmac_init_coalesce - init mitigation options. 3124 * @priv: driver private structure 3125 * Description: 3126 * This inits the coalesce parameters: i.e. timer rate, 3127 * timer handler and default threshold used for enabling the 3128 * interrupt on completion bit. 3129 */ 3130 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3131 { 3132 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3133 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3134 u32 chan; 3135 3136 for (chan = 0; chan < tx_channel_count; chan++) { 3137 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3138 3139 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3140 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3141 3142 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3143 tx_q->txtimer.function = stmmac_tx_timer; 3144 } 3145 3146 for (chan = 0; chan < rx_channel_count; chan++) 3147 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3148 } 3149 3150 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3151 { 3152 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3153 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3154 u32 chan; 3155 3156 /* set TX ring length */ 3157 for (chan = 0; chan < tx_channels_count; chan++) 3158 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3159 (priv->dma_conf.dma_tx_size - 1), chan); 3160 3161 /* set RX ring length */ 3162 for (chan = 0; chan < rx_channels_count; chan++) 3163 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3164 (priv->dma_conf.dma_rx_size - 1), chan); 3165 } 3166 3167 /** 3168 * stmmac_set_tx_queue_weight - Set TX queue weight 3169 * @priv: driver private structure 3170 * Description: It is used for setting TX queues weight 3171 */ 3172 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3173 { 3174 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3175 u32 weight; 3176 u32 queue; 3177 3178 for (queue = 0; queue < tx_queues_count; queue++) { 3179 weight = priv->plat->tx_queues_cfg[queue].weight; 3180 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3181 } 3182 } 3183 3184 /** 3185 * stmmac_configure_cbs - Configure CBS in TX queue 3186 * @priv: driver private structure 3187 * Description: It is used for configuring CBS in AVB TX queues 3188 */ 3189 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3190 { 3191 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3192 u32 mode_to_use; 3193 u32 queue; 3194 3195 /* queue 0 is reserved for legacy traffic */ 3196 for (queue = 1; queue < tx_queues_count; queue++) { 3197 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3198 if (mode_to_use == MTL_QUEUE_DCB) 3199 continue; 3200 3201 stmmac_config_cbs(priv, priv->hw, 3202 priv->plat->tx_queues_cfg[queue].send_slope, 3203 priv->plat->tx_queues_cfg[queue].idle_slope, 3204 priv->plat->tx_queues_cfg[queue].high_credit, 3205 priv->plat->tx_queues_cfg[queue].low_credit, 3206 queue); 3207 } 3208 } 3209 3210 /** 3211 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3212 * @priv: driver private structure 3213 * Description: It is used for mapping RX queues to RX dma channels 3214 */ 3215 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3216 { 3217 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3218 u32 queue; 3219 u32 chan; 3220 3221 for (queue = 0; queue < rx_queues_count; queue++) { 3222 chan = priv->plat->rx_queues_cfg[queue].chan; 3223 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3224 } 3225 } 3226 3227 /** 3228 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3229 * @priv: driver private structure 3230 * Description: It is used for configuring the RX Queue Priority 3231 */ 3232 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3233 { 3234 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3235 u32 queue; 3236 u32 prio; 3237 3238 for (queue = 0; queue < rx_queues_count; queue++) { 3239 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3240 continue; 3241 3242 prio = priv->plat->rx_queues_cfg[queue].prio; 3243 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3244 } 3245 } 3246 3247 /** 3248 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3249 * @priv: driver private structure 3250 * Description: It is used for configuring the TX Queue Priority 3251 */ 3252 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3253 { 3254 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3255 u32 queue; 3256 u32 prio; 3257 3258 for (queue = 0; queue < tx_queues_count; queue++) { 3259 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3260 continue; 3261 3262 prio = priv->plat->tx_queues_cfg[queue].prio; 3263 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3264 } 3265 } 3266 3267 /** 3268 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3269 * @priv: driver private structure 3270 * Description: It is used for configuring the RX queue routing 3271 */ 3272 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3273 { 3274 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3275 u32 queue; 3276 u8 packet; 3277 3278 for (queue = 0; queue < rx_queues_count; queue++) { 3279 /* no specific packet type routing specified for the queue */ 3280 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3281 continue; 3282 3283 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3284 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3285 } 3286 } 3287 3288 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3289 { 3290 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3291 priv->rss.enable = false; 3292 return; 3293 } 3294 3295 if (priv->dev->features & NETIF_F_RXHASH) 3296 priv->rss.enable = true; 3297 else 3298 priv->rss.enable = false; 3299 3300 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3301 priv->plat->rx_queues_to_use); 3302 } 3303 3304 /** 3305 * stmmac_mtl_configuration - Configure MTL 3306 * @priv: driver private structure 3307 * Description: It is used for configurring MTL 3308 */ 3309 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3310 { 3311 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3312 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3313 3314 if (tx_queues_count > 1) 3315 stmmac_set_tx_queue_weight(priv); 3316 3317 /* Configure MTL RX algorithms */ 3318 if (rx_queues_count > 1) 3319 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3320 priv->plat->rx_sched_algorithm); 3321 3322 /* Configure MTL TX algorithms */ 3323 if (tx_queues_count > 1) 3324 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3325 priv->plat->tx_sched_algorithm); 3326 3327 /* Configure CBS in AVB TX queues */ 3328 if (tx_queues_count > 1) 3329 stmmac_configure_cbs(priv); 3330 3331 /* Map RX MTL to DMA channels */ 3332 stmmac_rx_queue_dma_chan_map(priv); 3333 3334 /* Enable MAC RX Queues */ 3335 stmmac_mac_enable_rx_queues(priv); 3336 3337 /* Set RX priorities */ 3338 if (rx_queues_count > 1) 3339 stmmac_mac_config_rx_queues_prio(priv); 3340 3341 /* Set TX priorities */ 3342 if (tx_queues_count > 1) 3343 stmmac_mac_config_tx_queues_prio(priv); 3344 3345 /* Set RX routing */ 3346 if (rx_queues_count > 1) 3347 stmmac_mac_config_rx_queues_routing(priv); 3348 3349 /* Receive Side Scaling */ 3350 if (rx_queues_count > 1) 3351 stmmac_mac_config_rss(priv); 3352 } 3353 3354 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3355 { 3356 if (priv->dma_cap.asp) { 3357 netdev_info(priv->dev, "Enabling Safety Features\n"); 3358 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3359 priv->plat->safety_feat_cfg); 3360 } else { 3361 netdev_info(priv->dev, "No Safety Features support found\n"); 3362 } 3363 } 3364 3365 static int stmmac_fpe_start_wq(struct stmmac_priv *priv) 3366 { 3367 char *name; 3368 3369 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 3370 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); 3371 3372 name = priv->wq_name; 3373 sprintf(name, "%s-fpe", priv->dev->name); 3374 3375 priv->fpe_wq = create_singlethread_workqueue(name); 3376 if (!priv->fpe_wq) { 3377 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); 3378 3379 return -ENOMEM; 3380 } 3381 netdev_info(priv->dev, "FPE workqueue start"); 3382 3383 return 0; 3384 } 3385 3386 /** 3387 * stmmac_hw_setup - setup mac in a usable state. 3388 * @dev : pointer to the device structure. 3389 * @ptp_register: register PTP if set 3390 * Description: 3391 * this is the main function to setup the HW in a usable state because the 3392 * dma engine is reset, the core registers are configured (e.g. AXI, 3393 * Checksum features, timers). The DMA is ready to start receiving and 3394 * transmitting. 3395 * Return value: 3396 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3397 * file on failure. 3398 */ 3399 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) 3400 { 3401 struct stmmac_priv *priv = netdev_priv(dev); 3402 u32 rx_cnt = priv->plat->rx_queues_to_use; 3403 u32 tx_cnt = priv->plat->tx_queues_to_use; 3404 bool sph_en; 3405 u32 chan; 3406 int ret; 3407 3408 /* DMA initialization and SW reset */ 3409 ret = stmmac_init_dma_engine(priv); 3410 if (ret < 0) { 3411 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3412 __func__); 3413 return ret; 3414 } 3415 3416 /* Copy the MAC addr into the HW */ 3417 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3418 3419 /* PS and related bits will be programmed according to the speed */ 3420 if (priv->hw->pcs) { 3421 int speed = priv->plat->mac_port_sel_speed; 3422 3423 if ((speed == SPEED_10) || (speed == SPEED_100) || 3424 (speed == SPEED_1000)) { 3425 priv->hw->ps = speed; 3426 } else { 3427 dev_warn(priv->device, "invalid port speed\n"); 3428 priv->hw->ps = 0; 3429 } 3430 } 3431 3432 /* Initialize the MAC Core */ 3433 stmmac_core_init(priv, priv->hw, dev); 3434 3435 /* Initialize MTL*/ 3436 stmmac_mtl_configuration(priv); 3437 3438 /* Initialize Safety Features */ 3439 stmmac_safety_feat_configuration(priv); 3440 3441 ret = stmmac_rx_ipc(priv, priv->hw); 3442 if (!ret) { 3443 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3444 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3445 priv->hw->rx_csum = 0; 3446 } 3447 3448 /* Enable the MAC Rx/Tx */ 3449 stmmac_mac_set(priv, priv->ioaddr, true); 3450 3451 /* Set the HW DMA mode and the COE */ 3452 stmmac_dma_operation_mode(priv); 3453 3454 stmmac_mmc_setup(priv); 3455 3456 if (ptp_register) { 3457 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 3458 if (ret < 0) 3459 netdev_warn(priv->dev, 3460 "failed to enable PTP reference clock: %pe\n", 3461 ERR_PTR(ret)); 3462 } 3463 3464 ret = stmmac_init_ptp(priv); 3465 if (ret == -EOPNOTSUPP) 3466 netdev_info(priv->dev, "PTP not supported by HW\n"); 3467 else if (ret) 3468 netdev_warn(priv->dev, "PTP init failed\n"); 3469 else if (ptp_register) 3470 stmmac_ptp_register(priv); 3471 3472 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; 3473 3474 /* Convert the timer from msec to usec */ 3475 if (!priv->tx_lpi_timer) 3476 priv->tx_lpi_timer = eee_timer * 1000; 3477 3478 if (priv->use_riwt) { 3479 u32 queue; 3480 3481 for (queue = 0; queue < rx_cnt; queue++) { 3482 if (!priv->rx_riwt[queue]) 3483 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3484 3485 stmmac_rx_watchdog(priv, priv->ioaddr, 3486 priv->rx_riwt[queue], queue); 3487 } 3488 } 3489 3490 if (priv->hw->pcs) 3491 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); 3492 3493 /* set TX and RX rings length */ 3494 stmmac_set_rings_length(priv); 3495 3496 /* Enable TSO */ 3497 if (priv->tso) { 3498 for (chan = 0; chan < tx_cnt; chan++) { 3499 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3500 3501 /* TSO and TBS cannot co-exist */ 3502 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3503 continue; 3504 3505 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3506 } 3507 } 3508 3509 /* Enable Split Header */ 3510 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 3511 for (chan = 0; chan < rx_cnt; chan++) 3512 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3513 3514 3515 /* VLAN Tag Insertion */ 3516 if (priv->dma_cap.vlins) 3517 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3518 3519 /* TBS */ 3520 for (chan = 0; chan < tx_cnt; chan++) { 3521 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3522 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3523 3524 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3525 } 3526 3527 /* Configure real RX and TX queues */ 3528 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3529 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3530 3531 /* Start the ball rolling... */ 3532 stmmac_start_all_dma(priv); 3533 3534 stmmac_set_hw_vlan_mode(priv, priv->hw); 3535 3536 if (priv->dma_cap.fpesel) { 3537 stmmac_fpe_start_wq(priv); 3538 3539 if (priv->plat->fpe_cfg->enable) 3540 stmmac_fpe_handshake(priv, true); 3541 } 3542 3543 return 0; 3544 } 3545 3546 static void stmmac_hw_teardown(struct net_device *dev) 3547 { 3548 struct stmmac_priv *priv = netdev_priv(dev); 3549 3550 clk_disable_unprepare(priv->plat->clk_ptp_ref); 3551 } 3552 3553 static void stmmac_free_irq(struct net_device *dev, 3554 enum request_irq_err irq_err, int irq_idx) 3555 { 3556 struct stmmac_priv *priv = netdev_priv(dev); 3557 int j; 3558 3559 switch (irq_err) { 3560 case REQ_IRQ_ERR_ALL: 3561 irq_idx = priv->plat->tx_queues_to_use; 3562 fallthrough; 3563 case REQ_IRQ_ERR_TX: 3564 for (j = irq_idx - 1; j >= 0; j--) { 3565 if (priv->tx_irq[j] > 0) { 3566 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3567 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); 3568 } 3569 } 3570 irq_idx = priv->plat->rx_queues_to_use; 3571 fallthrough; 3572 case REQ_IRQ_ERR_RX: 3573 for (j = irq_idx - 1; j >= 0; j--) { 3574 if (priv->rx_irq[j] > 0) { 3575 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3576 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); 3577 } 3578 } 3579 3580 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3581 free_irq(priv->sfty_ue_irq, dev); 3582 fallthrough; 3583 case REQ_IRQ_ERR_SFTY_UE: 3584 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3585 free_irq(priv->sfty_ce_irq, dev); 3586 fallthrough; 3587 case REQ_IRQ_ERR_SFTY_CE: 3588 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) 3589 free_irq(priv->lpi_irq, dev); 3590 fallthrough; 3591 case REQ_IRQ_ERR_LPI: 3592 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3593 free_irq(priv->wol_irq, dev); 3594 fallthrough; 3595 case REQ_IRQ_ERR_WOL: 3596 free_irq(dev->irq, dev); 3597 fallthrough; 3598 case REQ_IRQ_ERR_MAC: 3599 case REQ_IRQ_ERR_NO: 3600 /* If MAC IRQ request error, no more IRQ to free */ 3601 break; 3602 } 3603 } 3604 3605 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3606 { 3607 struct stmmac_priv *priv = netdev_priv(dev); 3608 enum request_irq_err irq_err; 3609 cpumask_t cpu_mask; 3610 int irq_idx = 0; 3611 char *int_name; 3612 int ret; 3613 int i; 3614 3615 /* For common interrupt */ 3616 int_name = priv->int_name_mac; 3617 sprintf(int_name, "%s:%s", dev->name, "mac"); 3618 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3619 0, int_name, dev); 3620 if (unlikely(ret < 0)) { 3621 netdev_err(priv->dev, 3622 "%s: alloc mac MSI %d (error: %d)\n", 3623 __func__, dev->irq, ret); 3624 irq_err = REQ_IRQ_ERR_MAC; 3625 goto irq_error; 3626 } 3627 3628 /* Request the Wake IRQ in case of another line 3629 * is used for WoL 3630 */ 3631 priv->wol_irq_disabled = true; 3632 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3633 int_name = priv->int_name_wol; 3634 sprintf(int_name, "%s:%s", dev->name, "wol"); 3635 ret = request_irq(priv->wol_irq, 3636 stmmac_mac_interrupt, 3637 0, int_name, dev); 3638 if (unlikely(ret < 0)) { 3639 netdev_err(priv->dev, 3640 "%s: alloc wol MSI %d (error: %d)\n", 3641 __func__, priv->wol_irq, ret); 3642 irq_err = REQ_IRQ_ERR_WOL; 3643 goto irq_error; 3644 } 3645 } 3646 3647 /* Request the LPI IRQ in case of another line 3648 * is used for LPI 3649 */ 3650 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3651 int_name = priv->int_name_lpi; 3652 sprintf(int_name, "%s:%s", dev->name, "lpi"); 3653 ret = request_irq(priv->lpi_irq, 3654 stmmac_mac_interrupt, 3655 0, int_name, dev); 3656 if (unlikely(ret < 0)) { 3657 netdev_err(priv->dev, 3658 "%s: alloc lpi MSI %d (error: %d)\n", 3659 __func__, priv->lpi_irq, ret); 3660 irq_err = REQ_IRQ_ERR_LPI; 3661 goto irq_error; 3662 } 3663 } 3664 3665 /* Request the Safety Feature Correctible Error line in 3666 * case of another line is used 3667 */ 3668 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3669 int_name = priv->int_name_sfty_ce; 3670 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3671 ret = request_irq(priv->sfty_ce_irq, 3672 stmmac_safety_interrupt, 3673 0, int_name, dev); 3674 if (unlikely(ret < 0)) { 3675 netdev_err(priv->dev, 3676 "%s: alloc sfty ce MSI %d (error: %d)\n", 3677 __func__, priv->sfty_ce_irq, ret); 3678 irq_err = REQ_IRQ_ERR_SFTY_CE; 3679 goto irq_error; 3680 } 3681 } 3682 3683 /* Request the Safety Feature Uncorrectible Error line in 3684 * case of another line is used 3685 */ 3686 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3687 int_name = priv->int_name_sfty_ue; 3688 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3689 ret = request_irq(priv->sfty_ue_irq, 3690 stmmac_safety_interrupt, 3691 0, int_name, dev); 3692 if (unlikely(ret < 0)) { 3693 netdev_err(priv->dev, 3694 "%s: alloc sfty ue MSI %d (error: %d)\n", 3695 __func__, priv->sfty_ue_irq, ret); 3696 irq_err = REQ_IRQ_ERR_SFTY_UE; 3697 goto irq_error; 3698 } 3699 } 3700 3701 /* Request Rx MSI irq */ 3702 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3703 if (i >= MTL_MAX_RX_QUEUES) 3704 break; 3705 if (priv->rx_irq[i] == 0) 3706 continue; 3707 3708 int_name = priv->int_name_rx_irq[i]; 3709 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3710 ret = request_irq(priv->rx_irq[i], 3711 stmmac_msi_intr_rx, 3712 0, int_name, &priv->dma_conf.rx_queue[i]); 3713 if (unlikely(ret < 0)) { 3714 netdev_err(priv->dev, 3715 "%s: alloc rx-%d MSI %d (error: %d)\n", 3716 __func__, i, priv->rx_irq[i], ret); 3717 irq_err = REQ_IRQ_ERR_RX; 3718 irq_idx = i; 3719 goto irq_error; 3720 } 3721 cpumask_clear(&cpu_mask); 3722 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3723 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); 3724 } 3725 3726 /* Request Tx MSI irq */ 3727 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3728 if (i >= MTL_MAX_TX_QUEUES) 3729 break; 3730 if (priv->tx_irq[i] == 0) 3731 continue; 3732 3733 int_name = priv->int_name_tx_irq[i]; 3734 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3735 ret = request_irq(priv->tx_irq[i], 3736 stmmac_msi_intr_tx, 3737 0, int_name, &priv->dma_conf.tx_queue[i]); 3738 if (unlikely(ret < 0)) { 3739 netdev_err(priv->dev, 3740 "%s: alloc tx-%d MSI %d (error: %d)\n", 3741 __func__, i, priv->tx_irq[i], ret); 3742 irq_err = REQ_IRQ_ERR_TX; 3743 irq_idx = i; 3744 goto irq_error; 3745 } 3746 cpumask_clear(&cpu_mask); 3747 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3748 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); 3749 } 3750 3751 return 0; 3752 3753 irq_error: 3754 stmmac_free_irq(dev, irq_err, irq_idx); 3755 return ret; 3756 } 3757 3758 static int stmmac_request_irq_single(struct net_device *dev) 3759 { 3760 struct stmmac_priv *priv = netdev_priv(dev); 3761 enum request_irq_err irq_err; 3762 int ret; 3763 3764 ret = request_irq(dev->irq, stmmac_interrupt, 3765 IRQF_SHARED, dev->name, dev); 3766 if (unlikely(ret < 0)) { 3767 netdev_err(priv->dev, 3768 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3769 __func__, dev->irq, ret); 3770 irq_err = REQ_IRQ_ERR_MAC; 3771 goto irq_error; 3772 } 3773 3774 /* Request the Wake IRQ in case of another line 3775 * is used for WoL 3776 */ 3777 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3778 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3779 IRQF_SHARED, dev->name, dev); 3780 if (unlikely(ret < 0)) { 3781 netdev_err(priv->dev, 3782 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3783 __func__, priv->wol_irq, ret); 3784 irq_err = REQ_IRQ_ERR_WOL; 3785 goto irq_error; 3786 } 3787 } 3788 3789 /* Request the IRQ lines */ 3790 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3791 ret = request_irq(priv->lpi_irq, stmmac_interrupt, 3792 IRQF_SHARED, dev->name, dev); 3793 if (unlikely(ret < 0)) { 3794 netdev_err(priv->dev, 3795 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 3796 __func__, priv->lpi_irq, ret); 3797 irq_err = REQ_IRQ_ERR_LPI; 3798 goto irq_error; 3799 } 3800 } 3801 3802 return 0; 3803 3804 irq_error: 3805 stmmac_free_irq(dev, irq_err, 0); 3806 return ret; 3807 } 3808 3809 static int stmmac_request_irq(struct net_device *dev) 3810 { 3811 struct stmmac_priv *priv = netdev_priv(dev); 3812 int ret; 3813 3814 /* Request the IRQ lines */ 3815 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 3816 ret = stmmac_request_irq_multi_msi(dev); 3817 else 3818 ret = stmmac_request_irq_single(dev); 3819 3820 return ret; 3821 } 3822 3823 /** 3824 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 3825 * @priv: driver private structure 3826 * @mtu: MTU to setup the dma queue and buf with 3827 * Description: Allocate and generate a dma_conf based on the provided MTU. 3828 * Allocate the Tx/Rx DMA queue and init them. 3829 * Return value: 3830 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 3831 */ 3832 static struct stmmac_dma_conf * 3833 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 3834 { 3835 struct stmmac_dma_conf *dma_conf; 3836 int chan, bfsize, ret; 3837 3838 dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); 3839 if (!dma_conf) { 3840 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 3841 __func__); 3842 return ERR_PTR(-ENOMEM); 3843 } 3844 3845 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 3846 if (bfsize < 0) 3847 bfsize = 0; 3848 3849 if (bfsize < BUF_SIZE_16KiB) 3850 bfsize = stmmac_set_bfsize(mtu, 0); 3851 3852 dma_conf->dma_buf_sz = bfsize; 3853 /* Chose the tx/rx size from the already defined one in the 3854 * priv struct. (if defined) 3855 */ 3856 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 3857 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 3858 3859 if (!dma_conf->dma_tx_size) 3860 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3861 if (!dma_conf->dma_rx_size) 3862 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3863 3864 /* Earlier check for TBS */ 3865 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3866 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 3867 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3868 3869 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3870 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3871 } 3872 3873 ret = alloc_dma_desc_resources(priv, dma_conf); 3874 if (ret < 0) { 3875 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 3876 __func__); 3877 goto alloc_error; 3878 } 3879 3880 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 3881 if (ret < 0) { 3882 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 3883 __func__); 3884 goto init_error; 3885 } 3886 3887 return dma_conf; 3888 3889 init_error: 3890 free_dma_desc_resources(priv, dma_conf); 3891 alloc_error: 3892 kfree(dma_conf); 3893 return ERR_PTR(ret); 3894 } 3895 3896 /** 3897 * __stmmac_open - open entry point of the driver 3898 * @dev : pointer to the device structure. 3899 * @dma_conf : structure to take the dma data 3900 * Description: 3901 * This function is the open entry point of the driver. 3902 * Return value: 3903 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3904 * file on failure. 3905 */ 3906 static int __stmmac_open(struct net_device *dev, 3907 struct stmmac_dma_conf *dma_conf) 3908 { 3909 struct stmmac_priv *priv = netdev_priv(dev); 3910 int mode = priv->plat->phy_interface; 3911 u32 chan; 3912 int ret; 3913 3914 ret = pm_runtime_resume_and_get(priv->device); 3915 if (ret < 0) 3916 return ret; 3917 3918 if (priv->hw->pcs != STMMAC_PCS_TBI && 3919 priv->hw->pcs != STMMAC_PCS_RTBI && 3920 (!priv->hw->xpcs || 3921 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73) && 3922 !priv->hw->lynx_pcs) { 3923 ret = stmmac_init_phy(dev); 3924 if (ret) { 3925 netdev_err(priv->dev, 3926 "%s: Cannot attach to PHY (error: %d)\n", 3927 __func__, ret); 3928 goto init_phy_error; 3929 } 3930 } 3931 3932 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 3933 3934 buf_sz = dma_conf->dma_buf_sz; 3935 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 3936 3937 stmmac_reset_queues_param(priv); 3938 3939 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 3940 priv->plat->serdes_powerup) { 3941 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); 3942 if (ret < 0) { 3943 netdev_err(priv->dev, "%s: Serdes powerup failed\n", 3944 __func__); 3945 goto init_error; 3946 } 3947 } 3948 3949 ret = stmmac_hw_setup(dev, true); 3950 if (ret < 0) { 3951 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 3952 goto init_error; 3953 } 3954 3955 stmmac_init_coalesce(priv); 3956 3957 phylink_start(priv->phylink); 3958 /* We may have called phylink_speed_down before */ 3959 phylink_speed_up(priv->phylink); 3960 3961 ret = stmmac_request_irq(dev); 3962 if (ret) 3963 goto irq_error; 3964 3965 stmmac_enable_all_queues(priv); 3966 netif_tx_start_all_queues(priv->dev); 3967 stmmac_enable_all_dma_irq(priv); 3968 3969 return 0; 3970 3971 irq_error: 3972 phylink_stop(priv->phylink); 3973 3974 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3975 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3976 3977 stmmac_hw_teardown(dev); 3978 init_error: 3979 phylink_disconnect_phy(priv->phylink); 3980 init_phy_error: 3981 pm_runtime_put(priv->device); 3982 return ret; 3983 } 3984 3985 static int stmmac_open(struct net_device *dev) 3986 { 3987 struct stmmac_priv *priv = netdev_priv(dev); 3988 struct stmmac_dma_conf *dma_conf; 3989 int ret; 3990 3991 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 3992 if (IS_ERR(dma_conf)) 3993 return PTR_ERR(dma_conf); 3994 3995 ret = __stmmac_open(dev, dma_conf); 3996 if (ret) 3997 free_dma_desc_resources(priv, dma_conf); 3998 3999 kfree(dma_conf); 4000 return ret; 4001 } 4002 4003 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) 4004 { 4005 set_bit(__FPE_REMOVING, &priv->fpe_task_state); 4006 4007 if (priv->fpe_wq) 4008 destroy_workqueue(priv->fpe_wq); 4009 4010 netdev_info(priv->dev, "FPE workqueue stop"); 4011 } 4012 4013 /** 4014 * stmmac_release - close entry point of the driver 4015 * @dev : device pointer. 4016 * Description: 4017 * This is the stop entry point of the driver. 4018 */ 4019 static int stmmac_release(struct net_device *dev) 4020 { 4021 struct stmmac_priv *priv = netdev_priv(dev); 4022 u32 chan; 4023 4024 if (device_may_wakeup(priv->device)) 4025 phylink_speed_down(priv->phylink, false); 4026 /* Stop and disconnect the PHY */ 4027 phylink_stop(priv->phylink); 4028 phylink_disconnect_phy(priv->phylink); 4029 4030 stmmac_disable_all_queues(priv); 4031 4032 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4033 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4034 4035 netif_tx_disable(dev); 4036 4037 /* Free the IRQ lines */ 4038 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4039 4040 if (priv->eee_enabled) { 4041 priv->tx_path_in_lpi_mode = false; 4042 del_timer_sync(&priv->eee_ctrl_timer); 4043 } 4044 4045 /* Stop TX/RX DMA and clear the descriptors */ 4046 stmmac_stop_all_dma(priv); 4047 4048 /* Release and free the Rx/Tx resources */ 4049 free_dma_desc_resources(priv, &priv->dma_conf); 4050 4051 /* Disable the MAC Rx/Tx */ 4052 stmmac_mac_set(priv, priv->ioaddr, false); 4053 4054 /* Powerdown Serdes if there is */ 4055 if (priv->plat->serdes_powerdown) 4056 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); 4057 4058 netif_carrier_off(dev); 4059 4060 stmmac_release_ptp(priv); 4061 4062 pm_runtime_put(priv->device); 4063 4064 if (priv->dma_cap.fpesel) 4065 stmmac_fpe_stop_wq(priv); 4066 4067 return 0; 4068 } 4069 4070 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4071 struct stmmac_tx_queue *tx_q) 4072 { 4073 u16 tag = 0x0, inner_tag = 0x0; 4074 u32 inner_type = 0x0; 4075 struct dma_desc *p; 4076 4077 if (!priv->dma_cap.vlins) 4078 return false; 4079 if (!skb_vlan_tag_present(skb)) 4080 return false; 4081 if (skb->vlan_proto == htons(ETH_P_8021AD)) { 4082 inner_tag = skb_vlan_tag_get(skb); 4083 inner_type = STMMAC_VLAN_INSERT; 4084 } 4085 4086 tag = skb_vlan_tag_get(skb); 4087 4088 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4089 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4090 else 4091 p = &tx_q->dma_tx[tx_q->cur_tx]; 4092 4093 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) 4094 return false; 4095 4096 stmmac_set_tx_owner(priv, p); 4097 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4098 return true; 4099 } 4100 4101 /** 4102 * stmmac_tso_allocator - close entry point of the driver 4103 * @priv: driver private structure 4104 * @des: buffer start address 4105 * @total_len: total length to fill in descriptors 4106 * @last_segment: condition for the last descriptor 4107 * @queue: TX queue index 4108 * Description: 4109 * This function fills descriptor and request new descriptors according to 4110 * buffer length to fill 4111 */ 4112 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4113 int total_len, bool last_segment, u32 queue) 4114 { 4115 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4116 struct dma_desc *desc; 4117 u32 buff_size; 4118 int tmp_len; 4119 4120 tmp_len = total_len; 4121 4122 while (tmp_len > 0) { 4123 dma_addr_t curr_addr; 4124 4125 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4126 priv->dma_conf.dma_tx_size); 4127 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4128 4129 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4130 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4131 else 4132 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4133 4134 curr_addr = des + (total_len - tmp_len); 4135 if (priv->dma_cap.addr64 <= 32) 4136 desc->des0 = cpu_to_le32(curr_addr); 4137 else 4138 stmmac_set_desc_addr(priv, desc, curr_addr); 4139 4140 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4141 TSO_MAX_BUFF_SIZE : tmp_len; 4142 4143 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4144 0, 1, 4145 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4146 0, 0); 4147 4148 tmp_len -= TSO_MAX_BUFF_SIZE; 4149 } 4150 } 4151 4152 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4153 { 4154 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4155 int desc_size; 4156 4157 if (likely(priv->extend_desc)) 4158 desc_size = sizeof(struct dma_extended_desc); 4159 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4160 desc_size = sizeof(struct dma_edesc); 4161 else 4162 desc_size = sizeof(struct dma_desc); 4163 4164 /* The own bit must be the latest setting done when prepare the 4165 * descriptor and then barrier is needed to make sure that 4166 * all is coherent before granting the DMA engine. 4167 */ 4168 wmb(); 4169 4170 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 4171 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 4172 } 4173 4174 /** 4175 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4176 * @skb : the socket buffer 4177 * @dev : device pointer 4178 * Description: this is the transmit function that is called on TSO frames 4179 * (support available on GMAC4 and newer chips). 4180 * Diagram below show the ring programming in case of TSO frames: 4181 * 4182 * First Descriptor 4183 * -------- 4184 * | DES0 |---> buffer1 = L2/L3/L4 header 4185 * | DES1 |---> TCP Payload (can continue on next descr...) 4186 * | DES2 |---> buffer 1 and 2 len 4187 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4188 * -------- 4189 * | 4190 * ... 4191 * | 4192 * -------- 4193 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 4194 * | DES1 | --| 4195 * | DES2 | --> buffer 1 and 2 len 4196 * | DES3 | 4197 * -------- 4198 * 4199 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4200 */ 4201 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4202 { 4203 struct dma_desc *desc, *first, *mss_desc = NULL; 4204 struct stmmac_priv *priv = netdev_priv(dev); 4205 int nfrags = skb_shinfo(skb)->nr_frags; 4206 u32 queue = skb_get_queue_mapping(skb); 4207 unsigned int first_entry, tx_packets; 4208 struct stmmac_txq_stats *txq_stats; 4209 int tmp_pay_len = 0, first_tx; 4210 struct stmmac_tx_queue *tx_q; 4211 bool has_vlan, set_ic; 4212 u8 proto_hdr_len, hdr; 4213 unsigned long flags; 4214 u32 pay_len, mss; 4215 dma_addr_t des; 4216 int i; 4217 4218 tx_q = &priv->dma_conf.tx_queue[queue]; 4219 txq_stats = &priv->xstats.txq_stats[queue]; 4220 first_tx = tx_q->cur_tx; 4221 4222 /* Compute header lengths */ 4223 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 4224 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 4225 hdr = sizeof(struct udphdr); 4226 } else { 4227 proto_hdr_len = skb_tcp_all_headers(skb); 4228 hdr = tcp_hdrlen(skb); 4229 } 4230 4231 /* Desc availability based on threshold should be enough safe */ 4232 if (unlikely(stmmac_tx_avail(priv, queue) < 4233 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4234 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4235 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4236 queue)); 4237 /* This is a hard error, log it. */ 4238 netdev_err(priv->dev, 4239 "%s: Tx Ring full when queue awake\n", 4240 __func__); 4241 } 4242 return NETDEV_TX_BUSY; 4243 } 4244 4245 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4246 4247 mss = skb_shinfo(skb)->gso_size; 4248 4249 /* set new MSS value if needed */ 4250 if (mss != tx_q->mss) { 4251 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4252 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4253 else 4254 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4255 4256 stmmac_set_mss(priv, mss_desc, mss); 4257 tx_q->mss = mss; 4258 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4259 priv->dma_conf.dma_tx_size); 4260 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4261 } 4262 4263 if (netif_msg_tx_queued(priv)) { 4264 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4265 __func__, hdr, proto_hdr_len, pay_len, mss); 4266 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4267 skb->data_len); 4268 } 4269 4270 /* Check if VLAN can be inserted by HW */ 4271 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4272 4273 first_entry = tx_q->cur_tx; 4274 WARN_ON(tx_q->tx_skbuff[first_entry]); 4275 4276 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4277 desc = &tx_q->dma_entx[first_entry].basic; 4278 else 4279 desc = &tx_q->dma_tx[first_entry]; 4280 first = desc; 4281 4282 if (has_vlan) 4283 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4284 4285 /* first descriptor: fill Headers on Buf1 */ 4286 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4287 DMA_TO_DEVICE); 4288 if (dma_mapping_error(priv->device, des)) 4289 goto dma_map_err; 4290 4291 tx_q->tx_skbuff_dma[first_entry].buf = des; 4292 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 4293 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4294 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4295 4296 if (priv->dma_cap.addr64 <= 32) { 4297 first->des0 = cpu_to_le32(des); 4298 4299 /* Fill start of payload in buff2 of first descriptor */ 4300 if (pay_len) 4301 first->des1 = cpu_to_le32(des + proto_hdr_len); 4302 4303 /* If needed take extra descriptors to fill the remaining payload */ 4304 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 4305 } else { 4306 stmmac_set_desc_addr(priv, first, des); 4307 tmp_pay_len = pay_len; 4308 des += proto_hdr_len; 4309 pay_len = 0; 4310 } 4311 4312 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 4313 4314 /* Prepare fragments */ 4315 for (i = 0; i < nfrags; i++) { 4316 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4317 4318 des = skb_frag_dma_map(priv->device, frag, 0, 4319 skb_frag_size(frag), 4320 DMA_TO_DEVICE); 4321 if (dma_mapping_error(priv->device, des)) 4322 goto dma_map_err; 4323 4324 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4325 (i == nfrags - 1), queue); 4326 4327 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4328 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4329 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4330 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4331 } 4332 4333 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4334 4335 /* Only the last descriptor gets to point to the skb. */ 4336 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4337 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4338 4339 /* Manage tx mitigation */ 4340 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4341 tx_q->tx_count_frames += tx_packets; 4342 4343 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4344 set_ic = true; 4345 else if (!priv->tx_coal_frames[queue]) 4346 set_ic = false; 4347 else if (tx_packets > priv->tx_coal_frames[queue]) 4348 set_ic = true; 4349 else if ((tx_q->tx_count_frames % 4350 priv->tx_coal_frames[queue]) < tx_packets) 4351 set_ic = true; 4352 else 4353 set_ic = false; 4354 4355 if (set_ic) { 4356 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4357 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4358 else 4359 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4360 4361 tx_q->tx_count_frames = 0; 4362 stmmac_set_tx_ic(priv, desc); 4363 } 4364 4365 /* We've used all descriptors we need for this skb, however, 4366 * advance cur_tx so that it references a fresh descriptor. 4367 * ndo_start_xmit will fill this descriptor the next time it's 4368 * called and stmmac_tx_clean may clean up to this descriptor. 4369 */ 4370 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4371 4372 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4373 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4374 __func__); 4375 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4376 } 4377 4378 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 4379 txq_stats->tx_bytes += skb->len; 4380 txq_stats->tx_tso_frames++; 4381 txq_stats->tx_tso_nfrags += nfrags; 4382 if (set_ic) 4383 txq_stats->tx_set_ic_bit++; 4384 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 4385 4386 if (priv->sarc_type) 4387 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4388 4389 skb_tx_timestamp(skb); 4390 4391 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4392 priv->hwts_tx_en)) { 4393 /* declare that device is doing timestamping */ 4394 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4395 stmmac_enable_tx_timestamp(priv, first); 4396 } 4397 4398 /* Complete the first descriptor before granting the DMA */ 4399 stmmac_prepare_tso_tx_desc(priv, first, 1, 4400 proto_hdr_len, 4401 pay_len, 4402 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 4403 hdr / 4, (skb->len - proto_hdr_len)); 4404 4405 /* If context desc is used to change MSS */ 4406 if (mss_desc) { 4407 /* Make sure that first descriptor has been completely 4408 * written, including its own bit. This is because MSS is 4409 * actually before first descriptor, so we need to make 4410 * sure that MSS's own bit is the last thing written. 4411 */ 4412 dma_wmb(); 4413 stmmac_set_tx_owner(priv, mss_desc); 4414 } 4415 4416 if (netif_msg_pktdata(priv)) { 4417 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4418 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4419 tx_q->cur_tx, first, nfrags); 4420 pr_info(">>> frame to be transmitted: "); 4421 print_pkt(skb->data, skb_headlen(skb)); 4422 } 4423 4424 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4425 4426 stmmac_flush_tx_descriptors(priv, queue); 4427 stmmac_tx_timer_arm(priv, queue); 4428 4429 return NETDEV_TX_OK; 4430 4431 dma_map_err: 4432 dev_err(priv->device, "Tx dma map failed\n"); 4433 dev_kfree_skb(skb); 4434 priv->xstats.tx_dropped++; 4435 return NETDEV_TX_OK; 4436 } 4437 4438 /** 4439 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4440 * @skb: socket buffer to check 4441 * 4442 * Check if a packet has an ethertype that will trigger the IP header checks 4443 * and IP/TCP checksum engine of the stmmac core. 4444 * 4445 * Return: true if the ethertype can trigger the checksum engine, false 4446 * otherwise 4447 */ 4448 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4449 { 4450 int depth = 0; 4451 __be16 proto; 4452 4453 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4454 &depth); 4455 4456 return (depth <= ETH_HLEN) && 4457 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4458 } 4459 4460 /** 4461 * stmmac_xmit - Tx entry point of the driver 4462 * @skb : the socket buffer 4463 * @dev : device pointer 4464 * Description : this is the tx entry point of the driver. 4465 * It programs the chain or the ring and supports oversized frames 4466 * and SG feature. 4467 */ 4468 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4469 { 4470 unsigned int first_entry, tx_packets, enh_desc; 4471 struct stmmac_priv *priv = netdev_priv(dev); 4472 unsigned int nopaged_len = skb_headlen(skb); 4473 int i, csum_insertion = 0, is_jumbo = 0; 4474 u32 queue = skb_get_queue_mapping(skb); 4475 int nfrags = skb_shinfo(skb)->nr_frags; 4476 int gso = skb_shinfo(skb)->gso_type; 4477 struct stmmac_txq_stats *txq_stats; 4478 struct dma_edesc *tbs_desc = NULL; 4479 struct dma_desc *desc, *first; 4480 struct stmmac_tx_queue *tx_q; 4481 bool has_vlan, set_ic; 4482 int entry, first_tx; 4483 unsigned long flags; 4484 dma_addr_t des; 4485 4486 tx_q = &priv->dma_conf.tx_queue[queue]; 4487 txq_stats = &priv->xstats.txq_stats[queue]; 4488 first_tx = tx_q->cur_tx; 4489 4490 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4491 stmmac_disable_eee_mode(priv); 4492 4493 /* Manage oversized TCP frames for GMAC4 device */ 4494 if (skb_is_gso(skb) && priv->tso) { 4495 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4496 return stmmac_tso_xmit(skb, dev); 4497 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) 4498 return stmmac_tso_xmit(skb, dev); 4499 } 4500 4501 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4502 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4503 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4504 queue)); 4505 /* This is a hard error, log it. */ 4506 netdev_err(priv->dev, 4507 "%s: Tx Ring full when queue awake\n", 4508 __func__); 4509 } 4510 return NETDEV_TX_BUSY; 4511 } 4512 4513 /* Check if VLAN can be inserted by HW */ 4514 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4515 4516 entry = tx_q->cur_tx; 4517 first_entry = entry; 4518 WARN_ON(tx_q->tx_skbuff[first_entry]); 4519 4520 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4521 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 4522 * queues. In that case, checksum offloading for those queues that don't 4523 * support tx coe needs to fallback to software checksum calculation. 4524 * 4525 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 4526 * also have to be checksummed in software. 4527 */ 4528 if (csum_insertion && 4529 (priv->plat->tx_queues_cfg[queue].coe_unsupported || 4530 !stmmac_has_ip_ethertype(skb))) { 4531 if (unlikely(skb_checksum_help(skb))) 4532 goto dma_map_err; 4533 csum_insertion = !csum_insertion; 4534 } 4535 4536 if (likely(priv->extend_desc)) 4537 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4538 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4539 desc = &tx_q->dma_entx[entry].basic; 4540 else 4541 desc = tx_q->dma_tx + entry; 4542 4543 first = desc; 4544 4545 if (has_vlan) 4546 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4547 4548 enh_desc = priv->plat->enh_desc; 4549 /* To program the descriptors according to the size of the frame */ 4550 if (enh_desc) 4551 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4552 4553 if (unlikely(is_jumbo)) { 4554 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4555 if (unlikely(entry < 0) && (entry != -EINVAL)) 4556 goto dma_map_err; 4557 } 4558 4559 for (i = 0; i < nfrags; i++) { 4560 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4561 int len = skb_frag_size(frag); 4562 bool last_segment = (i == (nfrags - 1)); 4563 4564 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4565 WARN_ON(tx_q->tx_skbuff[entry]); 4566 4567 if (likely(priv->extend_desc)) 4568 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4569 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4570 desc = &tx_q->dma_entx[entry].basic; 4571 else 4572 desc = tx_q->dma_tx + entry; 4573 4574 des = skb_frag_dma_map(priv->device, frag, 0, len, 4575 DMA_TO_DEVICE); 4576 if (dma_mapping_error(priv->device, des)) 4577 goto dma_map_err; /* should reuse desc w/o issues */ 4578 4579 tx_q->tx_skbuff_dma[entry].buf = des; 4580 4581 stmmac_set_desc_addr(priv, desc, des); 4582 4583 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4584 tx_q->tx_skbuff_dma[entry].len = len; 4585 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4586 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4587 4588 /* Prepare the descriptor and set the own bit too */ 4589 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4590 priv->mode, 1, last_segment, skb->len); 4591 } 4592 4593 /* Only the last descriptor gets to point to the skb. */ 4594 tx_q->tx_skbuff[entry] = skb; 4595 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4596 4597 /* According to the coalesce parameter the IC bit for the latest 4598 * segment is reset and the timer re-started to clean the tx status. 4599 * This approach takes care about the fragments: desc is the first 4600 * element in case of no SG. 4601 */ 4602 tx_packets = (entry + 1) - first_tx; 4603 tx_q->tx_count_frames += tx_packets; 4604 4605 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4606 set_ic = true; 4607 else if (!priv->tx_coal_frames[queue]) 4608 set_ic = false; 4609 else if (tx_packets > priv->tx_coal_frames[queue]) 4610 set_ic = true; 4611 else if ((tx_q->tx_count_frames % 4612 priv->tx_coal_frames[queue]) < tx_packets) 4613 set_ic = true; 4614 else 4615 set_ic = false; 4616 4617 if (set_ic) { 4618 if (likely(priv->extend_desc)) 4619 desc = &tx_q->dma_etx[entry].basic; 4620 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4621 desc = &tx_q->dma_entx[entry].basic; 4622 else 4623 desc = &tx_q->dma_tx[entry]; 4624 4625 tx_q->tx_count_frames = 0; 4626 stmmac_set_tx_ic(priv, desc); 4627 } 4628 4629 /* We've used all descriptors we need for this skb, however, 4630 * advance cur_tx so that it references a fresh descriptor. 4631 * ndo_start_xmit will fill this descriptor the next time it's 4632 * called and stmmac_tx_clean may clean up to this descriptor. 4633 */ 4634 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4635 tx_q->cur_tx = entry; 4636 4637 if (netif_msg_pktdata(priv)) { 4638 netdev_dbg(priv->dev, 4639 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4640 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4641 entry, first, nfrags); 4642 4643 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4644 print_pkt(skb->data, skb->len); 4645 } 4646 4647 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4648 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4649 __func__); 4650 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4651 } 4652 4653 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 4654 txq_stats->tx_bytes += skb->len; 4655 if (set_ic) 4656 txq_stats->tx_set_ic_bit++; 4657 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 4658 4659 if (priv->sarc_type) 4660 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4661 4662 skb_tx_timestamp(skb); 4663 4664 /* Ready to fill the first descriptor and set the OWN bit w/o any 4665 * problems because all the descriptors are actually ready to be 4666 * passed to the DMA engine. 4667 */ 4668 if (likely(!is_jumbo)) { 4669 bool last_segment = (nfrags == 0); 4670 4671 des = dma_map_single(priv->device, skb->data, 4672 nopaged_len, DMA_TO_DEVICE); 4673 if (dma_mapping_error(priv->device, des)) 4674 goto dma_map_err; 4675 4676 tx_q->tx_skbuff_dma[first_entry].buf = des; 4677 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4678 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4679 4680 stmmac_set_desc_addr(priv, first, des); 4681 4682 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4683 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4684 4685 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4686 priv->hwts_tx_en)) { 4687 /* declare that device is doing timestamping */ 4688 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4689 stmmac_enable_tx_timestamp(priv, first); 4690 } 4691 4692 /* Prepare the first descriptor setting the OWN bit too */ 4693 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4694 csum_insertion, priv->mode, 0, last_segment, 4695 skb->len); 4696 } 4697 4698 if (tx_q->tbs & STMMAC_TBS_EN) { 4699 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4700 4701 tbs_desc = &tx_q->dma_entx[first_entry]; 4702 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4703 } 4704 4705 stmmac_set_tx_owner(priv, first); 4706 4707 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4708 4709 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4710 4711 stmmac_flush_tx_descriptors(priv, queue); 4712 stmmac_tx_timer_arm(priv, queue); 4713 4714 return NETDEV_TX_OK; 4715 4716 dma_map_err: 4717 netdev_err(priv->dev, "Tx DMA map failed\n"); 4718 dev_kfree_skb(skb); 4719 priv->xstats.tx_dropped++; 4720 return NETDEV_TX_OK; 4721 } 4722 4723 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4724 { 4725 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 4726 __be16 vlan_proto = veth->h_vlan_proto; 4727 u16 vlanid; 4728 4729 if ((vlan_proto == htons(ETH_P_8021Q) && 4730 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4731 (vlan_proto == htons(ETH_P_8021AD) && 4732 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4733 /* pop the vlan tag */ 4734 vlanid = ntohs(veth->h_vlan_TCI); 4735 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4736 skb_pull(skb, VLAN_HLEN); 4737 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4738 } 4739 } 4740 4741 /** 4742 * stmmac_rx_refill - refill used skb preallocated buffers 4743 * @priv: driver private structure 4744 * @queue: RX queue index 4745 * Description : this is to reallocate the skb for the reception process 4746 * that is based on zero-copy. 4747 */ 4748 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4749 { 4750 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4751 int dirty = stmmac_rx_dirty(priv, queue); 4752 unsigned int entry = rx_q->dirty_rx; 4753 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4754 4755 if (priv->dma_cap.host_dma_width <= 32) 4756 gfp |= GFP_DMA32; 4757 4758 while (dirty-- > 0) { 4759 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4760 struct dma_desc *p; 4761 bool use_rx_wd; 4762 4763 if (priv->extend_desc) 4764 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4765 else 4766 p = rx_q->dma_rx + entry; 4767 4768 if (!buf->page) { 4769 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4770 if (!buf->page) 4771 break; 4772 } 4773 4774 if (priv->sph && !buf->sec_page) { 4775 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4776 if (!buf->sec_page) 4777 break; 4778 4779 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4780 } 4781 4782 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4783 4784 stmmac_set_desc_addr(priv, p, buf->addr); 4785 if (priv->sph) 4786 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4787 else 4788 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4789 stmmac_refill_desc3(priv, rx_q, p); 4790 4791 rx_q->rx_count_frames++; 4792 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4793 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4794 rx_q->rx_count_frames = 0; 4795 4796 use_rx_wd = !priv->rx_coal_frames[queue]; 4797 use_rx_wd |= rx_q->rx_count_frames > 0; 4798 if (!priv->use_riwt) 4799 use_rx_wd = false; 4800 4801 dma_wmb(); 4802 stmmac_set_rx_owner(priv, p, use_rx_wd); 4803 4804 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 4805 } 4806 rx_q->dirty_rx = entry; 4807 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4808 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4809 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4810 } 4811 4812 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4813 struct dma_desc *p, 4814 int status, unsigned int len) 4815 { 4816 unsigned int plen = 0, hlen = 0; 4817 int coe = priv->hw->rx_csum; 4818 4819 /* Not first descriptor, buffer is always zero */ 4820 if (priv->sph && len) 4821 return 0; 4822 4823 /* First descriptor, get split header length */ 4824 stmmac_get_rx_header_len(priv, p, &hlen); 4825 if (priv->sph && hlen) { 4826 priv->xstats.rx_split_hdr_pkt_n++; 4827 return hlen; 4828 } 4829 4830 /* First descriptor, not last descriptor and not split header */ 4831 if (status & rx_not_ls) 4832 return priv->dma_conf.dma_buf_sz; 4833 4834 plen = stmmac_get_rx_frame_len(priv, p, coe); 4835 4836 /* First descriptor and last descriptor and not split header */ 4837 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 4838 } 4839 4840 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4841 struct dma_desc *p, 4842 int status, unsigned int len) 4843 { 4844 int coe = priv->hw->rx_csum; 4845 unsigned int plen = 0; 4846 4847 /* Not split header, buffer is not available */ 4848 if (!priv->sph) 4849 return 0; 4850 4851 /* Not last descriptor */ 4852 if (status & rx_not_ls) 4853 return priv->dma_conf.dma_buf_sz; 4854 4855 plen = stmmac_get_rx_frame_len(priv, p, coe); 4856 4857 /* Last descriptor */ 4858 return plen - len; 4859 } 4860 4861 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 4862 struct xdp_frame *xdpf, bool dma_map) 4863 { 4864 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 4865 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4866 unsigned int entry = tx_q->cur_tx; 4867 struct dma_desc *tx_desc; 4868 dma_addr_t dma_addr; 4869 bool set_ic; 4870 4871 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 4872 return STMMAC_XDP_CONSUMED; 4873 4874 if (likely(priv->extend_desc)) 4875 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4876 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4877 tx_desc = &tx_q->dma_entx[entry].basic; 4878 else 4879 tx_desc = tx_q->dma_tx + entry; 4880 4881 if (dma_map) { 4882 dma_addr = dma_map_single(priv->device, xdpf->data, 4883 xdpf->len, DMA_TO_DEVICE); 4884 if (dma_mapping_error(priv->device, dma_addr)) 4885 return STMMAC_XDP_CONSUMED; 4886 4887 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 4888 } else { 4889 struct page *page = virt_to_page(xdpf->data); 4890 4891 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 4892 xdpf->headroom; 4893 dma_sync_single_for_device(priv->device, dma_addr, 4894 xdpf->len, DMA_BIDIRECTIONAL); 4895 4896 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 4897 } 4898 4899 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 4900 tx_q->tx_skbuff_dma[entry].map_as_page = false; 4901 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 4902 tx_q->tx_skbuff_dma[entry].last_segment = true; 4903 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 4904 4905 tx_q->xdpf[entry] = xdpf; 4906 4907 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 4908 4909 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 4910 true, priv->mode, true, true, 4911 xdpf->len); 4912 4913 tx_q->tx_count_frames++; 4914 4915 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 4916 set_ic = true; 4917 else 4918 set_ic = false; 4919 4920 if (set_ic) { 4921 unsigned long flags; 4922 tx_q->tx_count_frames = 0; 4923 stmmac_set_tx_ic(priv, tx_desc); 4924 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 4925 txq_stats->tx_set_ic_bit++; 4926 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 4927 } 4928 4929 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4930 4931 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4932 tx_q->cur_tx = entry; 4933 4934 return STMMAC_XDP_TX; 4935 } 4936 4937 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 4938 int cpu) 4939 { 4940 int index = cpu; 4941 4942 if (unlikely(index < 0)) 4943 index = 0; 4944 4945 while (index >= priv->plat->tx_queues_to_use) 4946 index -= priv->plat->tx_queues_to_use; 4947 4948 return index; 4949 } 4950 4951 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 4952 struct xdp_buff *xdp) 4953 { 4954 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 4955 int cpu = smp_processor_id(); 4956 struct netdev_queue *nq; 4957 int queue; 4958 int res; 4959 4960 if (unlikely(!xdpf)) 4961 return STMMAC_XDP_CONSUMED; 4962 4963 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4964 nq = netdev_get_tx_queue(priv->dev, queue); 4965 4966 __netif_tx_lock(nq, cpu); 4967 /* Avoids TX time-out as we are sharing with slow path */ 4968 txq_trans_cond_update(nq); 4969 4970 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); 4971 if (res == STMMAC_XDP_TX) 4972 stmmac_flush_tx_descriptors(priv, queue); 4973 4974 __netif_tx_unlock(nq); 4975 4976 return res; 4977 } 4978 4979 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 4980 struct bpf_prog *prog, 4981 struct xdp_buff *xdp) 4982 { 4983 u32 act; 4984 int res; 4985 4986 act = bpf_prog_run_xdp(prog, xdp); 4987 switch (act) { 4988 case XDP_PASS: 4989 res = STMMAC_XDP_PASS; 4990 break; 4991 case XDP_TX: 4992 res = stmmac_xdp_xmit_back(priv, xdp); 4993 break; 4994 case XDP_REDIRECT: 4995 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 4996 res = STMMAC_XDP_CONSUMED; 4997 else 4998 res = STMMAC_XDP_REDIRECT; 4999 break; 5000 default: 5001 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5002 fallthrough; 5003 case XDP_ABORTED: 5004 trace_xdp_exception(priv->dev, prog, act); 5005 fallthrough; 5006 case XDP_DROP: 5007 res = STMMAC_XDP_CONSUMED; 5008 break; 5009 } 5010 5011 return res; 5012 } 5013 5014 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5015 struct xdp_buff *xdp) 5016 { 5017 struct bpf_prog *prog; 5018 int res; 5019 5020 prog = READ_ONCE(priv->xdp_prog); 5021 if (!prog) { 5022 res = STMMAC_XDP_PASS; 5023 goto out; 5024 } 5025 5026 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5027 out: 5028 return ERR_PTR(-res); 5029 } 5030 5031 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5032 int xdp_status) 5033 { 5034 int cpu = smp_processor_id(); 5035 int queue; 5036 5037 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5038 5039 if (xdp_status & STMMAC_XDP_TX) 5040 stmmac_tx_timer_arm(priv, queue); 5041 5042 if (xdp_status & STMMAC_XDP_REDIRECT) 5043 xdp_do_flush(); 5044 } 5045 5046 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5047 struct xdp_buff *xdp) 5048 { 5049 unsigned int metasize = xdp->data - xdp->data_meta; 5050 unsigned int datasize = xdp->data_end - xdp->data; 5051 struct sk_buff *skb; 5052 5053 skb = __napi_alloc_skb(&ch->rxtx_napi, 5054 xdp->data_end - xdp->data_hard_start, 5055 GFP_ATOMIC | __GFP_NOWARN); 5056 if (unlikely(!skb)) 5057 return NULL; 5058 5059 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5060 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5061 if (metasize) 5062 skb_metadata_set(skb, metasize); 5063 5064 return skb; 5065 } 5066 5067 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5068 struct dma_desc *p, struct dma_desc *np, 5069 struct xdp_buff *xdp) 5070 { 5071 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5072 struct stmmac_channel *ch = &priv->channel[queue]; 5073 unsigned int len = xdp->data_end - xdp->data; 5074 enum pkt_hash_types hash_type; 5075 int coe = priv->hw->rx_csum; 5076 unsigned long flags; 5077 struct sk_buff *skb; 5078 u32 hash; 5079 5080 skb = stmmac_construct_skb_zc(ch, xdp); 5081 if (!skb) { 5082 priv->xstats.rx_dropped++; 5083 return; 5084 } 5085 5086 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5087 if (priv->hw->hw_vlan_en) 5088 /* MAC level stripping. */ 5089 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5090 else 5091 /* Driver level stripping. */ 5092 stmmac_rx_vlan(priv->dev, skb); 5093 skb->protocol = eth_type_trans(skb, priv->dev); 5094 5095 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5096 skb_checksum_none_assert(skb); 5097 else 5098 skb->ip_summed = CHECKSUM_UNNECESSARY; 5099 5100 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5101 skb_set_hash(skb, hash, hash_type); 5102 5103 skb_record_rx_queue(skb, queue); 5104 napi_gro_receive(&ch->rxtx_napi, skb); 5105 5106 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5107 rxq_stats->rx_pkt_n++; 5108 rxq_stats->rx_bytes += len; 5109 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5110 } 5111 5112 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5113 { 5114 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5115 unsigned int entry = rx_q->dirty_rx; 5116 struct dma_desc *rx_desc = NULL; 5117 bool ret = true; 5118 5119 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5120 5121 while (budget-- > 0 && entry != rx_q->cur_rx) { 5122 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5123 dma_addr_t dma_addr; 5124 bool use_rx_wd; 5125 5126 if (!buf->xdp) { 5127 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5128 if (!buf->xdp) { 5129 ret = false; 5130 break; 5131 } 5132 } 5133 5134 if (priv->extend_desc) 5135 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 5136 else 5137 rx_desc = rx_q->dma_rx + entry; 5138 5139 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5140 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5141 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5142 stmmac_refill_desc3(priv, rx_q, rx_desc); 5143 5144 rx_q->rx_count_frames++; 5145 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5146 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5147 rx_q->rx_count_frames = 0; 5148 5149 use_rx_wd = !priv->rx_coal_frames[queue]; 5150 use_rx_wd |= rx_q->rx_count_frames > 0; 5151 if (!priv->use_riwt) 5152 use_rx_wd = false; 5153 5154 dma_wmb(); 5155 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5156 5157 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 5158 } 5159 5160 if (rx_desc) { 5161 rx_q->dirty_rx = entry; 5162 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 5163 (rx_q->dirty_rx * sizeof(struct dma_desc)); 5164 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 5165 } 5166 5167 return ret; 5168 } 5169 5170 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5171 { 5172 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5173 * to represent incoming packet, whereas cb field in the same structure 5174 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5175 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5176 */ 5177 return (struct stmmac_xdp_buff *)xdp; 5178 } 5179 5180 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5181 { 5182 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5183 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5184 unsigned int count = 0, error = 0, len = 0; 5185 int dirty = stmmac_rx_dirty(priv, queue); 5186 unsigned int next_entry = rx_q->cur_rx; 5187 u32 rx_errors = 0, rx_dropped = 0; 5188 unsigned int desc_size; 5189 struct bpf_prog *prog; 5190 bool failure = false; 5191 unsigned long flags; 5192 int xdp_status = 0; 5193 int status = 0; 5194 5195 if (netif_msg_rx_status(priv)) { 5196 void *rx_head; 5197 5198 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5199 if (priv->extend_desc) { 5200 rx_head = (void *)rx_q->dma_erx; 5201 desc_size = sizeof(struct dma_extended_desc); 5202 } else { 5203 rx_head = (void *)rx_q->dma_rx; 5204 desc_size = sizeof(struct dma_desc); 5205 } 5206 5207 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5208 rx_q->dma_rx_phy, desc_size); 5209 } 5210 while (count < limit) { 5211 struct stmmac_rx_buffer *buf; 5212 struct stmmac_xdp_buff *ctx; 5213 unsigned int buf1_len = 0; 5214 struct dma_desc *np, *p; 5215 int entry; 5216 int res; 5217 5218 if (!count && rx_q->state_saved) { 5219 error = rx_q->state.error; 5220 len = rx_q->state.len; 5221 } else { 5222 rx_q->state_saved = false; 5223 error = 0; 5224 len = 0; 5225 } 5226 5227 if (count >= limit) 5228 break; 5229 5230 read_again: 5231 buf1_len = 0; 5232 entry = next_entry; 5233 buf = &rx_q->buf_pool[entry]; 5234 5235 if (dirty >= STMMAC_RX_FILL_BATCH) { 5236 failure = failure || 5237 !stmmac_rx_refill_zc(priv, queue, dirty); 5238 dirty = 0; 5239 } 5240 5241 if (priv->extend_desc) 5242 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5243 else 5244 p = rx_q->dma_rx + entry; 5245 5246 /* read the status of the incoming frame */ 5247 status = stmmac_rx_status(priv, &priv->xstats, p); 5248 /* check if managed by the DMA otherwise go ahead */ 5249 if (unlikely(status & dma_own)) 5250 break; 5251 5252 /* Prefetch the next RX descriptor */ 5253 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5254 priv->dma_conf.dma_rx_size); 5255 next_entry = rx_q->cur_rx; 5256 5257 if (priv->extend_desc) 5258 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5259 else 5260 np = rx_q->dma_rx + next_entry; 5261 5262 prefetch(np); 5263 5264 /* Ensure a valid XSK buffer before proceed */ 5265 if (!buf->xdp) 5266 break; 5267 5268 if (priv->extend_desc) 5269 stmmac_rx_extended_status(priv, &priv->xstats, 5270 rx_q->dma_erx + entry); 5271 if (unlikely(status == discard_frame)) { 5272 xsk_buff_free(buf->xdp); 5273 buf->xdp = NULL; 5274 dirty++; 5275 error = 1; 5276 if (!priv->hwts_rx_en) 5277 rx_errors++; 5278 } 5279 5280 if (unlikely(error && (status & rx_not_ls))) 5281 goto read_again; 5282 if (unlikely(error)) { 5283 count++; 5284 continue; 5285 } 5286 5287 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5288 if (likely(status & rx_not_ls)) { 5289 xsk_buff_free(buf->xdp); 5290 buf->xdp = NULL; 5291 dirty++; 5292 count++; 5293 goto read_again; 5294 } 5295 5296 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5297 ctx->priv = priv; 5298 ctx->desc = p; 5299 ctx->ndesc = np; 5300 5301 /* XDP ZC Frame only support primary buffers for now */ 5302 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5303 len += buf1_len; 5304 5305 /* ACS is disabled; strip manually. */ 5306 if (likely(!(status & rx_not_ls))) { 5307 buf1_len -= ETH_FCS_LEN; 5308 len -= ETH_FCS_LEN; 5309 } 5310 5311 /* RX buffer is good and fit into a XSK pool buffer */ 5312 buf->xdp->data_end = buf->xdp->data + buf1_len; 5313 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); 5314 5315 prog = READ_ONCE(priv->xdp_prog); 5316 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5317 5318 switch (res) { 5319 case STMMAC_XDP_PASS: 5320 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5321 xsk_buff_free(buf->xdp); 5322 break; 5323 case STMMAC_XDP_CONSUMED: 5324 xsk_buff_free(buf->xdp); 5325 rx_dropped++; 5326 break; 5327 case STMMAC_XDP_TX: 5328 case STMMAC_XDP_REDIRECT: 5329 xdp_status |= res; 5330 break; 5331 } 5332 5333 buf->xdp = NULL; 5334 dirty++; 5335 count++; 5336 } 5337 5338 if (status & rx_not_ls) { 5339 rx_q->state_saved = true; 5340 rx_q->state.error = error; 5341 rx_q->state.len = len; 5342 } 5343 5344 stmmac_finalize_xdp_rx(priv, xdp_status); 5345 5346 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5347 rxq_stats->rx_pkt_n += count; 5348 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5349 5350 priv->xstats.rx_dropped += rx_dropped; 5351 priv->xstats.rx_errors += rx_errors; 5352 5353 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5354 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5355 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5356 else 5357 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5358 5359 return (int)count; 5360 } 5361 5362 return failure ? limit : (int)count; 5363 } 5364 5365 /** 5366 * stmmac_rx - manage the receive process 5367 * @priv: driver private structure 5368 * @limit: napi bugget 5369 * @queue: RX queue index. 5370 * Description : this the function called by the napi poll method. 5371 * It gets all the frames inside the ring. 5372 */ 5373 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5374 { 5375 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5376 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5377 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5378 struct stmmac_channel *ch = &priv->channel[queue]; 5379 unsigned int count = 0, error = 0, len = 0; 5380 int status = 0, coe = priv->hw->rx_csum; 5381 unsigned int next_entry = rx_q->cur_rx; 5382 enum dma_data_direction dma_dir; 5383 unsigned int desc_size; 5384 struct sk_buff *skb = NULL; 5385 struct stmmac_xdp_buff ctx; 5386 unsigned long flags; 5387 int xdp_status = 0; 5388 int buf_sz; 5389 5390 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5391 buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5392 limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); 5393 5394 if (netif_msg_rx_status(priv)) { 5395 void *rx_head; 5396 5397 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5398 if (priv->extend_desc) { 5399 rx_head = (void *)rx_q->dma_erx; 5400 desc_size = sizeof(struct dma_extended_desc); 5401 } else { 5402 rx_head = (void *)rx_q->dma_rx; 5403 desc_size = sizeof(struct dma_desc); 5404 } 5405 5406 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5407 rx_q->dma_rx_phy, desc_size); 5408 } 5409 while (count < limit) { 5410 unsigned int buf1_len = 0, buf2_len = 0; 5411 enum pkt_hash_types hash_type; 5412 struct stmmac_rx_buffer *buf; 5413 struct dma_desc *np, *p; 5414 int entry; 5415 u32 hash; 5416 5417 if (!count && rx_q->state_saved) { 5418 skb = rx_q->state.skb; 5419 error = rx_q->state.error; 5420 len = rx_q->state.len; 5421 } else { 5422 rx_q->state_saved = false; 5423 skb = NULL; 5424 error = 0; 5425 len = 0; 5426 } 5427 5428 read_again: 5429 if (count >= limit) 5430 break; 5431 5432 buf1_len = 0; 5433 buf2_len = 0; 5434 entry = next_entry; 5435 buf = &rx_q->buf_pool[entry]; 5436 5437 if (priv->extend_desc) 5438 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5439 else 5440 p = rx_q->dma_rx + entry; 5441 5442 /* read the status of the incoming frame */ 5443 status = stmmac_rx_status(priv, &priv->xstats, p); 5444 /* check if managed by the DMA otherwise go ahead */ 5445 if (unlikely(status & dma_own)) 5446 break; 5447 5448 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5449 priv->dma_conf.dma_rx_size); 5450 next_entry = rx_q->cur_rx; 5451 5452 if (priv->extend_desc) 5453 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5454 else 5455 np = rx_q->dma_rx + next_entry; 5456 5457 prefetch(np); 5458 5459 if (priv->extend_desc) 5460 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5461 if (unlikely(status == discard_frame)) { 5462 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5463 buf->page = NULL; 5464 error = 1; 5465 if (!priv->hwts_rx_en) 5466 rx_errors++; 5467 } 5468 5469 if (unlikely(error && (status & rx_not_ls))) 5470 goto read_again; 5471 if (unlikely(error)) { 5472 dev_kfree_skb(skb); 5473 skb = NULL; 5474 count++; 5475 continue; 5476 } 5477 5478 /* Buffer is good. Go on. */ 5479 5480 prefetch(page_address(buf->page) + buf->page_offset); 5481 if (buf->sec_page) 5482 prefetch(page_address(buf->sec_page)); 5483 5484 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5485 len += buf1_len; 5486 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5487 len += buf2_len; 5488 5489 /* ACS is disabled; strip manually. */ 5490 if (likely(!(status & rx_not_ls))) { 5491 if (buf2_len) { 5492 buf2_len -= ETH_FCS_LEN; 5493 len -= ETH_FCS_LEN; 5494 } else if (buf1_len) { 5495 buf1_len -= ETH_FCS_LEN; 5496 len -= ETH_FCS_LEN; 5497 } 5498 } 5499 5500 if (!skb) { 5501 unsigned int pre_len, sync_len; 5502 5503 dma_sync_single_for_cpu(priv->device, buf->addr, 5504 buf1_len, dma_dir); 5505 5506 xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq); 5507 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5508 buf->page_offset, buf1_len, true); 5509 5510 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5511 buf->page_offset; 5512 5513 ctx.priv = priv; 5514 ctx.desc = p; 5515 ctx.ndesc = np; 5516 5517 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5518 /* Due xdp_adjust_tail: DMA sync for_device 5519 * cover max len CPU touch 5520 */ 5521 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5522 buf->page_offset; 5523 sync_len = max(sync_len, pre_len); 5524 5525 /* For Not XDP_PASS verdict */ 5526 if (IS_ERR(skb)) { 5527 unsigned int xdp_res = -PTR_ERR(skb); 5528 5529 if (xdp_res & STMMAC_XDP_CONSUMED) { 5530 page_pool_put_page(rx_q->page_pool, 5531 virt_to_head_page(ctx.xdp.data), 5532 sync_len, true); 5533 buf->page = NULL; 5534 rx_dropped++; 5535 5536 /* Clear skb as it was set as 5537 * status by XDP program. 5538 */ 5539 skb = NULL; 5540 5541 if (unlikely((status & rx_not_ls))) 5542 goto read_again; 5543 5544 count++; 5545 continue; 5546 } else if (xdp_res & (STMMAC_XDP_TX | 5547 STMMAC_XDP_REDIRECT)) { 5548 xdp_status |= xdp_res; 5549 buf->page = NULL; 5550 skb = NULL; 5551 count++; 5552 continue; 5553 } 5554 } 5555 } 5556 5557 if (!skb) { 5558 /* XDP program may expand or reduce tail */ 5559 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5560 5561 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); 5562 if (!skb) { 5563 rx_dropped++; 5564 count++; 5565 goto drain_data; 5566 } 5567 5568 /* XDP program may adjust header */ 5569 skb_copy_to_linear_data(skb, ctx.xdp.data, buf1_len); 5570 skb_put(skb, buf1_len); 5571 5572 /* Data payload copied into SKB, page ready for recycle */ 5573 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5574 buf->page = NULL; 5575 } else if (buf1_len) { 5576 dma_sync_single_for_cpu(priv->device, buf->addr, 5577 buf1_len, dma_dir); 5578 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5579 buf->page, buf->page_offset, buf1_len, 5580 priv->dma_conf.dma_buf_sz); 5581 5582 /* Data payload appended into SKB */ 5583 skb_mark_for_recycle(skb); 5584 buf->page = NULL; 5585 } 5586 5587 if (buf2_len) { 5588 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5589 buf2_len, dma_dir); 5590 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5591 buf->sec_page, 0, buf2_len, 5592 priv->dma_conf.dma_buf_sz); 5593 5594 /* Data payload appended into SKB */ 5595 skb_mark_for_recycle(skb); 5596 buf->sec_page = NULL; 5597 } 5598 5599 drain_data: 5600 if (likely(status & rx_not_ls)) 5601 goto read_again; 5602 if (!skb) 5603 continue; 5604 5605 /* Got entire packet into SKB. Finish it. */ 5606 5607 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5608 5609 if (priv->hw->hw_vlan_en) 5610 /* MAC level stripping. */ 5611 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5612 else 5613 /* Driver level stripping. */ 5614 stmmac_rx_vlan(priv->dev, skb); 5615 5616 skb->protocol = eth_type_trans(skb, priv->dev); 5617 5618 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5619 skb_checksum_none_assert(skb); 5620 else 5621 skb->ip_summed = CHECKSUM_UNNECESSARY; 5622 5623 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5624 skb_set_hash(skb, hash, hash_type); 5625 5626 skb_record_rx_queue(skb, queue); 5627 napi_gro_receive(&ch->rx_napi, skb); 5628 skb = NULL; 5629 5630 rx_packets++; 5631 rx_bytes += len; 5632 count++; 5633 } 5634 5635 if (status & rx_not_ls || skb) { 5636 rx_q->state_saved = true; 5637 rx_q->state.skb = skb; 5638 rx_q->state.error = error; 5639 rx_q->state.len = len; 5640 } 5641 5642 stmmac_finalize_xdp_rx(priv, xdp_status); 5643 5644 stmmac_rx_refill(priv, queue); 5645 5646 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5647 rxq_stats->rx_packets += rx_packets; 5648 rxq_stats->rx_bytes += rx_bytes; 5649 rxq_stats->rx_pkt_n += count; 5650 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5651 5652 priv->xstats.rx_dropped += rx_dropped; 5653 priv->xstats.rx_errors += rx_errors; 5654 5655 return count; 5656 } 5657 5658 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5659 { 5660 struct stmmac_channel *ch = 5661 container_of(napi, struct stmmac_channel, rx_napi); 5662 struct stmmac_priv *priv = ch->priv_data; 5663 struct stmmac_rxq_stats *rxq_stats; 5664 u32 chan = ch->index; 5665 unsigned long flags; 5666 int work_done; 5667 5668 rxq_stats = &priv->xstats.rxq_stats[chan]; 5669 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5670 rxq_stats->napi_poll++; 5671 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5672 5673 work_done = stmmac_rx(priv, budget, chan); 5674 if (work_done < budget && napi_complete_done(napi, work_done)) { 5675 unsigned long flags; 5676 5677 spin_lock_irqsave(&ch->lock, flags); 5678 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5679 spin_unlock_irqrestore(&ch->lock, flags); 5680 } 5681 5682 return work_done; 5683 } 5684 5685 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5686 { 5687 struct stmmac_channel *ch = 5688 container_of(napi, struct stmmac_channel, tx_napi); 5689 struct stmmac_priv *priv = ch->priv_data; 5690 struct stmmac_txq_stats *txq_stats; 5691 bool pending_packets = false; 5692 u32 chan = ch->index; 5693 unsigned long flags; 5694 int work_done; 5695 5696 txq_stats = &priv->xstats.txq_stats[chan]; 5697 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 5698 txq_stats->napi_poll++; 5699 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 5700 5701 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 5702 work_done = min(work_done, budget); 5703 5704 if (work_done < budget && napi_complete_done(napi, work_done)) { 5705 unsigned long flags; 5706 5707 spin_lock_irqsave(&ch->lock, flags); 5708 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5709 spin_unlock_irqrestore(&ch->lock, flags); 5710 } 5711 5712 /* TX still have packet to handle, check if we need to arm tx timer */ 5713 if (pending_packets) 5714 stmmac_tx_timer_arm(priv, chan); 5715 5716 return work_done; 5717 } 5718 5719 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5720 { 5721 struct stmmac_channel *ch = 5722 container_of(napi, struct stmmac_channel, rxtx_napi); 5723 struct stmmac_priv *priv = ch->priv_data; 5724 bool tx_pending_packets = false; 5725 int rx_done, tx_done, rxtx_done; 5726 struct stmmac_rxq_stats *rxq_stats; 5727 struct stmmac_txq_stats *txq_stats; 5728 u32 chan = ch->index; 5729 unsigned long flags; 5730 5731 rxq_stats = &priv->xstats.rxq_stats[chan]; 5732 flags = u64_stats_update_begin_irqsave(&rxq_stats->syncp); 5733 rxq_stats->napi_poll++; 5734 u64_stats_update_end_irqrestore(&rxq_stats->syncp, flags); 5735 5736 txq_stats = &priv->xstats.txq_stats[chan]; 5737 flags = u64_stats_update_begin_irqsave(&txq_stats->syncp); 5738 txq_stats->napi_poll++; 5739 u64_stats_update_end_irqrestore(&txq_stats->syncp, flags); 5740 5741 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 5742 tx_done = min(tx_done, budget); 5743 5744 rx_done = stmmac_rx_zc(priv, budget, chan); 5745 5746 rxtx_done = max(tx_done, rx_done); 5747 5748 /* If either TX or RX work is not complete, return budget 5749 * and keep pooling 5750 */ 5751 if (rxtx_done >= budget) 5752 return budget; 5753 5754 /* all work done, exit the polling mode */ 5755 if (napi_complete_done(napi, rxtx_done)) { 5756 unsigned long flags; 5757 5758 spin_lock_irqsave(&ch->lock, flags); 5759 /* Both RX and TX work done are compelte, 5760 * so enable both RX & TX IRQs. 5761 */ 5762 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5763 spin_unlock_irqrestore(&ch->lock, flags); 5764 } 5765 5766 /* TX still have packet to handle, check if we need to arm tx timer */ 5767 if (tx_pending_packets) 5768 stmmac_tx_timer_arm(priv, chan); 5769 5770 return min(rxtx_done, budget - 1); 5771 } 5772 5773 /** 5774 * stmmac_tx_timeout 5775 * @dev : Pointer to net device structure 5776 * @txqueue: the index of the hanging transmit queue 5777 * Description: this function is called when a packet transmission fails to 5778 * complete within a reasonable time. The driver will mark the error in the 5779 * netdev structure and arrange for the device to be reset to a sane state 5780 * in order to transmit a new packet. 5781 */ 5782 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5783 { 5784 struct stmmac_priv *priv = netdev_priv(dev); 5785 5786 stmmac_global_err(priv); 5787 } 5788 5789 /** 5790 * stmmac_set_rx_mode - entry point for multicast addressing 5791 * @dev : pointer to the device structure 5792 * Description: 5793 * This function is a driver entry point which gets called by the kernel 5794 * whenever multicast addresses must be enabled/disabled. 5795 * Return value: 5796 * void. 5797 */ 5798 static void stmmac_set_rx_mode(struct net_device *dev) 5799 { 5800 struct stmmac_priv *priv = netdev_priv(dev); 5801 5802 stmmac_set_filter(priv, priv->hw, dev); 5803 } 5804 5805 /** 5806 * stmmac_change_mtu - entry point to change MTU size for the device. 5807 * @dev : device pointer. 5808 * @new_mtu : the new MTU size for the device. 5809 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5810 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5811 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5812 * Return value: 5813 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5814 * file on failure. 5815 */ 5816 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5817 { 5818 struct stmmac_priv *priv = netdev_priv(dev); 5819 int txfifosz = priv->plat->tx_fifo_size; 5820 struct stmmac_dma_conf *dma_conf; 5821 const int mtu = new_mtu; 5822 int ret; 5823 5824 if (txfifosz == 0) 5825 txfifosz = priv->dma_cap.tx_fifo_size; 5826 5827 txfifosz /= priv->plat->tx_queues_to_use; 5828 5829 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5830 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 5831 return -EINVAL; 5832 } 5833 5834 new_mtu = STMMAC_ALIGN(new_mtu); 5835 5836 /* If condition true, FIFO is too small or MTU too large */ 5837 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 5838 return -EINVAL; 5839 5840 if (netif_running(dev)) { 5841 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 5842 /* Try to allocate the new DMA conf with the new mtu */ 5843 dma_conf = stmmac_setup_dma_desc(priv, mtu); 5844 if (IS_ERR(dma_conf)) { 5845 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 5846 mtu); 5847 return PTR_ERR(dma_conf); 5848 } 5849 5850 stmmac_release(dev); 5851 5852 ret = __stmmac_open(dev, dma_conf); 5853 if (ret) { 5854 free_dma_desc_resources(priv, dma_conf); 5855 kfree(dma_conf); 5856 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 5857 return ret; 5858 } 5859 5860 kfree(dma_conf); 5861 5862 stmmac_set_rx_mode(dev); 5863 } 5864 5865 dev->mtu = mtu; 5866 netdev_update_features(dev); 5867 5868 return 0; 5869 } 5870 5871 static netdev_features_t stmmac_fix_features(struct net_device *dev, 5872 netdev_features_t features) 5873 { 5874 struct stmmac_priv *priv = netdev_priv(dev); 5875 5876 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 5877 features &= ~NETIF_F_RXCSUM; 5878 5879 if (!priv->plat->tx_coe) 5880 features &= ~NETIF_F_CSUM_MASK; 5881 5882 /* Some GMAC devices have a bugged Jumbo frame support that 5883 * needs to have the Tx COE disabled for oversized frames 5884 * (due to limited buffer sizes). In this case we disable 5885 * the TX csum insertion in the TDES and not use SF. 5886 */ 5887 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 5888 features &= ~NETIF_F_CSUM_MASK; 5889 5890 /* Disable tso if asked by ethtool */ 5891 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 5892 if (features & NETIF_F_TSO) 5893 priv->tso = true; 5894 else 5895 priv->tso = false; 5896 } 5897 5898 return features; 5899 } 5900 5901 static int stmmac_set_features(struct net_device *netdev, 5902 netdev_features_t features) 5903 { 5904 struct stmmac_priv *priv = netdev_priv(netdev); 5905 5906 /* Keep the COE Type in case of csum is supporting */ 5907 if (features & NETIF_F_RXCSUM) 5908 priv->hw->rx_csum = priv->plat->rx_coe; 5909 else 5910 priv->hw->rx_csum = 0; 5911 /* No check needed because rx_coe has been set before and it will be 5912 * fixed in case of issue. 5913 */ 5914 stmmac_rx_ipc(priv, priv->hw); 5915 5916 if (priv->sph_cap) { 5917 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; 5918 u32 chan; 5919 5920 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 5921 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 5922 } 5923 5924 if (features & NETIF_F_HW_VLAN_CTAG_RX) 5925 priv->hw->hw_vlan_en = true; 5926 else 5927 priv->hw->hw_vlan_en = false; 5928 5929 stmmac_set_hw_vlan_mode(priv, priv->hw); 5930 5931 return 0; 5932 } 5933 5934 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) 5935 { 5936 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 5937 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 5938 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 5939 bool *hs_enable = &fpe_cfg->hs_enable; 5940 5941 if (status == FPE_EVENT_UNKNOWN || !*hs_enable) 5942 return; 5943 5944 /* If LP has sent verify mPacket, LP is FPE capable */ 5945 if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) { 5946 if (*lp_state < FPE_STATE_CAPABLE) 5947 *lp_state = FPE_STATE_CAPABLE; 5948 5949 /* If user has requested FPE enable, quickly response */ 5950 if (*hs_enable) 5951 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 5952 fpe_cfg, 5953 MPACKET_RESPONSE); 5954 } 5955 5956 /* If Local has sent verify mPacket, Local is FPE capable */ 5957 if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) { 5958 if (*lo_state < FPE_STATE_CAPABLE) 5959 *lo_state = FPE_STATE_CAPABLE; 5960 } 5961 5962 /* If LP has sent response mPacket, LP is entering FPE ON */ 5963 if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP) 5964 *lp_state = FPE_STATE_ENTERING_ON; 5965 5966 /* If Local has sent response mPacket, Local is entering FPE ON */ 5967 if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP) 5968 *lo_state = FPE_STATE_ENTERING_ON; 5969 5970 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && 5971 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && 5972 priv->fpe_wq) { 5973 queue_work(priv->fpe_wq, &priv->fpe_task); 5974 } 5975 } 5976 5977 static void stmmac_common_interrupt(struct stmmac_priv *priv) 5978 { 5979 u32 rx_cnt = priv->plat->rx_queues_to_use; 5980 u32 tx_cnt = priv->plat->tx_queues_to_use; 5981 u32 queues_count; 5982 u32 queue; 5983 bool xmac; 5984 5985 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 5986 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 5987 5988 if (priv->irq_wake) 5989 pm_wakeup_event(priv->device, 0); 5990 5991 if (priv->dma_cap.estsel) 5992 stmmac_est_irq_status(priv, priv, priv->dev, 5993 &priv->xstats, tx_cnt); 5994 5995 if (priv->dma_cap.fpesel) { 5996 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, 5997 priv->dev); 5998 5999 stmmac_fpe_event_status(priv, status); 6000 } 6001 6002 /* To handle GMAC own interrupts */ 6003 if ((priv->plat->has_gmac) || xmac) { 6004 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 6005 6006 if (unlikely(status)) { 6007 /* For LPI we need to save the tx status */ 6008 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 6009 priv->tx_path_in_lpi_mode = true; 6010 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 6011 priv->tx_path_in_lpi_mode = false; 6012 } 6013 6014 for (queue = 0; queue < queues_count; queue++) { 6015 status = stmmac_host_mtl_irq_status(priv, priv->hw, 6016 queue); 6017 } 6018 6019 /* PCS link status */ 6020 if (priv->hw->pcs && 6021 !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { 6022 if (priv->xstats.pcs_link) 6023 netif_carrier_on(priv->dev); 6024 else 6025 netif_carrier_off(priv->dev); 6026 } 6027 6028 stmmac_timestamp_interrupt(priv, priv); 6029 } 6030 } 6031 6032 /** 6033 * stmmac_interrupt - main ISR 6034 * @irq: interrupt number. 6035 * @dev_id: to pass the net device pointer. 6036 * Description: this is the main driver interrupt service routine. 6037 * It can call: 6038 * o DMA service routine (to manage incoming frame reception and transmission 6039 * status) 6040 * o Core interrupts to manage: remote wake-up, management counter, LPI 6041 * interrupts. 6042 */ 6043 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6044 { 6045 struct net_device *dev = (struct net_device *)dev_id; 6046 struct stmmac_priv *priv = netdev_priv(dev); 6047 6048 /* Check if adapter is up */ 6049 if (test_bit(STMMAC_DOWN, &priv->state)) 6050 return IRQ_HANDLED; 6051 6052 /* Check if a fatal error happened */ 6053 if (stmmac_safety_feat_interrupt(priv)) 6054 return IRQ_HANDLED; 6055 6056 /* To handle Common interrupts */ 6057 stmmac_common_interrupt(priv); 6058 6059 /* To handle DMA interrupts */ 6060 stmmac_dma_interrupt(priv); 6061 6062 return IRQ_HANDLED; 6063 } 6064 6065 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6066 { 6067 struct net_device *dev = (struct net_device *)dev_id; 6068 struct stmmac_priv *priv = netdev_priv(dev); 6069 6070 if (unlikely(!dev)) { 6071 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 6072 return IRQ_NONE; 6073 } 6074 6075 /* Check if adapter is up */ 6076 if (test_bit(STMMAC_DOWN, &priv->state)) 6077 return IRQ_HANDLED; 6078 6079 /* To handle Common interrupts */ 6080 stmmac_common_interrupt(priv); 6081 6082 return IRQ_HANDLED; 6083 } 6084 6085 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6086 { 6087 struct net_device *dev = (struct net_device *)dev_id; 6088 struct stmmac_priv *priv = netdev_priv(dev); 6089 6090 if (unlikely(!dev)) { 6091 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 6092 return IRQ_NONE; 6093 } 6094 6095 /* Check if adapter is up */ 6096 if (test_bit(STMMAC_DOWN, &priv->state)) 6097 return IRQ_HANDLED; 6098 6099 /* Check if a fatal error happened */ 6100 stmmac_safety_feat_interrupt(priv); 6101 6102 return IRQ_HANDLED; 6103 } 6104 6105 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6106 { 6107 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6108 struct stmmac_dma_conf *dma_conf; 6109 int chan = tx_q->queue_index; 6110 struct stmmac_priv *priv; 6111 int status; 6112 6113 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6114 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6115 6116 if (unlikely(!data)) { 6117 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 6118 return IRQ_NONE; 6119 } 6120 6121 /* Check if adapter is up */ 6122 if (test_bit(STMMAC_DOWN, &priv->state)) 6123 return IRQ_HANDLED; 6124 6125 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6126 6127 if (unlikely(status & tx_hard_error_bump_tc)) { 6128 /* Try to bump up the dma threshold on this failure */ 6129 stmmac_bump_dma_threshold(priv, chan); 6130 } else if (unlikely(status == tx_hard_error)) { 6131 stmmac_tx_err(priv, chan); 6132 } 6133 6134 return IRQ_HANDLED; 6135 } 6136 6137 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6138 { 6139 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6140 struct stmmac_dma_conf *dma_conf; 6141 int chan = rx_q->queue_index; 6142 struct stmmac_priv *priv; 6143 6144 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6145 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6146 6147 if (unlikely(!data)) { 6148 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 6149 return IRQ_NONE; 6150 } 6151 6152 /* Check if adapter is up */ 6153 if (test_bit(STMMAC_DOWN, &priv->state)) 6154 return IRQ_HANDLED; 6155 6156 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6157 6158 return IRQ_HANDLED; 6159 } 6160 6161 /** 6162 * stmmac_ioctl - Entry point for the Ioctl 6163 * @dev: Device pointer. 6164 * @rq: An IOCTL specefic structure, that can contain a pointer to 6165 * a proprietary structure used to pass information to the driver. 6166 * @cmd: IOCTL command 6167 * Description: 6168 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6169 */ 6170 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6171 { 6172 struct stmmac_priv *priv = netdev_priv (dev); 6173 int ret = -EOPNOTSUPP; 6174 6175 if (!netif_running(dev)) 6176 return -EINVAL; 6177 6178 switch (cmd) { 6179 case SIOCGMIIPHY: 6180 case SIOCGMIIREG: 6181 case SIOCSMIIREG: 6182 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6183 break; 6184 case SIOCSHWTSTAMP: 6185 ret = stmmac_hwtstamp_set(dev, rq); 6186 break; 6187 case SIOCGHWTSTAMP: 6188 ret = stmmac_hwtstamp_get(dev, rq); 6189 break; 6190 default: 6191 break; 6192 } 6193 6194 return ret; 6195 } 6196 6197 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6198 void *cb_priv) 6199 { 6200 struct stmmac_priv *priv = cb_priv; 6201 int ret = -EOPNOTSUPP; 6202 6203 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6204 return ret; 6205 6206 __stmmac_disable_all_queues(priv); 6207 6208 switch (type) { 6209 case TC_SETUP_CLSU32: 6210 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6211 break; 6212 case TC_SETUP_CLSFLOWER: 6213 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6214 break; 6215 default: 6216 break; 6217 } 6218 6219 stmmac_enable_all_queues(priv); 6220 return ret; 6221 } 6222 6223 static LIST_HEAD(stmmac_block_cb_list); 6224 6225 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6226 void *type_data) 6227 { 6228 struct stmmac_priv *priv = netdev_priv(ndev); 6229 6230 switch (type) { 6231 case TC_QUERY_CAPS: 6232 return stmmac_tc_query_caps(priv, priv, type_data); 6233 case TC_SETUP_BLOCK: 6234 return flow_block_cb_setup_simple(type_data, 6235 &stmmac_block_cb_list, 6236 stmmac_setup_tc_block_cb, 6237 priv, priv, true); 6238 case TC_SETUP_QDISC_CBS: 6239 return stmmac_tc_setup_cbs(priv, priv, type_data); 6240 case TC_SETUP_QDISC_TAPRIO: 6241 return stmmac_tc_setup_taprio(priv, priv, type_data); 6242 case TC_SETUP_QDISC_ETF: 6243 return stmmac_tc_setup_etf(priv, priv, type_data); 6244 default: 6245 return -EOPNOTSUPP; 6246 } 6247 } 6248 6249 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6250 struct net_device *sb_dev) 6251 { 6252 int gso = skb_shinfo(skb)->gso_type; 6253 6254 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6255 /* 6256 * There is no way to determine the number of TSO/USO 6257 * capable Queues. Let's use always the Queue 0 6258 * because if TSO/USO is supported then at least this 6259 * one will be capable. 6260 */ 6261 return 0; 6262 } 6263 6264 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6265 } 6266 6267 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6268 { 6269 struct stmmac_priv *priv = netdev_priv(ndev); 6270 int ret = 0; 6271 6272 ret = pm_runtime_resume_and_get(priv->device); 6273 if (ret < 0) 6274 return ret; 6275 6276 ret = eth_mac_addr(ndev, addr); 6277 if (ret) 6278 goto set_mac_error; 6279 6280 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6281 6282 set_mac_error: 6283 pm_runtime_put(priv->device); 6284 6285 return ret; 6286 } 6287 6288 #ifdef CONFIG_DEBUG_FS 6289 static struct dentry *stmmac_fs_dir; 6290 6291 static void sysfs_display_ring(void *head, int size, int extend_desc, 6292 struct seq_file *seq, dma_addr_t dma_phy_addr) 6293 { 6294 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6295 struct dma_desc *p = (struct dma_desc *)head; 6296 unsigned int desc_size; 6297 dma_addr_t dma_addr; 6298 int i; 6299 6300 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6301 for (i = 0; i < size; i++) { 6302 dma_addr = dma_phy_addr + i * desc_size; 6303 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6304 i, &dma_addr, 6305 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6306 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6307 if (extend_desc) 6308 p = &(++ep)->basic; 6309 else 6310 p++; 6311 } 6312 } 6313 6314 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6315 { 6316 struct net_device *dev = seq->private; 6317 struct stmmac_priv *priv = netdev_priv(dev); 6318 u32 rx_count = priv->plat->rx_queues_to_use; 6319 u32 tx_count = priv->plat->tx_queues_to_use; 6320 u32 queue; 6321 6322 if ((dev->flags & IFF_UP) == 0) 6323 return 0; 6324 6325 for (queue = 0; queue < rx_count; queue++) { 6326 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6327 6328 seq_printf(seq, "RX Queue %d:\n", queue); 6329 6330 if (priv->extend_desc) { 6331 seq_printf(seq, "Extended descriptor ring:\n"); 6332 sysfs_display_ring((void *)rx_q->dma_erx, 6333 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6334 } else { 6335 seq_printf(seq, "Descriptor ring:\n"); 6336 sysfs_display_ring((void *)rx_q->dma_rx, 6337 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6338 } 6339 } 6340 6341 for (queue = 0; queue < tx_count; queue++) { 6342 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6343 6344 seq_printf(seq, "TX Queue %d:\n", queue); 6345 6346 if (priv->extend_desc) { 6347 seq_printf(seq, "Extended descriptor ring:\n"); 6348 sysfs_display_ring((void *)tx_q->dma_etx, 6349 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6350 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6351 seq_printf(seq, "Descriptor ring:\n"); 6352 sysfs_display_ring((void *)tx_q->dma_tx, 6353 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6354 } 6355 } 6356 6357 return 0; 6358 } 6359 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6360 6361 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6362 { 6363 static const char * const dwxgmac_timestamp_source[] = { 6364 "None", 6365 "Internal", 6366 "External", 6367 "Both", 6368 }; 6369 static const char * const dwxgmac_safety_feature_desc[] = { 6370 "No", 6371 "All Safety Features with ECC and Parity", 6372 "All Safety Features without ECC or Parity", 6373 "All Safety Features with Parity Only", 6374 "ECC Only", 6375 "UNDEFINED", 6376 "UNDEFINED", 6377 "UNDEFINED", 6378 }; 6379 struct net_device *dev = seq->private; 6380 struct stmmac_priv *priv = netdev_priv(dev); 6381 6382 if (!priv->hw_cap_support) { 6383 seq_printf(seq, "DMA HW features not supported\n"); 6384 return 0; 6385 } 6386 6387 seq_printf(seq, "==============================\n"); 6388 seq_printf(seq, "\tDMA HW features\n"); 6389 seq_printf(seq, "==============================\n"); 6390 6391 seq_printf(seq, "\t10/100 Mbps: %s\n", 6392 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6393 seq_printf(seq, "\t1000 Mbps: %s\n", 6394 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6395 seq_printf(seq, "\tHalf duplex: %s\n", 6396 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6397 if (priv->plat->has_xgmac) { 6398 seq_printf(seq, 6399 "\tNumber of Additional MAC address registers: %d\n", 6400 priv->dma_cap.multi_addr); 6401 } else { 6402 seq_printf(seq, "\tHash Filter: %s\n", 6403 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6404 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6405 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6406 } 6407 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6408 (priv->dma_cap.pcs) ? "Y" : "N"); 6409 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6410 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6411 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6412 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6413 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6414 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6415 seq_printf(seq, "\tRMON module: %s\n", 6416 (priv->dma_cap.rmon) ? "Y" : "N"); 6417 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6418 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6419 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6420 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6421 if (priv->plat->has_xgmac) 6422 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6423 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6424 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6425 (priv->dma_cap.eee) ? "Y" : "N"); 6426 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6427 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6428 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6429 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6430 priv->plat->has_xgmac) { 6431 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6432 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6433 } else { 6434 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6435 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6436 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6437 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6438 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6439 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6440 } 6441 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6442 priv->dma_cap.number_rx_channel); 6443 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6444 priv->dma_cap.number_tx_channel); 6445 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6446 priv->dma_cap.number_rx_queues); 6447 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6448 priv->dma_cap.number_tx_queues); 6449 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6450 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6451 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6452 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6453 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6454 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6455 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6456 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6457 priv->dma_cap.pps_out_num); 6458 seq_printf(seq, "\tSafety Features: %s\n", 6459 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6460 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6461 priv->dma_cap.frpsel ? "Y" : "N"); 6462 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6463 priv->dma_cap.host_dma_width); 6464 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6465 priv->dma_cap.rssen ? "Y" : "N"); 6466 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6467 priv->dma_cap.vlhash ? "Y" : "N"); 6468 seq_printf(seq, "\tSplit Header: %s\n", 6469 priv->dma_cap.sphen ? "Y" : "N"); 6470 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6471 priv->dma_cap.vlins ? "Y" : "N"); 6472 seq_printf(seq, "\tDouble VLAN: %s\n", 6473 priv->dma_cap.dvlan ? "Y" : "N"); 6474 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6475 priv->dma_cap.l3l4fnum); 6476 seq_printf(seq, "\tARP Offloading: %s\n", 6477 priv->dma_cap.arpoffsel ? "Y" : "N"); 6478 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6479 priv->dma_cap.estsel ? "Y" : "N"); 6480 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6481 priv->dma_cap.fpesel ? "Y" : "N"); 6482 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6483 priv->dma_cap.tbssel ? "Y" : "N"); 6484 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6485 priv->dma_cap.tbs_ch_num); 6486 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6487 priv->dma_cap.sgfsel ? "Y" : "N"); 6488 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6489 BIT(priv->dma_cap.ttsfd) >> 1); 6490 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6491 priv->dma_cap.numtc); 6492 seq_printf(seq, "\tDCB Feature: %s\n", 6493 priv->dma_cap.dcben ? "Y" : "N"); 6494 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6495 priv->dma_cap.advthword ? "Y" : "N"); 6496 seq_printf(seq, "\tPTP Offload: %s\n", 6497 priv->dma_cap.ptoen ? "Y" : "N"); 6498 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6499 priv->dma_cap.osten ? "Y" : "N"); 6500 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6501 priv->dma_cap.pfcen ? "Y" : "N"); 6502 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6503 BIT(priv->dma_cap.frpes) << 6); 6504 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6505 BIT(priv->dma_cap.frpbs) << 6); 6506 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6507 priv->dma_cap.frppipe_num); 6508 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6509 priv->dma_cap.nrvf_num ? 6510 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6511 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6512 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6513 seq_printf(seq, "\tDepth of GCL: %lu\n", 6514 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6515 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6516 priv->dma_cap.cbtisel ? "Y" : "N"); 6517 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6518 priv->dma_cap.aux_snapshot_n); 6519 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6520 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6521 seq_printf(seq, "\tEnhanced DMA: %s\n", 6522 priv->dma_cap.edma ? "Y" : "N"); 6523 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6524 priv->dma_cap.ediffc ? "Y" : "N"); 6525 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6526 priv->dma_cap.vxn ? "Y" : "N"); 6527 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6528 priv->dma_cap.dbgmem ? "Y" : "N"); 6529 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6530 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6531 return 0; 6532 } 6533 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6534 6535 /* Use network device events to rename debugfs file entries. 6536 */ 6537 static int stmmac_device_event(struct notifier_block *unused, 6538 unsigned long event, void *ptr) 6539 { 6540 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6541 struct stmmac_priv *priv = netdev_priv(dev); 6542 6543 if (dev->netdev_ops != &stmmac_netdev_ops) 6544 goto done; 6545 6546 switch (event) { 6547 case NETDEV_CHANGENAME: 6548 if (priv->dbgfs_dir) 6549 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, 6550 priv->dbgfs_dir, 6551 stmmac_fs_dir, 6552 dev->name); 6553 break; 6554 } 6555 done: 6556 return NOTIFY_DONE; 6557 } 6558 6559 static struct notifier_block stmmac_notifier = { 6560 .notifier_call = stmmac_device_event, 6561 }; 6562 6563 static void stmmac_init_fs(struct net_device *dev) 6564 { 6565 struct stmmac_priv *priv = netdev_priv(dev); 6566 6567 rtnl_lock(); 6568 6569 /* Create per netdev entries */ 6570 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6571 6572 /* Entry to report DMA RX/TX rings */ 6573 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6574 &stmmac_rings_status_fops); 6575 6576 /* Entry to report the DMA HW features */ 6577 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6578 &stmmac_dma_cap_fops); 6579 6580 rtnl_unlock(); 6581 } 6582 6583 static void stmmac_exit_fs(struct net_device *dev) 6584 { 6585 struct stmmac_priv *priv = netdev_priv(dev); 6586 6587 debugfs_remove_recursive(priv->dbgfs_dir); 6588 } 6589 #endif /* CONFIG_DEBUG_FS */ 6590 6591 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6592 { 6593 unsigned char *data = (unsigned char *)&vid_le; 6594 unsigned char data_byte = 0; 6595 u32 crc = ~0x0; 6596 u32 temp = 0; 6597 int i, bits; 6598 6599 bits = get_bitmask_order(VLAN_VID_MASK); 6600 for (i = 0; i < bits; i++) { 6601 if ((i % 8) == 0) 6602 data_byte = data[i / 8]; 6603 6604 temp = ((crc & 1) ^ data_byte) & 1; 6605 crc >>= 1; 6606 data_byte >>= 1; 6607 6608 if (temp) 6609 crc ^= 0xedb88320; 6610 } 6611 6612 return crc; 6613 } 6614 6615 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6616 { 6617 u32 crc, hash = 0; 6618 __le16 pmatch = 0; 6619 int count = 0; 6620 u16 vid = 0; 6621 6622 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6623 __le16 vid_le = cpu_to_le16(vid); 6624 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6625 hash |= (1 << crc); 6626 count++; 6627 } 6628 6629 if (!priv->dma_cap.vlhash) { 6630 if (count > 2) /* VID = 0 always passes filter */ 6631 return -EOPNOTSUPP; 6632 6633 pmatch = cpu_to_le16(vid); 6634 hash = 0; 6635 } 6636 6637 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6638 } 6639 6640 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6641 { 6642 struct stmmac_priv *priv = netdev_priv(ndev); 6643 bool is_double = false; 6644 int ret; 6645 6646 ret = pm_runtime_resume_and_get(priv->device); 6647 if (ret < 0) 6648 return ret; 6649 6650 if (be16_to_cpu(proto) == ETH_P_8021AD) 6651 is_double = true; 6652 6653 set_bit(vid, priv->active_vlans); 6654 ret = stmmac_vlan_update(priv, is_double); 6655 if (ret) { 6656 clear_bit(vid, priv->active_vlans); 6657 goto err_pm_put; 6658 } 6659 6660 if (priv->hw->num_vlan) { 6661 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6662 if (ret) 6663 goto err_pm_put; 6664 } 6665 err_pm_put: 6666 pm_runtime_put(priv->device); 6667 6668 return ret; 6669 } 6670 6671 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6672 { 6673 struct stmmac_priv *priv = netdev_priv(ndev); 6674 bool is_double = false; 6675 int ret; 6676 6677 ret = pm_runtime_resume_and_get(priv->device); 6678 if (ret < 0) 6679 return ret; 6680 6681 if (be16_to_cpu(proto) == ETH_P_8021AD) 6682 is_double = true; 6683 6684 clear_bit(vid, priv->active_vlans); 6685 6686 if (priv->hw->num_vlan) { 6687 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6688 if (ret) 6689 goto del_vlan_error; 6690 } 6691 6692 ret = stmmac_vlan_update(priv, is_double); 6693 6694 del_vlan_error: 6695 pm_runtime_put(priv->device); 6696 6697 return ret; 6698 } 6699 6700 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6701 { 6702 struct stmmac_priv *priv = netdev_priv(dev); 6703 6704 switch (bpf->command) { 6705 case XDP_SETUP_PROG: 6706 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6707 case XDP_SETUP_XSK_POOL: 6708 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6709 bpf->xsk.queue_id); 6710 default: 6711 return -EOPNOTSUPP; 6712 } 6713 } 6714 6715 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6716 struct xdp_frame **frames, u32 flags) 6717 { 6718 struct stmmac_priv *priv = netdev_priv(dev); 6719 int cpu = smp_processor_id(); 6720 struct netdev_queue *nq; 6721 int i, nxmit = 0; 6722 int queue; 6723 6724 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6725 return -ENETDOWN; 6726 6727 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6728 return -EINVAL; 6729 6730 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6731 nq = netdev_get_tx_queue(priv->dev, queue); 6732 6733 __netif_tx_lock(nq, cpu); 6734 /* Avoids TX time-out as we are sharing with slow path */ 6735 txq_trans_cond_update(nq); 6736 6737 for (i = 0; i < num_frames; i++) { 6738 int res; 6739 6740 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6741 if (res == STMMAC_XDP_CONSUMED) 6742 break; 6743 6744 nxmit++; 6745 } 6746 6747 if (flags & XDP_XMIT_FLUSH) { 6748 stmmac_flush_tx_descriptors(priv, queue); 6749 stmmac_tx_timer_arm(priv, queue); 6750 } 6751 6752 __netif_tx_unlock(nq); 6753 6754 return nxmit; 6755 } 6756 6757 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6758 { 6759 struct stmmac_channel *ch = &priv->channel[queue]; 6760 unsigned long flags; 6761 6762 spin_lock_irqsave(&ch->lock, flags); 6763 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6764 spin_unlock_irqrestore(&ch->lock, flags); 6765 6766 stmmac_stop_rx_dma(priv, queue); 6767 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6768 } 6769 6770 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6771 { 6772 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6773 struct stmmac_channel *ch = &priv->channel[queue]; 6774 unsigned long flags; 6775 u32 buf_size; 6776 int ret; 6777 6778 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6779 if (ret) { 6780 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6781 return; 6782 } 6783 6784 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 6785 if (ret) { 6786 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6787 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6788 return; 6789 } 6790 6791 stmmac_reset_rx_queue(priv, queue); 6792 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 6793 6794 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6795 rx_q->dma_rx_phy, rx_q->queue_index); 6796 6797 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6798 sizeof(struct dma_desc)); 6799 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6800 rx_q->rx_tail_addr, rx_q->queue_index); 6801 6802 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6803 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6804 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6805 buf_size, 6806 rx_q->queue_index); 6807 } else { 6808 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6809 priv->dma_conf.dma_buf_sz, 6810 rx_q->queue_index); 6811 } 6812 6813 stmmac_start_rx_dma(priv, queue); 6814 6815 spin_lock_irqsave(&ch->lock, flags); 6816 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6817 spin_unlock_irqrestore(&ch->lock, flags); 6818 } 6819 6820 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6821 { 6822 struct stmmac_channel *ch = &priv->channel[queue]; 6823 unsigned long flags; 6824 6825 spin_lock_irqsave(&ch->lock, flags); 6826 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6827 spin_unlock_irqrestore(&ch->lock, flags); 6828 6829 stmmac_stop_tx_dma(priv, queue); 6830 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6831 } 6832 6833 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6834 { 6835 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6836 struct stmmac_channel *ch = &priv->channel[queue]; 6837 unsigned long flags; 6838 int ret; 6839 6840 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6841 if (ret) { 6842 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6843 return; 6844 } 6845 6846 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 6847 if (ret) { 6848 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6849 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6850 return; 6851 } 6852 6853 stmmac_reset_tx_queue(priv, queue); 6854 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 6855 6856 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6857 tx_q->dma_tx_phy, tx_q->queue_index); 6858 6859 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6860 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6861 6862 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6863 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6864 tx_q->tx_tail_addr, tx_q->queue_index); 6865 6866 stmmac_start_tx_dma(priv, queue); 6867 6868 spin_lock_irqsave(&ch->lock, flags); 6869 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6870 spin_unlock_irqrestore(&ch->lock, flags); 6871 } 6872 6873 void stmmac_xdp_release(struct net_device *dev) 6874 { 6875 struct stmmac_priv *priv = netdev_priv(dev); 6876 u32 chan; 6877 6878 /* Ensure tx function is not running */ 6879 netif_tx_disable(dev); 6880 6881 /* Disable NAPI process */ 6882 stmmac_disable_all_queues(priv); 6883 6884 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6885 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6886 6887 /* Free the IRQ lines */ 6888 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 6889 6890 /* Stop TX/RX DMA channels */ 6891 stmmac_stop_all_dma(priv); 6892 6893 /* Release and free the Rx/Tx resources */ 6894 free_dma_desc_resources(priv, &priv->dma_conf); 6895 6896 /* Disable the MAC Rx/Tx */ 6897 stmmac_mac_set(priv, priv->ioaddr, false); 6898 6899 /* set trans_start so we don't get spurious 6900 * watchdogs during reset 6901 */ 6902 netif_trans_update(dev); 6903 netif_carrier_off(dev); 6904 } 6905 6906 int stmmac_xdp_open(struct net_device *dev) 6907 { 6908 struct stmmac_priv *priv = netdev_priv(dev); 6909 u32 rx_cnt = priv->plat->rx_queues_to_use; 6910 u32 tx_cnt = priv->plat->tx_queues_to_use; 6911 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 6912 struct stmmac_rx_queue *rx_q; 6913 struct stmmac_tx_queue *tx_q; 6914 u32 buf_size; 6915 bool sph_en; 6916 u32 chan; 6917 int ret; 6918 6919 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 6920 if (ret < 0) { 6921 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 6922 __func__); 6923 goto dma_desc_error; 6924 } 6925 6926 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 6927 if (ret < 0) { 6928 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 6929 __func__); 6930 goto init_error; 6931 } 6932 6933 stmmac_reset_queues_param(priv); 6934 6935 /* DMA CSR Channel configuration */ 6936 for (chan = 0; chan < dma_csr_ch; chan++) { 6937 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 6938 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6939 } 6940 6941 /* Adjust Split header */ 6942 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 6943 6944 /* DMA RX Channel Configuration */ 6945 for (chan = 0; chan < rx_cnt; chan++) { 6946 rx_q = &priv->dma_conf.rx_queue[chan]; 6947 6948 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6949 rx_q->dma_rx_phy, chan); 6950 6951 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 6952 (rx_q->buf_alloc_num * 6953 sizeof(struct dma_desc)); 6954 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6955 rx_q->rx_tail_addr, chan); 6956 6957 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6958 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6959 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6960 buf_size, 6961 rx_q->queue_index); 6962 } else { 6963 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6964 priv->dma_conf.dma_buf_sz, 6965 rx_q->queue_index); 6966 } 6967 6968 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6969 } 6970 6971 /* DMA TX Channel Configuration */ 6972 for (chan = 0; chan < tx_cnt; chan++) { 6973 tx_q = &priv->dma_conf.tx_queue[chan]; 6974 6975 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6976 tx_q->dma_tx_phy, chan); 6977 6978 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6979 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6980 tx_q->tx_tail_addr, chan); 6981 6982 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6983 tx_q->txtimer.function = stmmac_tx_timer; 6984 } 6985 6986 /* Enable the MAC Rx/Tx */ 6987 stmmac_mac_set(priv, priv->ioaddr, true); 6988 6989 /* Start Rx & Tx DMA Channels */ 6990 stmmac_start_all_dma(priv); 6991 6992 ret = stmmac_request_irq(dev); 6993 if (ret) 6994 goto irq_error; 6995 6996 /* Enable NAPI process*/ 6997 stmmac_enable_all_queues(priv); 6998 netif_carrier_on(dev); 6999 netif_tx_start_all_queues(dev); 7000 stmmac_enable_all_dma_irq(priv); 7001 7002 return 0; 7003 7004 irq_error: 7005 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7006 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7007 7008 stmmac_hw_teardown(dev); 7009 init_error: 7010 free_dma_desc_resources(priv, &priv->dma_conf); 7011 dma_desc_error: 7012 return ret; 7013 } 7014 7015 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 7016 { 7017 struct stmmac_priv *priv = netdev_priv(dev); 7018 struct stmmac_rx_queue *rx_q; 7019 struct stmmac_tx_queue *tx_q; 7020 struct stmmac_channel *ch; 7021 7022 if (test_bit(STMMAC_DOWN, &priv->state) || 7023 !netif_carrier_ok(priv->dev)) 7024 return -ENETDOWN; 7025 7026 if (!stmmac_xdp_is_enabled(priv)) 7027 return -EINVAL; 7028 7029 if (queue >= priv->plat->rx_queues_to_use || 7030 queue >= priv->plat->tx_queues_to_use) 7031 return -EINVAL; 7032 7033 rx_q = &priv->dma_conf.rx_queue[queue]; 7034 tx_q = &priv->dma_conf.tx_queue[queue]; 7035 ch = &priv->channel[queue]; 7036 7037 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 7038 return -EINVAL; 7039 7040 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 7041 /* EQoS does not have per-DMA channel SW interrupt, 7042 * so we schedule RX Napi straight-away. 7043 */ 7044 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 7045 __napi_schedule(&ch->rxtx_napi); 7046 } 7047 7048 return 0; 7049 } 7050 7051 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7052 { 7053 struct stmmac_priv *priv = netdev_priv(dev); 7054 u32 tx_cnt = priv->plat->tx_queues_to_use; 7055 u32 rx_cnt = priv->plat->rx_queues_to_use; 7056 unsigned int start; 7057 int q; 7058 7059 for (q = 0; q < tx_cnt; q++) { 7060 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7061 u64 tx_packets; 7062 u64 tx_bytes; 7063 7064 do { 7065 start = u64_stats_fetch_begin(&txq_stats->syncp); 7066 tx_packets = txq_stats->tx_packets; 7067 tx_bytes = txq_stats->tx_bytes; 7068 } while (u64_stats_fetch_retry(&txq_stats->syncp, start)); 7069 7070 stats->tx_packets += tx_packets; 7071 stats->tx_bytes += tx_bytes; 7072 } 7073 7074 for (q = 0; q < rx_cnt; q++) { 7075 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7076 u64 rx_packets; 7077 u64 rx_bytes; 7078 7079 do { 7080 start = u64_stats_fetch_begin(&rxq_stats->syncp); 7081 rx_packets = rxq_stats->rx_packets; 7082 rx_bytes = rxq_stats->rx_bytes; 7083 } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 7084 7085 stats->rx_packets += rx_packets; 7086 stats->rx_bytes += rx_bytes; 7087 } 7088 7089 stats->rx_dropped = priv->xstats.rx_dropped; 7090 stats->rx_errors = priv->xstats.rx_errors; 7091 stats->tx_dropped = priv->xstats.tx_dropped; 7092 stats->tx_errors = priv->xstats.tx_errors; 7093 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7094 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7095 stats->rx_length_errors = priv->xstats.rx_length; 7096 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7097 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7098 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7099 } 7100 7101 static const struct net_device_ops stmmac_netdev_ops = { 7102 .ndo_open = stmmac_open, 7103 .ndo_start_xmit = stmmac_xmit, 7104 .ndo_stop = stmmac_release, 7105 .ndo_change_mtu = stmmac_change_mtu, 7106 .ndo_fix_features = stmmac_fix_features, 7107 .ndo_set_features = stmmac_set_features, 7108 .ndo_set_rx_mode = stmmac_set_rx_mode, 7109 .ndo_tx_timeout = stmmac_tx_timeout, 7110 .ndo_eth_ioctl = stmmac_ioctl, 7111 .ndo_get_stats64 = stmmac_get_stats64, 7112 .ndo_setup_tc = stmmac_setup_tc, 7113 .ndo_select_queue = stmmac_select_queue, 7114 .ndo_set_mac_address = stmmac_set_mac_address, 7115 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7116 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7117 .ndo_bpf = stmmac_bpf, 7118 .ndo_xdp_xmit = stmmac_xdp_xmit, 7119 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7120 }; 7121 7122 static void stmmac_reset_subtask(struct stmmac_priv *priv) 7123 { 7124 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7125 return; 7126 if (test_bit(STMMAC_DOWN, &priv->state)) 7127 return; 7128 7129 netdev_err(priv->dev, "Reset adapter.\n"); 7130 7131 rtnl_lock(); 7132 netif_trans_update(priv->dev); 7133 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7134 usleep_range(1000, 2000); 7135 7136 set_bit(STMMAC_DOWN, &priv->state); 7137 dev_close(priv->dev); 7138 dev_open(priv->dev, NULL); 7139 clear_bit(STMMAC_DOWN, &priv->state); 7140 clear_bit(STMMAC_RESETING, &priv->state); 7141 rtnl_unlock(); 7142 } 7143 7144 static void stmmac_service_task(struct work_struct *work) 7145 { 7146 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7147 service_task); 7148 7149 stmmac_reset_subtask(priv); 7150 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7151 } 7152 7153 /** 7154 * stmmac_hw_init - Init the MAC device 7155 * @priv: driver private structure 7156 * Description: this function is to configure the MAC device according to 7157 * some platform parameters or the HW capability register. It prepares the 7158 * driver to use either ring or chain modes and to setup either enhanced or 7159 * normal descriptors. 7160 */ 7161 static int stmmac_hw_init(struct stmmac_priv *priv) 7162 { 7163 int ret; 7164 7165 /* dwmac-sun8i only work in chain mode */ 7166 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7167 chain_mode = 1; 7168 priv->chain_mode = chain_mode; 7169 7170 /* Initialize HW Interface */ 7171 ret = stmmac_hwif_init(priv); 7172 if (ret) 7173 return ret; 7174 7175 /* Get the HW capability (new GMAC newer than 3.50a) */ 7176 priv->hw_cap_support = stmmac_get_hw_features(priv); 7177 if (priv->hw_cap_support) { 7178 dev_info(priv->device, "DMA HW capability register supported\n"); 7179 7180 /* We can override some gmac/dma configuration fields: e.g. 7181 * enh_desc, tx_coe (e.g. that are passed through the 7182 * platform) with the values from the HW capability 7183 * register (if supported). 7184 */ 7185 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7186 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7187 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7188 priv->hw->pmt = priv->plat->pmt; 7189 if (priv->dma_cap.hash_tb_sz) { 7190 priv->hw->multicast_filter_bins = 7191 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7192 priv->hw->mcast_bits_log2 = 7193 ilog2(priv->hw->multicast_filter_bins); 7194 } 7195 7196 /* TXCOE doesn't work in thresh DMA mode */ 7197 if (priv->plat->force_thresh_dma_mode) 7198 priv->plat->tx_coe = 0; 7199 else 7200 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7201 7202 /* In case of GMAC4 rx_coe is from HW cap register. */ 7203 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7204 7205 if (priv->dma_cap.rx_coe_type2) 7206 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7207 else if (priv->dma_cap.rx_coe_type1) 7208 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7209 7210 } else { 7211 dev_info(priv->device, "No HW DMA feature register supported\n"); 7212 } 7213 7214 if (priv->plat->rx_coe) { 7215 priv->hw->rx_csum = priv->plat->rx_coe; 7216 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7217 if (priv->synopsys_id < DWMAC_CORE_4_00) 7218 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7219 } 7220 if (priv->plat->tx_coe) 7221 dev_info(priv->device, "TX Checksum insertion supported\n"); 7222 7223 if (priv->plat->pmt) { 7224 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7225 device_set_wakeup_capable(priv->device, 1); 7226 } 7227 7228 if (priv->dma_cap.tsoen) 7229 dev_info(priv->device, "TSO supported\n"); 7230 7231 priv->hw->vlan_fail_q_en = 7232 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7233 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7234 7235 /* Run HW quirks, if any */ 7236 if (priv->hwif_quirks) { 7237 ret = priv->hwif_quirks(priv); 7238 if (ret) 7239 return ret; 7240 } 7241 7242 /* Rx Watchdog is available in the COREs newer than the 3.40. 7243 * In some case, for example on bugged HW this feature 7244 * has to be disable and this can be done by passing the 7245 * riwt_off field from the platform. 7246 */ 7247 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 7248 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 7249 priv->use_riwt = 1; 7250 dev_info(priv->device, 7251 "Enable RX Mitigation via HW Watchdog Timer\n"); 7252 } 7253 7254 return 0; 7255 } 7256 7257 static void stmmac_napi_add(struct net_device *dev) 7258 { 7259 struct stmmac_priv *priv = netdev_priv(dev); 7260 u32 queue, maxq; 7261 7262 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7263 7264 for (queue = 0; queue < maxq; queue++) { 7265 struct stmmac_channel *ch = &priv->channel[queue]; 7266 7267 ch->priv_data = priv; 7268 ch->index = queue; 7269 spin_lock_init(&ch->lock); 7270 7271 if (queue < priv->plat->rx_queues_to_use) { 7272 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7273 } 7274 if (queue < priv->plat->tx_queues_to_use) { 7275 netif_napi_add_tx(dev, &ch->tx_napi, 7276 stmmac_napi_poll_tx); 7277 } 7278 if (queue < priv->plat->rx_queues_to_use && 7279 queue < priv->plat->tx_queues_to_use) { 7280 netif_napi_add(dev, &ch->rxtx_napi, 7281 stmmac_napi_poll_rxtx); 7282 } 7283 } 7284 } 7285 7286 static void stmmac_napi_del(struct net_device *dev) 7287 { 7288 struct stmmac_priv *priv = netdev_priv(dev); 7289 u32 queue, maxq; 7290 7291 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7292 7293 for (queue = 0; queue < maxq; queue++) { 7294 struct stmmac_channel *ch = &priv->channel[queue]; 7295 7296 if (queue < priv->plat->rx_queues_to_use) 7297 netif_napi_del(&ch->rx_napi); 7298 if (queue < priv->plat->tx_queues_to_use) 7299 netif_napi_del(&ch->tx_napi); 7300 if (queue < priv->plat->rx_queues_to_use && 7301 queue < priv->plat->tx_queues_to_use) { 7302 netif_napi_del(&ch->rxtx_napi); 7303 } 7304 } 7305 } 7306 7307 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 7308 { 7309 struct stmmac_priv *priv = netdev_priv(dev); 7310 int ret = 0, i; 7311 7312 if (netif_running(dev)) 7313 stmmac_release(dev); 7314 7315 stmmac_napi_del(dev); 7316 7317 priv->plat->rx_queues_to_use = rx_cnt; 7318 priv->plat->tx_queues_to_use = tx_cnt; 7319 if (!netif_is_rxfh_configured(dev)) 7320 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7321 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7322 rx_cnt); 7323 7324 stmmac_set_half_duplex(priv); 7325 stmmac_napi_add(dev); 7326 7327 if (netif_running(dev)) 7328 ret = stmmac_open(dev); 7329 7330 return ret; 7331 } 7332 7333 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7334 { 7335 struct stmmac_priv *priv = netdev_priv(dev); 7336 int ret = 0; 7337 7338 if (netif_running(dev)) 7339 stmmac_release(dev); 7340 7341 priv->dma_conf.dma_rx_size = rx_size; 7342 priv->dma_conf.dma_tx_size = tx_size; 7343 7344 if (netif_running(dev)) 7345 ret = stmmac_open(dev); 7346 7347 return ret; 7348 } 7349 7350 #define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n" 7351 static void stmmac_fpe_lp_task(struct work_struct *work) 7352 { 7353 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7354 fpe_task); 7355 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 7356 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 7357 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 7358 bool *hs_enable = &fpe_cfg->hs_enable; 7359 bool *enable = &fpe_cfg->enable; 7360 int retries = 20; 7361 7362 while (retries-- > 0) { 7363 /* Bail out immediately if FPE handshake is OFF */ 7364 if (*lo_state == FPE_STATE_OFF || !*hs_enable) 7365 break; 7366 7367 if (*lo_state == FPE_STATE_ENTERING_ON && 7368 *lp_state == FPE_STATE_ENTERING_ON) { 7369 stmmac_fpe_configure(priv, priv->ioaddr, 7370 fpe_cfg, 7371 priv->plat->tx_queues_to_use, 7372 priv->plat->rx_queues_to_use, 7373 *enable); 7374 7375 netdev_info(priv->dev, "configured FPE\n"); 7376 7377 *lo_state = FPE_STATE_ON; 7378 *lp_state = FPE_STATE_ON; 7379 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); 7380 break; 7381 } 7382 7383 if ((*lo_state == FPE_STATE_CAPABLE || 7384 *lo_state == FPE_STATE_ENTERING_ON) && 7385 *lp_state != FPE_STATE_ON) { 7386 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, 7387 *lo_state, *lp_state); 7388 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7389 fpe_cfg, 7390 MPACKET_VERIFY); 7391 } 7392 /* Sleep then retry */ 7393 msleep(500); 7394 } 7395 7396 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 7397 } 7398 7399 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) 7400 { 7401 if (priv->plat->fpe_cfg->hs_enable != enable) { 7402 if (enable) { 7403 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7404 priv->plat->fpe_cfg, 7405 MPACKET_VERIFY); 7406 } else { 7407 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; 7408 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; 7409 } 7410 7411 priv->plat->fpe_cfg->hs_enable = enable; 7412 } 7413 } 7414 7415 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7416 { 7417 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7418 struct dma_desc *desc_contains_ts = ctx->desc; 7419 struct stmmac_priv *priv = ctx->priv; 7420 struct dma_desc *ndesc = ctx->ndesc; 7421 struct dma_desc *desc = ctx->desc; 7422 u64 ns = 0; 7423 7424 if (!priv->hwts_rx_en) 7425 return -ENODATA; 7426 7427 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7428 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 7429 desc_contains_ts = ndesc; 7430 7431 /* Check if timestamp is available */ 7432 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7433 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7434 ns -= priv->plat->cdc_error_adj; 7435 *timestamp = ns_to_ktime(ns); 7436 return 0; 7437 } 7438 7439 return -ENODATA; 7440 } 7441 7442 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7443 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7444 }; 7445 7446 /** 7447 * stmmac_dvr_probe 7448 * @device: device pointer 7449 * @plat_dat: platform data pointer 7450 * @res: stmmac resource pointer 7451 * Description: this is the main probe function used to 7452 * call the alloc_etherdev, allocate the priv structure. 7453 * Return: 7454 * returns 0 on success, otherwise errno. 7455 */ 7456 int stmmac_dvr_probe(struct device *device, 7457 struct plat_stmmacenet_data *plat_dat, 7458 struct stmmac_resources *res) 7459 { 7460 struct net_device *ndev = NULL; 7461 struct stmmac_priv *priv; 7462 u32 rxq; 7463 int i, ret = 0; 7464 7465 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7466 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7467 if (!ndev) 7468 return -ENOMEM; 7469 7470 SET_NETDEV_DEV(ndev, device); 7471 7472 priv = netdev_priv(ndev); 7473 priv->device = device; 7474 priv->dev = ndev; 7475 7476 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7477 u64_stats_init(&priv->xstats.rxq_stats[i].syncp); 7478 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7479 u64_stats_init(&priv->xstats.txq_stats[i].syncp); 7480 7481 stmmac_set_ethtool_ops(ndev); 7482 priv->pause = pause; 7483 priv->plat = plat_dat; 7484 priv->ioaddr = res->addr; 7485 priv->dev->base_addr = (unsigned long)res->addr; 7486 priv->plat->dma_cfg->multi_msi_en = 7487 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7488 7489 priv->dev->irq = res->irq; 7490 priv->wol_irq = res->wol_irq; 7491 priv->lpi_irq = res->lpi_irq; 7492 priv->sfty_ce_irq = res->sfty_ce_irq; 7493 priv->sfty_ue_irq = res->sfty_ue_irq; 7494 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7495 priv->rx_irq[i] = res->rx_irq[i]; 7496 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7497 priv->tx_irq[i] = res->tx_irq[i]; 7498 7499 if (!is_zero_ether_addr(res->mac)) 7500 eth_hw_addr_set(priv->dev, res->mac); 7501 7502 dev_set_drvdata(device, priv->dev); 7503 7504 /* Verify driver arguments */ 7505 stmmac_verify_args(); 7506 7507 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7508 if (!priv->af_xdp_zc_qps) 7509 return -ENOMEM; 7510 7511 /* Allocate workqueue */ 7512 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7513 if (!priv->wq) { 7514 dev_err(priv->device, "failed to create workqueue\n"); 7515 ret = -ENOMEM; 7516 goto error_wq_init; 7517 } 7518 7519 INIT_WORK(&priv->service_task, stmmac_service_task); 7520 7521 /* Initialize Link Partner FPE workqueue */ 7522 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); 7523 7524 /* Override with kernel parameters if supplied XXX CRS XXX 7525 * this needs to have multiple instances 7526 */ 7527 if ((phyaddr >= 0) && (phyaddr <= 31)) 7528 priv->plat->phy_addr = phyaddr; 7529 7530 if (priv->plat->stmmac_rst) { 7531 ret = reset_control_assert(priv->plat->stmmac_rst); 7532 reset_control_deassert(priv->plat->stmmac_rst); 7533 /* Some reset controllers have only reset callback instead of 7534 * assert + deassert callbacks pair. 7535 */ 7536 if (ret == -ENOTSUPP) 7537 reset_control_reset(priv->plat->stmmac_rst); 7538 } 7539 7540 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7541 if (ret == -ENOTSUPP) 7542 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7543 ERR_PTR(ret)); 7544 7545 /* Wait a bit for the reset to take effect */ 7546 udelay(10); 7547 7548 /* Init MAC and get the capabilities */ 7549 ret = stmmac_hw_init(priv); 7550 if (ret) 7551 goto error_hw_init; 7552 7553 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7554 */ 7555 if (priv->synopsys_id < DWMAC_CORE_5_20) 7556 priv->plat->dma_cfg->dche = false; 7557 7558 stmmac_check_ether_addr(priv); 7559 7560 ndev->netdev_ops = &stmmac_netdev_ops; 7561 7562 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7563 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7564 7565 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7566 NETIF_F_RXCSUM; 7567 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7568 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7569 7570 ret = stmmac_tc_init(priv, priv); 7571 if (!ret) { 7572 ndev->hw_features |= NETIF_F_HW_TC; 7573 } 7574 7575 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 7576 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7577 if (priv->plat->has_gmac4) 7578 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7579 priv->tso = true; 7580 dev_info(priv->device, "TSO feature enabled\n"); 7581 } 7582 7583 if (priv->dma_cap.sphen && 7584 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7585 ndev->hw_features |= NETIF_F_GRO; 7586 priv->sph_cap = true; 7587 priv->sph = priv->sph_cap; 7588 dev_info(priv->device, "SPH feature enabled\n"); 7589 } 7590 7591 /* Ideally our host DMA address width is the same as for the 7592 * device. However, it may differ and then we have to use our 7593 * host DMA width for allocation and the device DMA width for 7594 * register handling. 7595 */ 7596 if (priv->plat->host_dma_width) 7597 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7598 else 7599 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7600 7601 if (priv->dma_cap.host_dma_width) { 7602 ret = dma_set_mask_and_coherent(device, 7603 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7604 if (!ret) { 7605 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7606 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7607 7608 /* 7609 * If more than 32 bits can be addressed, make sure to 7610 * enable enhanced addressing mode. 7611 */ 7612 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7613 priv->plat->dma_cfg->eame = true; 7614 } else { 7615 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7616 if (ret) { 7617 dev_err(priv->device, "Failed to set DMA Mask\n"); 7618 goto error_hw_init; 7619 } 7620 7621 priv->dma_cap.host_dma_width = 32; 7622 } 7623 } 7624 7625 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7626 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7627 #ifdef STMMAC_VLAN_TAG_USED 7628 /* Both mac100 and gmac support receive VLAN tag detection */ 7629 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7630 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 7631 priv->hw->hw_vlan_en = true; 7632 7633 if (priv->dma_cap.vlhash) { 7634 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7635 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7636 } 7637 if (priv->dma_cap.vlins) { 7638 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7639 if (priv->dma_cap.dvlan) 7640 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; 7641 } 7642 #endif 7643 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7644 7645 priv->xstats.threshold = tc; 7646 7647 /* Initialize RSS */ 7648 rxq = priv->plat->rx_queues_to_use; 7649 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7650 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7651 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7652 7653 if (priv->dma_cap.rssen && priv->plat->rss_en) 7654 ndev->features |= NETIF_F_RXHASH; 7655 7656 ndev->vlan_features |= ndev->features; 7657 /* TSO doesn't work on VLANs yet */ 7658 ndev->vlan_features &= ~NETIF_F_TSO; 7659 7660 /* MTU range: 46 - hw-specific max */ 7661 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7662 if (priv->plat->has_xgmac) 7663 ndev->max_mtu = XGMAC_JUMBO_LEN; 7664 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 7665 ndev->max_mtu = JUMBO_LEN; 7666 else 7667 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7668 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 7669 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 7670 */ 7671 if ((priv->plat->maxmtu < ndev->max_mtu) && 7672 (priv->plat->maxmtu >= ndev->min_mtu)) 7673 ndev->max_mtu = priv->plat->maxmtu; 7674 else if (priv->plat->maxmtu < ndev->min_mtu) 7675 dev_warn(priv->device, 7676 "%s: warning: maxmtu having invalid value (%d)\n", 7677 __func__, priv->plat->maxmtu); 7678 7679 if (flow_ctrl) 7680 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 7681 7682 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 7683 7684 /* Setup channels NAPI */ 7685 stmmac_napi_add(ndev); 7686 7687 mutex_init(&priv->lock); 7688 7689 /* If a specific clk_csr value is passed from the platform 7690 * this means that the CSR Clock Range selection cannot be 7691 * changed at run-time and it is fixed. Viceversa the driver'll try to 7692 * set the MDC clock dynamically according to the csr actual 7693 * clock input. 7694 */ 7695 if (priv->plat->clk_csr >= 0) 7696 priv->clk_csr = priv->plat->clk_csr; 7697 else 7698 stmmac_clk_csr_set(priv); 7699 7700 stmmac_check_pcs_mode(priv); 7701 7702 pm_runtime_get_noresume(device); 7703 pm_runtime_set_active(device); 7704 if (!pm_runtime_enabled(device)) 7705 pm_runtime_enable(device); 7706 7707 if (priv->hw->pcs != STMMAC_PCS_TBI && 7708 priv->hw->pcs != STMMAC_PCS_RTBI) { 7709 /* MDIO bus Registration */ 7710 ret = stmmac_mdio_register(ndev); 7711 if (ret < 0) { 7712 dev_err_probe(priv->device, ret, 7713 "%s: MDIO bus (id: %d) registration failed\n", 7714 __func__, priv->plat->bus_id); 7715 goto error_mdio_register; 7716 } 7717 } 7718 7719 if (priv->plat->speed_mode_2500) 7720 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); 7721 7722 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { 7723 ret = stmmac_xpcs_setup(priv->mii); 7724 if (ret) 7725 goto error_xpcs_setup; 7726 } 7727 7728 ret = stmmac_phy_setup(priv); 7729 if (ret) { 7730 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7731 goto error_phy_setup; 7732 } 7733 7734 ret = register_netdev(ndev); 7735 if (ret) { 7736 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7737 __func__, ret); 7738 goto error_netdev_register; 7739 } 7740 7741 #ifdef CONFIG_DEBUG_FS 7742 stmmac_init_fs(ndev); 7743 #endif 7744 7745 if (priv->plat->dump_debug_regs) 7746 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7747 7748 /* Let pm_runtime_put() disable the clocks. 7749 * If CONFIG_PM is not enabled, the clocks will stay powered. 7750 */ 7751 pm_runtime_put(device); 7752 7753 return ret; 7754 7755 error_netdev_register: 7756 phylink_destroy(priv->phylink); 7757 error_xpcs_setup: 7758 error_phy_setup: 7759 if (priv->hw->pcs != STMMAC_PCS_TBI && 7760 priv->hw->pcs != STMMAC_PCS_RTBI) 7761 stmmac_mdio_unregister(ndev); 7762 error_mdio_register: 7763 stmmac_napi_del(ndev); 7764 error_hw_init: 7765 destroy_workqueue(priv->wq); 7766 error_wq_init: 7767 bitmap_free(priv->af_xdp_zc_qps); 7768 7769 return ret; 7770 } 7771 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 7772 7773 /** 7774 * stmmac_dvr_remove 7775 * @dev: device pointer 7776 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 7777 * changes the link status, releases the DMA descriptor rings. 7778 */ 7779 void stmmac_dvr_remove(struct device *dev) 7780 { 7781 struct net_device *ndev = dev_get_drvdata(dev); 7782 struct stmmac_priv *priv = netdev_priv(ndev); 7783 7784 netdev_info(priv->dev, "%s: removing driver", __func__); 7785 7786 pm_runtime_get_sync(dev); 7787 7788 stmmac_stop_all_dma(priv); 7789 stmmac_mac_set(priv, priv->ioaddr, false); 7790 netif_carrier_off(ndev); 7791 unregister_netdev(ndev); 7792 7793 #ifdef CONFIG_DEBUG_FS 7794 stmmac_exit_fs(ndev); 7795 #endif 7796 phylink_destroy(priv->phylink); 7797 if (priv->plat->stmmac_rst) 7798 reset_control_assert(priv->plat->stmmac_rst); 7799 reset_control_assert(priv->plat->stmmac_ahb_rst); 7800 if (priv->hw->pcs != STMMAC_PCS_TBI && 7801 priv->hw->pcs != STMMAC_PCS_RTBI) 7802 stmmac_mdio_unregister(ndev); 7803 destroy_workqueue(priv->wq); 7804 mutex_destroy(&priv->lock); 7805 bitmap_free(priv->af_xdp_zc_qps); 7806 7807 pm_runtime_disable(dev); 7808 pm_runtime_put_noidle(dev); 7809 } 7810 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 7811 7812 /** 7813 * stmmac_suspend - suspend callback 7814 * @dev: device pointer 7815 * Description: this is the function to suspend the device and it is called 7816 * by the platform driver to stop the network queue, release the resources, 7817 * program the PMT register (for WoL), clean and release driver resources. 7818 */ 7819 int stmmac_suspend(struct device *dev) 7820 { 7821 struct net_device *ndev = dev_get_drvdata(dev); 7822 struct stmmac_priv *priv = netdev_priv(ndev); 7823 u32 chan; 7824 7825 if (!ndev || !netif_running(ndev)) 7826 return 0; 7827 7828 mutex_lock(&priv->lock); 7829 7830 netif_device_detach(ndev); 7831 7832 stmmac_disable_all_queues(priv); 7833 7834 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7835 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7836 7837 if (priv->eee_enabled) { 7838 priv->tx_path_in_lpi_mode = false; 7839 del_timer_sync(&priv->eee_ctrl_timer); 7840 } 7841 7842 /* Stop TX/RX DMA */ 7843 stmmac_stop_all_dma(priv); 7844 7845 if (priv->plat->serdes_powerdown) 7846 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7847 7848 /* Enable Power down mode by programming the PMT regs */ 7849 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7850 stmmac_pmt(priv, priv->hw, priv->wolopts); 7851 priv->irq_wake = 1; 7852 } else { 7853 stmmac_mac_set(priv, priv->ioaddr, false); 7854 pinctrl_pm_select_sleep_state(priv->device); 7855 } 7856 7857 mutex_unlock(&priv->lock); 7858 7859 rtnl_lock(); 7860 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7861 phylink_suspend(priv->phylink, true); 7862 } else { 7863 if (device_may_wakeup(priv->device)) 7864 phylink_speed_down(priv->phylink, false); 7865 phylink_suspend(priv->phylink, false); 7866 } 7867 rtnl_unlock(); 7868 7869 if (priv->dma_cap.fpesel) { 7870 /* Disable FPE */ 7871 stmmac_fpe_configure(priv, priv->ioaddr, 7872 priv->plat->fpe_cfg, 7873 priv->plat->tx_queues_to_use, 7874 priv->plat->rx_queues_to_use, false); 7875 7876 stmmac_fpe_handshake(priv, false); 7877 stmmac_fpe_stop_wq(priv); 7878 } 7879 7880 priv->speed = SPEED_UNKNOWN; 7881 return 0; 7882 } 7883 EXPORT_SYMBOL_GPL(stmmac_suspend); 7884 7885 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 7886 { 7887 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7888 7889 rx_q->cur_rx = 0; 7890 rx_q->dirty_rx = 0; 7891 } 7892 7893 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 7894 { 7895 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7896 7897 tx_q->cur_tx = 0; 7898 tx_q->dirty_tx = 0; 7899 tx_q->mss = 0; 7900 7901 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 7902 } 7903 7904 /** 7905 * stmmac_reset_queues_param - reset queue parameters 7906 * @priv: device pointer 7907 */ 7908 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 7909 { 7910 u32 rx_cnt = priv->plat->rx_queues_to_use; 7911 u32 tx_cnt = priv->plat->tx_queues_to_use; 7912 u32 queue; 7913 7914 for (queue = 0; queue < rx_cnt; queue++) 7915 stmmac_reset_rx_queue(priv, queue); 7916 7917 for (queue = 0; queue < tx_cnt; queue++) 7918 stmmac_reset_tx_queue(priv, queue); 7919 } 7920 7921 /** 7922 * stmmac_resume - resume callback 7923 * @dev: device pointer 7924 * Description: when resume this function is invoked to setup the DMA and CORE 7925 * in a usable state. 7926 */ 7927 int stmmac_resume(struct device *dev) 7928 { 7929 struct net_device *ndev = dev_get_drvdata(dev); 7930 struct stmmac_priv *priv = netdev_priv(ndev); 7931 int ret; 7932 7933 if (!netif_running(ndev)) 7934 return 0; 7935 7936 /* Power Down bit, into the PM register, is cleared 7937 * automatically as soon as a magic packet or a Wake-up frame 7938 * is received. Anyway, it's better to manually clear 7939 * this bit because it can generate problems while resuming 7940 * from another devices (e.g. serial console). 7941 */ 7942 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7943 mutex_lock(&priv->lock); 7944 stmmac_pmt(priv, priv->hw, 0); 7945 mutex_unlock(&priv->lock); 7946 priv->irq_wake = 0; 7947 } else { 7948 pinctrl_pm_select_default_state(priv->device); 7949 /* reset the phy so that it's ready */ 7950 if (priv->mii) 7951 stmmac_mdio_reset(priv->mii); 7952 } 7953 7954 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 7955 priv->plat->serdes_powerup) { 7956 ret = priv->plat->serdes_powerup(ndev, 7957 priv->plat->bsp_priv); 7958 7959 if (ret < 0) 7960 return ret; 7961 } 7962 7963 rtnl_lock(); 7964 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7965 phylink_resume(priv->phylink); 7966 } else { 7967 phylink_resume(priv->phylink); 7968 if (device_may_wakeup(priv->device)) 7969 phylink_speed_up(priv->phylink); 7970 } 7971 rtnl_unlock(); 7972 7973 rtnl_lock(); 7974 mutex_lock(&priv->lock); 7975 7976 stmmac_reset_queues_param(priv); 7977 7978 stmmac_free_tx_skbufs(priv); 7979 stmmac_clear_descriptors(priv, &priv->dma_conf); 7980 7981 stmmac_hw_setup(ndev, false); 7982 stmmac_init_coalesce(priv); 7983 stmmac_set_rx_mode(ndev); 7984 7985 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); 7986 7987 stmmac_enable_all_queues(priv); 7988 stmmac_enable_all_dma_irq(priv); 7989 7990 mutex_unlock(&priv->lock); 7991 rtnl_unlock(); 7992 7993 netif_device_attach(ndev); 7994 7995 return 0; 7996 } 7997 EXPORT_SYMBOL_GPL(stmmac_resume); 7998 7999 #ifndef MODULE 8000 static int __init stmmac_cmdline_opt(char *str) 8001 { 8002 char *opt; 8003 8004 if (!str || !*str) 8005 return 1; 8006 while ((opt = strsep(&str, ",")) != NULL) { 8007 if (!strncmp(opt, "debug:", 6)) { 8008 if (kstrtoint(opt + 6, 0, &debug)) 8009 goto err; 8010 } else if (!strncmp(opt, "phyaddr:", 8)) { 8011 if (kstrtoint(opt + 8, 0, &phyaddr)) 8012 goto err; 8013 } else if (!strncmp(opt, "buf_sz:", 7)) { 8014 if (kstrtoint(opt + 7, 0, &buf_sz)) 8015 goto err; 8016 } else if (!strncmp(opt, "tc:", 3)) { 8017 if (kstrtoint(opt + 3, 0, &tc)) 8018 goto err; 8019 } else if (!strncmp(opt, "watchdog:", 9)) { 8020 if (kstrtoint(opt + 9, 0, &watchdog)) 8021 goto err; 8022 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 8023 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 8024 goto err; 8025 } else if (!strncmp(opt, "pause:", 6)) { 8026 if (kstrtoint(opt + 6, 0, &pause)) 8027 goto err; 8028 } else if (!strncmp(opt, "eee_timer:", 10)) { 8029 if (kstrtoint(opt + 10, 0, &eee_timer)) 8030 goto err; 8031 } else if (!strncmp(opt, "chain_mode:", 11)) { 8032 if (kstrtoint(opt + 11, 0, &chain_mode)) 8033 goto err; 8034 } 8035 } 8036 return 1; 8037 8038 err: 8039 pr_err("%s: ERROR broken module parameter conversion", __func__); 8040 return 1; 8041 } 8042 8043 __setup("stmmaceth=", stmmac_cmdline_opt); 8044 #endif /* MODULE */ 8045 8046 static int __init stmmac_init(void) 8047 { 8048 #ifdef CONFIG_DEBUG_FS 8049 /* Create debugfs main directory if it doesn't exist yet */ 8050 if (!stmmac_fs_dir) 8051 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 8052 register_netdevice_notifier(&stmmac_notifier); 8053 #endif 8054 8055 return 0; 8056 } 8057 8058 static void __exit stmmac_exit(void) 8059 { 8060 #ifdef CONFIG_DEBUG_FS 8061 unregister_netdevice_notifier(&stmmac_notifier); 8062 debugfs_remove_recursive(stmmac_fs_dir); 8063 #endif 8064 } 8065 8066 module_init(stmmac_init) 8067 module_exit(stmmac_exit) 8068 8069 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 8070 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 8071 MODULE_LICENSE("GPL"); 8072