1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/circ_buf.h> 18 #include <linux/clk.h> 19 #include <linux/kernel.h> 20 #include <linux/interrupt.h> 21 #include <linux/ip.h> 22 #include <linux/tcp.h> 23 #include <linux/skbuff.h> 24 #include <linux/ethtool.h> 25 #include <linux/if_ether.h> 26 #include <linux/crc32.h> 27 #include <linux/mii.h> 28 #include <linux/if.h> 29 #include <linux/if_vlan.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/slab.h> 32 #include <linux/pm_runtime.h> 33 #include <linux/pm_wakeirq.h> 34 #include <linux/prefetch.h> 35 #include <linux/pinctrl/consumer.h> 36 #ifdef CONFIG_DEBUG_FS 37 #include <linux/debugfs.h> 38 #include <linux/seq_file.h> 39 #endif /* CONFIG_DEBUG_FS */ 40 #include <linux/net_tstamp.h> 41 #include <linux/phylink.h> 42 #include <linux/udp.h> 43 #include <linux/bpf_trace.h> 44 #include <net/devlink.h> 45 #include <net/page_pool/helpers.h> 46 #include <net/pkt_cls.h> 47 #include <net/xdp_sock_drv.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac_fpe.h" 50 #include "stmmac.h" 51 #include "stmmac_pcs.h" 52 #include "stmmac_xdp.h" 53 #include <linux/reset.h> 54 #include <linux/of_mdio.h> 55 #include "dwmac1000.h" 56 #include "dwxgmac2.h" 57 #include "hwif.h" 58 59 /* As long as the interface is active, we keep the timestamping counter enabled 60 * with fine resolution and binary rollover. This avoid non-monotonic behavior 61 * (clock jumps) when changing timestamping settings at runtime. 62 */ 63 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCTRLSSR) 64 65 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 66 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 67 68 /* Module parameters */ 69 #define TX_TIMEO 5000 70 static int watchdog = TX_TIMEO; 71 module_param(watchdog, int, 0644); 72 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 73 74 static int debug = -1; 75 module_param(debug, int, 0644); 76 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 77 78 static int phyaddr = -1; 79 module_param(phyaddr, int, 0444); 80 MODULE_PARM_DESC(phyaddr, "Physical device address"); 81 82 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 83 84 /* Limit to make sure XDP TX and slow path can coexist */ 85 #define STMMAC_XSK_TX_BUDGET_MAX 256 86 #define STMMAC_TX_XSK_AVAIL 16 87 #define STMMAC_RX_FILL_BATCH 16 88 89 #define STMMAC_XDP_PASS 0 90 #define STMMAC_XDP_CONSUMED BIT(0) 91 #define STMMAC_XDP_TX BIT(1) 92 #define STMMAC_XDP_REDIRECT BIT(2) 93 #define STMMAC_XSK_CONSUMED BIT(3) 94 95 static int flow_ctrl = 0xdead; 96 module_param(flow_ctrl, int, 0644); 97 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)"); 98 99 static int pause = PAUSE_TIME; 100 module_param(pause, int, 0644); 101 MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)"); 102 103 #define TC_DEFAULT 64 104 static int tc = TC_DEFAULT; 105 module_param(tc, int, 0644); 106 MODULE_PARM_DESC(tc, "DMA threshold control value"); 107 108 /* This is unused */ 109 #define DEFAULT_BUFSIZE 1536 110 static int buf_sz = DEFAULT_BUFSIZE; 111 module_param(buf_sz, int, 0644); 112 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 113 114 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 115 NETIF_MSG_LINK | NETIF_MSG_IFUP | 116 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 117 118 #define STMMAC_DEFAULT_LPI_TIMER 1000 119 static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 120 module_param(eee_timer, uint, 0644); 121 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 122 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 123 124 /* By default the driver will use the ring mode to manage tx and rx descriptors, 125 * but allow user to force to use the chain instead of the ring 126 */ 127 static unsigned int chain_mode; 128 module_param(chain_mode, int, 0444); 129 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 130 131 static const char *stmmac_dwmac_actphyif[8] = { 132 [PHY_INTF_SEL_GMII_MII] = "GMII/MII", 133 [PHY_INTF_SEL_RGMII] = "RGMII", 134 [PHY_INTF_SEL_SGMII] = "SGMII", 135 [PHY_INTF_SEL_TBI] = "TBI", 136 [PHY_INTF_SEL_RMII] = "RMII", 137 [PHY_INTF_SEL_RTBI] = "RTBI", 138 [PHY_INTF_SEL_SMII] = "SMII", 139 [PHY_INTF_SEL_REVMII] = "REVMII", 140 }; 141 142 static const char *stmmac_dwxgmac_phyif[4] = { 143 [PHY_INTF_GMII] = "GMII", 144 [PHY_INTF_RGMII] = "RGMII", 145 }; 146 147 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 148 /* For MSI interrupts handling */ 149 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 150 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 151 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 152 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 153 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 154 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 155 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 156 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 157 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 158 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 159 u32 rxmode, u32 chan); 160 static void stmmac_vlan_restore(struct stmmac_priv *priv); 161 162 #ifdef CONFIG_DEBUG_FS 163 static const struct net_device_ops stmmac_netdev_ops; 164 static void stmmac_init_fs(struct net_device *dev); 165 static void stmmac_exit_fs(struct net_device *dev); 166 #endif 167 168 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 169 170 struct stmmac_devlink_priv { 171 struct stmmac_priv *stmmac_priv; 172 }; 173 174 enum stmmac_dl_param_id { 175 STMMAC_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, 176 STMMAC_DEVLINK_PARAM_ID_TS_COARSE, 177 }; 178 179 /** 180 * stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock 181 * @bsp_priv: BSP private data structure (unused) 182 * @clk_tx_i: the transmit clock 183 * @interface: the selected interface mode 184 * @speed: the speed that the MAC will be operating at 185 * 186 * Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps, 187 * 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least 188 * MII, GMII, RGMII and RMII interface modes. Platforms can hook this into 189 * the plat_data->set_clk_tx_rate method directly, call it via their own 190 * implementation, or implement their own method should they have more 191 * complex requirements. It is intended to only be used in this method. 192 * 193 * plat_data->clk_tx_i must be filled in. 194 */ 195 int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, 196 phy_interface_t interface, int speed) 197 { 198 long rate = rgmii_clock(speed); 199 200 /* Silently ignore unsupported speeds as rgmii_clock() only 201 * supports 10, 100 and 1000Mbps. We do not want to spit 202 * errors for 2500 and higher speeds here. 203 */ 204 if (rate < 0) 205 return 0; 206 207 return clk_set_rate(clk_tx_i, rate); 208 } 209 EXPORT_SYMBOL_GPL(stmmac_set_clk_tx_rate); 210 211 /** 212 * stmmac_axi_blen_to_mask() - convert a burst length array to reg value 213 * @regval: pointer to a u32 for the resulting register value 214 * @blen: pointer to an array of u32 containing the burst length values in bytes 215 * @len: the number of entries in the @blen array 216 */ 217 void stmmac_axi_blen_to_mask(u32 *regval, const u32 *blen, size_t len) 218 { 219 size_t i; 220 u32 val; 221 222 for (val = i = 0; i < len; i++) { 223 u32 burst = blen[i]; 224 225 /* Burst values of zero must be skipped. */ 226 if (!burst) 227 continue; 228 229 /* The valid range for the burst length is 4 to 256 inclusive, 230 * and it must be a power of two. 231 */ 232 if (burst < 4 || burst > 256 || !is_power_of_2(burst)) { 233 pr_err("stmmac: invalid burst length %u at index %zu\n", 234 burst, i); 235 continue; 236 } 237 238 /* Since burst is a power of two, and the register field starts 239 * with burst = 4, shift right by two bits so bit 0 of the field 240 * corresponds with the minimum value. 241 */ 242 val |= burst >> 2; 243 } 244 245 *regval = FIELD_PREP(DMA_AXI_BLEN_MASK, val); 246 } 247 EXPORT_SYMBOL_GPL(stmmac_axi_blen_to_mask); 248 249 /** 250 * stmmac_verify_args - verify the driver parameters. 251 * Description: it checks the driver parameters and set a default in case of 252 * errors. 253 */ 254 static void stmmac_verify_args(void) 255 { 256 if (unlikely(watchdog < 0)) 257 watchdog = TX_TIMEO; 258 if (unlikely((pause < 0) || (pause > 0xffff))) 259 pause = PAUSE_TIME; 260 261 if (flow_ctrl != 0xdead) 262 pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n"); 263 } 264 265 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 266 { 267 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 268 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 269 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 270 u8 queue; 271 272 for (queue = 0; queue < maxq; queue++) { 273 struct stmmac_channel *ch = &priv->channel[queue]; 274 275 if (stmmac_xdp_is_enabled(priv) && 276 test_bit(queue, priv->af_xdp_zc_qps)) { 277 napi_disable(&ch->rxtx_napi); 278 continue; 279 } 280 281 if (queue < rx_queues_cnt) 282 napi_disable(&ch->rx_napi); 283 if (queue < tx_queues_cnt) 284 napi_disable(&ch->tx_napi); 285 } 286 } 287 288 /** 289 * stmmac_disable_all_queues - Disable all queues 290 * @priv: driver private structure 291 */ 292 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 293 { 294 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 295 struct stmmac_rx_queue *rx_q; 296 u8 queue; 297 298 /* synchronize_rcu() needed for pending XDP buffers to drain */ 299 for (queue = 0; queue < rx_queues_cnt; queue++) { 300 rx_q = &priv->dma_conf.rx_queue[queue]; 301 if (rx_q->xsk_pool) { 302 synchronize_rcu(); 303 break; 304 } 305 } 306 307 __stmmac_disable_all_queues(priv); 308 } 309 310 /** 311 * stmmac_enable_all_queues - Enable all queues 312 * @priv: driver private structure 313 */ 314 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 315 { 316 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 317 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 318 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 319 u8 queue; 320 321 for (queue = 0; queue < maxq; queue++) { 322 struct stmmac_channel *ch = &priv->channel[queue]; 323 324 if (stmmac_xdp_is_enabled(priv) && 325 test_bit(queue, priv->af_xdp_zc_qps)) { 326 napi_enable(&ch->rxtx_napi); 327 continue; 328 } 329 330 if (queue < rx_queues_cnt) 331 napi_enable(&ch->rx_napi); 332 if (queue < tx_queues_cnt) 333 napi_enable(&ch->tx_napi); 334 } 335 } 336 337 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 338 { 339 if (!test_bit(STMMAC_DOWN, &priv->state) && 340 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 341 queue_work(priv->wq, &priv->service_task); 342 } 343 344 static void stmmac_global_err(struct stmmac_priv *priv) 345 { 346 netif_carrier_off(priv->dev); 347 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 348 stmmac_service_event_schedule(priv); 349 } 350 351 static void print_pkt(unsigned char *buf, int len) 352 { 353 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 354 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 355 } 356 357 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 358 { 359 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 360 361 return CIRC_SPACE(tx_q->cur_tx, tx_q->dirty_tx, 362 priv->dma_conf.dma_tx_size); 363 } 364 365 static size_t stmmac_get_tx_desc_size(struct stmmac_priv *priv, 366 struct stmmac_tx_queue *tx_q) 367 { 368 if (priv->extend_desc) 369 return sizeof(struct dma_extended_desc); 370 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 371 return sizeof(struct dma_edesc); 372 else 373 return sizeof(struct dma_desc); 374 } 375 376 static struct dma_desc *stmmac_get_tx_desc(struct stmmac_priv *priv, 377 struct stmmac_tx_queue *tx_q, 378 unsigned int index) 379 { 380 if (priv->extend_desc) 381 return &tx_q->dma_etx[index].basic; 382 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 383 return &tx_q->dma_entx[index].basic; 384 else 385 return &tx_q->dma_tx[index]; 386 } 387 388 static void stmmac_set_queue_tx_tail_ptr(struct stmmac_priv *priv, 389 struct stmmac_tx_queue *tx_q, 390 unsigned int chan, unsigned int index) 391 { 392 size_t desc_size; 393 u32 tx_tail_addr; 394 395 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 396 397 tx_tail_addr = tx_q->dma_tx_phy + index * desc_size; 398 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_tail_addr, chan); 399 } 400 401 static size_t stmmac_get_rx_desc_size(struct stmmac_priv *priv) 402 { 403 if (priv->extend_desc) 404 return sizeof(struct dma_extended_desc); 405 else 406 return sizeof(struct dma_desc); 407 } 408 409 static struct dma_desc *stmmac_get_rx_desc(struct stmmac_priv *priv, 410 struct stmmac_rx_queue *rx_q, 411 unsigned int index) 412 { 413 if (priv->extend_desc) 414 return &rx_q->dma_erx[index].basic; 415 else 416 return &rx_q->dma_rx[index]; 417 } 418 419 static void stmmac_set_queue_rx_tail_ptr(struct stmmac_priv *priv, 420 struct stmmac_rx_queue *rx_q, 421 unsigned int chan, unsigned int index) 422 { 423 /* This only needs to deal with normal descriptors as enhanced 424 * descriptiors are only supported with dwmac1000 (<v4.0) which 425 * does not implement .set_rx_tail_ptr 426 */ 427 u32 rx_tail_addr = rx_q->dma_rx_phy + index * sizeof(struct dma_desc); 428 429 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_tail_addr, chan); 430 } 431 432 static void stmmac_set_queue_rx_buf_size(struct stmmac_priv *priv, 433 struct stmmac_rx_queue *rx_q, 434 unsigned int chan) 435 { 436 u32 buf_size; 437 438 if (rx_q->xsk_pool && rx_q->buf_alloc_num) 439 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 440 else 441 buf_size = priv->dma_conf.dma_buf_sz; 442 443 stmmac_set_dma_bfsize(priv, priv->ioaddr, buf_size, chan); 444 } 445 446 /** 447 * stmmac_rx_dirty - Get RX queue dirty 448 * @priv: driver private structure 449 * @queue: RX queue index 450 */ 451 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 452 { 453 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 454 455 return CIRC_CNT(rx_q->cur_rx, rx_q->dirty_rx, 456 priv->dma_conf.dma_rx_size); 457 } 458 459 static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) 460 { 461 u8 tx_cnt = priv->plat->tx_queues_to_use; 462 u8 queue; 463 464 /* check if all TX queues have the work finished */ 465 for (queue = 0; queue < tx_cnt; queue++) { 466 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 467 468 if (tx_q->dirty_tx != tx_q->cur_tx) 469 return true; /* still unfinished work */ 470 } 471 472 return false; 473 } 474 475 static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv) 476 { 477 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 478 } 479 480 /** 481 * stmmac_try_to_start_sw_lpi - check and enter in LPI mode 482 * @priv: driver private structure 483 * Description: this function is to verify and enter in LPI mode in case of 484 * EEE. 485 */ 486 static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv) 487 { 488 if (stmmac_eee_tx_busy(priv)) { 489 stmmac_restart_sw_lpi_timer(priv); 490 return; 491 } 492 493 /* Check and enter in LPI mode */ 494 if (!priv->tx_path_in_lpi_mode) 495 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED, 496 priv->tx_lpi_clk_stop, 0); 497 } 498 499 /** 500 * stmmac_stop_sw_lpi - stop transmitting LPI 501 * @priv: driver private structure 502 * Description: When using software-controlled LPI, stop transmitting LPI state. 503 */ 504 static void stmmac_stop_sw_lpi(struct stmmac_priv *priv) 505 { 506 timer_delete_sync(&priv->eee_ctrl_timer); 507 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 508 priv->tx_path_in_lpi_mode = false; 509 } 510 511 /** 512 * stmmac_eee_ctrl_timer - EEE TX SW timer. 513 * @t: timer_list struct containing private info 514 * Description: 515 * if there is no data transfer and if we are not in LPI state, 516 * then MAC Transmitter can be moved to LPI state. 517 */ 518 static void stmmac_eee_ctrl_timer(struct timer_list *t) 519 { 520 struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer); 521 522 stmmac_try_to_start_sw_lpi(priv); 523 } 524 525 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 526 * @priv: driver private structure 527 * @p : descriptor pointer 528 * @skb : the socket buffer 529 * Description : 530 * This function will read timestamp from the descriptor & pass it to stack. 531 * and also perform some sanity checks. 532 */ 533 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 534 struct dma_desc *p, struct sk_buff *skb) 535 { 536 struct skb_shared_hwtstamps shhwtstamp; 537 bool found = false; 538 u64 ns = 0; 539 540 if (!priv->hwts_tx_en) 541 return; 542 543 /* exit if skb doesn't support hw tstamp */ 544 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 545 return; 546 547 /* check tx tstamp status */ 548 if (stmmac_get_tx_timestamp_status(priv, p)) { 549 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 550 found = true; 551 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 552 found = true; 553 } 554 555 if (found) { 556 ns -= priv->plat->cdc_error_adj; 557 558 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 559 shhwtstamp.hwtstamp = ns_to_ktime(ns); 560 561 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 562 /* pass tstamp to stack */ 563 skb_tstamp_tx(skb, &shhwtstamp); 564 } 565 } 566 567 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 568 * @priv: driver private structure 569 * @p : descriptor pointer 570 * @np : next descriptor pointer 571 * @skb : the socket buffer 572 * Description : 573 * This function will read received packet's timestamp from the descriptor 574 * and pass it to stack. It also perform some sanity checks. 575 */ 576 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 577 struct dma_desc *np, struct sk_buff *skb) 578 { 579 struct skb_shared_hwtstamps *shhwtstamp = NULL; 580 struct dma_desc *desc = p; 581 u64 ns = 0; 582 583 if (!priv->hwts_rx_en) 584 return; 585 /* For GMAC4, the valid timestamp is from CTX next desc. */ 586 if (dwmac_is_xmac(priv->plat->core_type)) 587 desc = np; 588 589 /* Check if timestamp is available */ 590 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 591 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 592 593 ns -= priv->plat->cdc_error_adj; 594 595 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 596 shhwtstamp = skb_hwtstamps(skb); 597 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 598 shhwtstamp->hwtstamp = ns_to_ktime(ns); 599 } else { 600 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 601 } 602 } 603 604 static void stmmac_update_subsecond_increment(struct stmmac_priv *priv) 605 { 606 bool xmac = dwmac_is_xmac(priv->plat->core_type); 607 u32 sec_inc = 0; 608 u64 temp = 0; 609 610 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 611 612 /* program Sub Second Increment reg */ 613 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 614 priv->plat->clk_ptp_rate, 615 xmac, &sec_inc); 616 temp = div_u64(1000000000ULL, sec_inc); 617 618 /* Store sub second increment for later use */ 619 priv->sub_second_inc = sec_inc; 620 621 /* calculate default added value: 622 * formula is : 623 * addend = (2^32)/freq_div_ratio; 624 * where, freq_div_ratio = 1e9ns/sec_inc 625 */ 626 temp = (u64)(temp << 32); 627 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 628 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 629 } 630 631 /** 632 * stmmac_hwtstamp_set - control hardware timestamping. 633 * @dev: device pointer. 634 * @config: the timestamping configuration. 635 * @extack: netlink extended ack structure for error reporting. 636 * Description: 637 * This function configures the MAC to enable/disable both outgoing(TX) 638 * and incoming(RX) packets time stamping based on user input. 639 * Return Value: 640 * 0 on success and an appropriate -ve integer on failure. 641 */ 642 static int stmmac_hwtstamp_set(struct net_device *dev, 643 struct kernel_hwtstamp_config *config, 644 struct netlink_ext_ack *extack) 645 { 646 struct stmmac_priv *priv = netdev_priv(dev); 647 u32 ptp_v2 = 0; 648 u32 tstamp_all = 0; 649 u32 ptp_over_ipv4_udp = 0; 650 u32 ptp_over_ipv6_udp = 0; 651 u32 ptp_over_ethernet = 0; 652 u32 snap_type_sel = 0; 653 u32 ts_master_en = 0; 654 u32 ts_event_en = 0; 655 656 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 657 NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping"); 658 priv->hwts_tx_en = 0; 659 priv->hwts_rx_en = 0; 660 661 return -EOPNOTSUPP; 662 } 663 664 if (!netif_running(dev)) { 665 NL_SET_ERR_MSG_MOD(extack, 666 "Cannot change timestamping configuration while down"); 667 return -ENODEV; 668 } 669 670 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 671 __func__, config->flags, config->tx_type, config->rx_filter); 672 673 if (config->tx_type != HWTSTAMP_TX_OFF && 674 config->tx_type != HWTSTAMP_TX_ON) 675 return -ERANGE; 676 677 if (priv->adv_ts) { 678 switch (config->rx_filter) { 679 case HWTSTAMP_FILTER_NONE: 680 /* time stamp no incoming packet at all */ 681 config->rx_filter = HWTSTAMP_FILTER_NONE; 682 break; 683 684 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 685 /* PTP v1, UDP, any kind of event packet */ 686 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 687 /* 'xmac' hardware can support Sync, Pdelay_Req and 688 * Pdelay_resp by setting bit14 and bits17/16 to 01 689 * This leaves Delay_Req timestamps out. 690 * Enable all events *and* general purpose message 691 * timestamping 692 */ 693 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 694 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 695 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 696 break; 697 698 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 699 /* PTP v1, UDP, Sync packet */ 700 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 701 /* take time stamp for SYNC messages only */ 702 ts_event_en = PTP_TCR_TSEVNTENA; 703 704 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 705 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 706 break; 707 708 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 709 /* PTP v1, UDP, Delay_req packet */ 710 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 711 /* take time stamp for Delay_Req messages only */ 712 ts_master_en = PTP_TCR_TSMSTRENA; 713 ts_event_en = PTP_TCR_TSEVNTENA; 714 715 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 716 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 717 break; 718 719 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 720 /* PTP v2, UDP, any kind of event packet */ 721 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 722 ptp_v2 = PTP_TCR_TSVER2ENA; 723 /* take time stamp for all event messages */ 724 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 725 726 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 727 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 728 break; 729 730 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 731 /* PTP v2, UDP, Sync packet */ 732 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 733 ptp_v2 = PTP_TCR_TSVER2ENA; 734 /* take time stamp for SYNC messages only */ 735 ts_event_en = PTP_TCR_TSEVNTENA; 736 737 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 738 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 739 break; 740 741 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 742 /* PTP v2, UDP, Delay_req packet */ 743 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 744 ptp_v2 = PTP_TCR_TSVER2ENA; 745 /* take time stamp for Delay_Req messages only */ 746 ts_master_en = PTP_TCR_TSMSTRENA; 747 ts_event_en = PTP_TCR_TSEVNTENA; 748 749 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 750 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 751 break; 752 753 case HWTSTAMP_FILTER_PTP_V2_EVENT: 754 /* PTP v2/802.AS1 any layer, any kind of event packet */ 755 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 756 ptp_v2 = PTP_TCR_TSVER2ENA; 757 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 758 if (priv->synopsys_id < DWMAC_CORE_3_70 && 759 priv->plat->core_type != DWMAC_CORE_XGMAC) 760 ts_event_en = PTP_TCR_TSEVNTENA; 761 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 762 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 763 ptp_over_ethernet = PTP_TCR_TSIPENA; 764 break; 765 766 case HWTSTAMP_FILTER_PTP_V2_SYNC: 767 /* PTP v2/802.AS1, any layer, Sync packet */ 768 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 769 ptp_v2 = PTP_TCR_TSVER2ENA; 770 /* take time stamp for SYNC messages only */ 771 ts_event_en = PTP_TCR_TSEVNTENA; 772 773 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 774 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 775 ptp_over_ethernet = PTP_TCR_TSIPENA; 776 break; 777 778 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 779 /* PTP v2/802.AS1, any layer, Delay_req packet */ 780 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 781 ptp_v2 = PTP_TCR_TSVER2ENA; 782 /* take time stamp for Delay_Req messages only */ 783 ts_master_en = PTP_TCR_TSMSTRENA; 784 ts_event_en = PTP_TCR_TSEVNTENA; 785 786 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 787 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 788 ptp_over_ethernet = PTP_TCR_TSIPENA; 789 break; 790 791 case HWTSTAMP_FILTER_NTP_ALL: 792 case HWTSTAMP_FILTER_ALL: 793 /* time stamp any incoming packet */ 794 config->rx_filter = HWTSTAMP_FILTER_ALL; 795 tstamp_all = PTP_TCR_TSENALL; 796 break; 797 798 default: 799 return -ERANGE; 800 } 801 } else { 802 switch (config->rx_filter) { 803 case HWTSTAMP_FILTER_NONE: 804 config->rx_filter = HWTSTAMP_FILTER_NONE; 805 break; 806 default: 807 /* PTP v1, UDP, any kind of event packet */ 808 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 809 break; 810 } 811 } 812 priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE; 813 priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON; 814 815 priv->systime_flags = STMMAC_HWTS_ACTIVE; 816 if (!priv->tsfupdt_coarse) 817 priv->systime_flags |= PTP_TCR_TSCFUPDT; 818 819 if (priv->hwts_tx_en || priv->hwts_rx_en) { 820 priv->systime_flags |= tstamp_all | ptp_v2 | 821 ptp_over_ethernet | ptp_over_ipv6_udp | 822 ptp_over_ipv4_udp | ts_event_en | 823 ts_master_en | snap_type_sel; 824 } 825 826 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 827 828 priv->tstamp_config = *config; 829 830 return 0; 831 } 832 833 /** 834 * stmmac_hwtstamp_get - read hardware timestamping. 835 * @dev: device pointer. 836 * @config: the timestamping configuration. 837 * Description: 838 * This function obtain the current hardware timestamping settings 839 * as requested. 840 */ 841 static int stmmac_hwtstamp_get(struct net_device *dev, 842 struct kernel_hwtstamp_config *config) 843 { 844 struct stmmac_priv *priv = netdev_priv(dev); 845 846 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 847 return -EOPNOTSUPP; 848 849 *config = priv->tstamp_config; 850 851 return 0; 852 } 853 854 /** 855 * stmmac_init_tstamp_counter - init hardware timestamping counter 856 * @priv: driver private structure 857 * @systime_flags: timestamping flags 858 * Description: 859 * Initialize hardware counter for packet timestamping. 860 * This is valid as long as the interface is open and not suspended. 861 * Will be rerun after resuming from suspend, case in which the timestamping 862 * flags updated by stmmac_hwtstamp_set() also need to be restored. 863 */ 864 static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, 865 u32 systime_flags) 866 { 867 struct timespec64 now; 868 869 if (!priv->plat->clk_ptp_rate) { 870 netdev_err(priv->dev, "Invalid PTP clock rate"); 871 return -EINVAL; 872 } 873 874 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 875 priv->systime_flags = systime_flags; 876 877 stmmac_update_subsecond_increment(priv); 878 879 /* initialize system time */ 880 ktime_get_real_ts64(&now); 881 882 /* lower 32 bits of tv_sec are safe until y2106 */ 883 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 884 885 return 0; 886 } 887 888 /** 889 * stmmac_init_timestamping - initialise timestamping 890 * @priv: driver private structure 891 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 892 * This is done by looking at the HW cap. register. 893 * This function also registers the ptp driver. 894 */ 895 static int stmmac_init_timestamping(struct stmmac_priv *priv) 896 { 897 bool xmac = dwmac_is_xmac(priv->plat->core_type); 898 int ret; 899 900 if (priv->plat->ptp_clk_freq_config) 901 priv->plat->ptp_clk_freq_config(priv); 902 903 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) { 904 netdev_info(priv->dev, "PTP not supported by HW\n"); 905 return -EOPNOTSUPP; 906 } 907 908 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE | 909 PTP_TCR_TSCFUPDT); 910 if (ret) { 911 netdev_warn(priv->dev, "PTP init failed\n"); 912 return ret; 913 } 914 915 priv->adv_ts = 0; 916 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 917 if (xmac && priv->dma_cap.atime_stamp) 918 priv->adv_ts = 1; 919 /* Dwmac 3.x core with extend_desc can support adv_ts */ 920 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 921 priv->adv_ts = 1; 922 923 if (priv->dma_cap.time_stamp) 924 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 925 926 if (priv->adv_ts) 927 netdev_info(priv->dev, 928 "IEEE 1588-2008 Advanced Timestamp supported\n"); 929 930 memset(&priv->tstamp_config, 0, sizeof(priv->tstamp_config)); 931 priv->hwts_tx_en = 0; 932 priv->hwts_rx_en = 0; 933 934 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 935 stmmac_hwtstamp_correct_latency(priv, priv); 936 937 return 0; 938 } 939 940 static void stmmac_setup_ptp(struct stmmac_priv *priv) 941 { 942 int ret; 943 944 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 945 if (ret < 0) 946 netdev_warn(priv->dev, 947 "failed to enable PTP reference clock: %pe\n", 948 ERR_PTR(ret)); 949 950 if (stmmac_init_timestamping(priv) == 0) 951 stmmac_ptp_register(priv); 952 } 953 954 static void stmmac_release_ptp(struct stmmac_priv *priv) 955 { 956 stmmac_ptp_unregister(priv); 957 clk_disable_unprepare(priv->plat->clk_ptp_ref); 958 } 959 960 static void stmmac_legacy_serdes_power_down(struct stmmac_priv *priv) 961 { 962 if (priv->plat->serdes_powerdown && priv->legacy_serdes_is_powered) 963 priv->plat->serdes_powerdown(priv->dev, priv->plat->bsp_priv); 964 965 priv->legacy_serdes_is_powered = false; 966 } 967 968 static int stmmac_legacy_serdes_power_up(struct stmmac_priv *priv) 969 { 970 int ret; 971 972 if (!priv->plat->serdes_powerup) 973 return 0; 974 975 ret = priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 976 if (ret < 0) 977 netdev_err(priv->dev, "SerDes powerup failed\n"); 978 else 979 priv->legacy_serdes_is_powered = true; 980 981 return ret; 982 } 983 984 /** 985 * stmmac_mac_flow_ctrl - Configure flow control in all queues 986 * @priv: driver private structure 987 * @duplex: duplex passed to the next function 988 * @flow_ctrl: desired flow control modes 989 * Description: It is used for configuring the flow control in all queues 990 */ 991 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex, 992 unsigned int flow_ctrl) 993 { 994 u8 tx_cnt = priv->plat->tx_queues_to_use; 995 996 stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time, 997 tx_cnt); 998 } 999 1000 static unsigned long stmmac_mac_get_caps(struct phylink_config *config, 1001 phy_interface_t interface) 1002 { 1003 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1004 1005 /* Refresh the MAC-specific capabilities */ 1006 stmmac_mac_update_caps(priv); 1007 1008 if (priv->hw_cap_support && !priv->dma_cap.half_duplex) 1009 priv->hw->link.caps &= ~(MAC_1000HD | MAC_100HD | MAC_10HD); 1010 1011 config->mac_capabilities = priv->hw->link.caps; 1012 1013 if (priv->plat->max_speed) 1014 phylink_limit_mac_speed(config, priv->plat->max_speed); 1015 1016 return config->mac_capabilities; 1017 } 1018 1019 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 1020 phy_interface_t interface) 1021 { 1022 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1023 struct phylink_pcs *pcs; 1024 1025 if (priv->plat->select_pcs) { 1026 pcs = priv->plat->select_pcs(priv, interface); 1027 if (!IS_ERR(pcs)) 1028 return pcs; 1029 } 1030 1031 if (priv->integrated_pcs && 1032 test_bit(interface, priv->integrated_pcs->pcs.supported_interfaces)) 1033 return &priv->integrated_pcs->pcs; 1034 1035 return NULL; 1036 } 1037 1038 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 1039 const struct phylink_link_state *state) 1040 { 1041 /* Nothing to do, xpcs_config() handles everything */ 1042 } 1043 1044 static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode, 1045 phy_interface_t interface) 1046 { 1047 struct net_device *ndev = to_net_dev(config->dev); 1048 struct stmmac_priv *priv = netdev_priv(ndev); 1049 1050 if (priv->plat->mac_finish) 1051 priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, 1052 interface); 1053 1054 return 0; 1055 } 1056 1057 static void stmmac_mac_link_down(struct phylink_config *config, 1058 unsigned int mode, phy_interface_t interface) 1059 { 1060 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1061 1062 stmmac_mac_set(priv, priv->ioaddr, false); 1063 if (priv->dma_cap.eee) 1064 stmmac_set_eee_pls(priv, priv->hw, false); 1065 1066 if (stmmac_fpe_supported(priv)) 1067 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, false); 1068 } 1069 1070 static void stmmac_mac_link_up(struct phylink_config *config, 1071 struct phy_device *phy, 1072 unsigned int mode, phy_interface_t interface, 1073 int speed, int duplex, 1074 bool tx_pause, bool rx_pause) 1075 { 1076 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1077 unsigned int flow_ctrl; 1078 u32 old_ctrl, ctrl; 1079 int ret; 1080 1081 if (priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) 1082 stmmac_legacy_serdes_power_up(priv); 1083 1084 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1085 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1086 1087 switch (speed) { 1088 case SPEED_100000: 1089 ctrl |= priv->hw->link.xlgmii.speed100000; 1090 break; 1091 case SPEED_50000: 1092 ctrl |= priv->hw->link.xlgmii.speed50000; 1093 break; 1094 case SPEED_40000: 1095 ctrl |= priv->hw->link.xlgmii.speed40000; 1096 break; 1097 case SPEED_25000: 1098 ctrl |= priv->hw->link.xlgmii.speed25000; 1099 break; 1100 case SPEED_10000: 1101 ctrl |= priv->hw->link.xgmii.speed10000; 1102 break; 1103 case SPEED_5000: 1104 ctrl |= priv->hw->link.xgmii.speed5000; 1105 break; 1106 case SPEED_2500: 1107 if (interface == PHY_INTERFACE_MODE_USXGMII) 1108 ctrl |= priv->hw->link.xgmii.speed2500; 1109 else 1110 ctrl |= priv->hw->link.speed2500; 1111 break; 1112 case SPEED_1000: 1113 ctrl |= priv->hw->link.speed1000; 1114 break; 1115 case SPEED_100: 1116 ctrl |= priv->hw->link.speed100; 1117 break; 1118 case SPEED_10: 1119 ctrl |= priv->hw->link.speed10; 1120 break; 1121 default: 1122 netdev_err(priv->dev, 1123 "unsupported speed %s on %s, leaving the MAC disabled\n", 1124 phy_speed_to_str(speed), phy_modes(interface)); 1125 return; 1126 } 1127 1128 if (priv->plat->fix_mac_speed) 1129 priv->plat->fix_mac_speed(priv->plat->bsp_priv, interface, 1130 speed, mode); 1131 1132 if (!duplex) 1133 ctrl &= ~priv->hw->link.duplex; 1134 else 1135 ctrl |= priv->hw->link.duplex; 1136 1137 /* Flow Control operation */ 1138 if (rx_pause && tx_pause) 1139 flow_ctrl = FLOW_AUTO; 1140 else if (rx_pause && !tx_pause) 1141 flow_ctrl = FLOW_RX; 1142 else if (!rx_pause && tx_pause) 1143 flow_ctrl = FLOW_TX; 1144 else 1145 flow_ctrl = FLOW_OFF; 1146 1147 stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl); 1148 1149 if (ctrl != old_ctrl) 1150 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1151 1152 if (priv->plat->set_clk_tx_rate) { 1153 ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv, 1154 priv->plat->clk_tx_i, 1155 interface, speed); 1156 if (ret < 0) 1157 netdev_err(priv->dev, 1158 "failed to configure %s transmit clock for %dMbps: %pe\n", 1159 phy_modes(interface), speed, ERR_PTR(ret)); 1160 } 1161 1162 stmmac_mac_set(priv, priv->ioaddr, true); 1163 if (priv->dma_cap.eee) 1164 stmmac_set_eee_pls(priv, priv->hw, true); 1165 1166 if (stmmac_fpe_supported(priv)) 1167 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, true); 1168 1169 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1170 stmmac_hwtstamp_correct_latency(priv, priv); 1171 } 1172 1173 static void stmmac_mac_disable_tx_lpi(struct phylink_config *config) 1174 { 1175 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1176 1177 priv->eee_active = false; 1178 1179 mutex_lock(&priv->lock); 1180 1181 priv->eee_enabled = false; 1182 1183 netdev_dbg(priv->dev, "disable EEE\n"); 1184 priv->eee_sw_timer_en = false; 1185 timer_delete_sync(&priv->eee_ctrl_timer); 1186 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 1187 priv->tx_path_in_lpi_mode = false; 1188 1189 stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS); 1190 mutex_unlock(&priv->lock); 1191 } 1192 1193 static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 1194 bool tx_clk_stop) 1195 { 1196 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1197 int ret; 1198 1199 priv->tx_lpi_timer = timer; 1200 priv->eee_active = true; 1201 1202 mutex_lock(&priv->lock); 1203 1204 priv->eee_enabled = true; 1205 1206 /* Update the transmit clock stop according to PHY capability if 1207 * the platform allows 1208 */ 1209 if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP) 1210 priv->tx_lpi_clk_stop = tx_clk_stop; 1211 1212 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 1213 STMMAC_DEFAULT_TWT_LS); 1214 1215 /* Try to configure the hardware timer. */ 1216 ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER, 1217 priv->tx_lpi_clk_stop, priv->tx_lpi_timer); 1218 1219 if (ret) { 1220 /* Hardware timer mode not supported, or value out of range. 1221 * Fall back to using software LPI mode 1222 */ 1223 priv->eee_sw_timer_en = true; 1224 stmmac_restart_sw_lpi_timer(priv); 1225 } 1226 1227 mutex_unlock(&priv->lock); 1228 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 1229 1230 return 0; 1231 } 1232 1233 static int stmmac_mac_wol_set(struct phylink_config *config, u32 wolopts, 1234 const u8 *sopass) 1235 { 1236 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1237 1238 device_set_wakeup_enable(priv->device, !!wolopts); 1239 1240 mutex_lock(&priv->lock); 1241 priv->wolopts = wolopts; 1242 mutex_unlock(&priv->lock); 1243 1244 return 0; 1245 } 1246 1247 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1248 .mac_get_caps = stmmac_mac_get_caps, 1249 .mac_select_pcs = stmmac_mac_select_pcs, 1250 .mac_config = stmmac_mac_config, 1251 .mac_finish = stmmac_mac_finish, 1252 .mac_link_down = stmmac_mac_link_down, 1253 .mac_link_up = stmmac_mac_link_up, 1254 .mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi, 1255 .mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi, 1256 .mac_wol_set = stmmac_mac_wol_set, 1257 }; 1258 1259 /** 1260 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1261 * @priv: driver private structure 1262 * Description: this is to verify if the HW supports the PCS. 1263 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1264 * configured for the TBI, RTBI, or SGMII PHY interface. 1265 */ 1266 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1267 { 1268 int interface = priv->plat->phy_interface; 1269 int speed = priv->plat->mac_port_sel_speed; 1270 1271 if (priv->dma_cap.pcs && interface == PHY_INTERFACE_MODE_SGMII) { 1272 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1273 1274 switch (speed) { 1275 case SPEED_10: 1276 case SPEED_100: 1277 case SPEED_1000: 1278 priv->hw->reverse_sgmii_enable = true; 1279 break; 1280 1281 default: 1282 dev_warn(priv->device, "invalid port speed\n"); 1283 fallthrough; 1284 case 0: 1285 priv->hw->reverse_sgmii_enable = false; 1286 break; 1287 } 1288 } 1289 } 1290 1291 /** 1292 * stmmac_init_phy - PHY initialization 1293 * @dev: net device structure 1294 * Description: it initializes the driver's PHY state, and attaches the PHY 1295 * to the mac driver. 1296 * Return value: 1297 * 0 on success 1298 */ 1299 static int stmmac_init_phy(struct net_device *dev) 1300 { 1301 struct stmmac_priv *priv = netdev_priv(dev); 1302 int mode = priv->plat->phy_interface; 1303 struct fwnode_handle *phy_fwnode; 1304 struct fwnode_handle *fwnode; 1305 struct ethtool_keee eee; 1306 u32 dev_flags = 0; 1307 int ret; 1308 1309 if (!phylink_expects_phy(priv->phylink)) 1310 return 0; 1311 1312 if (priv->hw->xpcs && 1313 xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73) 1314 return 0; 1315 1316 fwnode = dev_fwnode(priv->device); 1317 if (fwnode) 1318 phy_fwnode = fwnode_get_phy_node(fwnode); 1319 else 1320 phy_fwnode = NULL; 1321 1322 if (priv->plat->flags & STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD) 1323 dev_flags |= PHY_F_KEEP_PREAMBLE_BEFORE_SFD; 1324 1325 /* Some DT bindings do not set-up the PHY handle. Let's try to 1326 * manually parse it 1327 */ 1328 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1329 int addr = priv->plat->phy_addr; 1330 struct phy_device *phydev; 1331 1332 if (addr < 0) { 1333 /* If a custom PCS is in use, no PHY is needed */ 1334 if (priv->hw->phylink_pcs) 1335 return 0; 1336 1337 netdev_err(priv->dev, "no phy found\n"); 1338 return -ENODEV; 1339 } 1340 1341 phydev = mdiobus_get_phy(priv->mii, addr); 1342 if (!phydev) { 1343 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1344 return -ENODEV; 1345 } 1346 1347 phydev->dev_flags |= dev_flags; 1348 1349 ret = phylink_connect_phy(priv->phylink, phydev); 1350 } else { 1351 fwnode_handle_put(phy_fwnode); 1352 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, dev_flags); 1353 } 1354 1355 if (ret) { 1356 netdev_err(priv->dev, "cannot attach to PHY (error: %pe)\n", 1357 ERR_PTR(ret)); 1358 return ret; 1359 } 1360 1361 /* Configure phylib's copy of the LPI timer. Normally, 1362 * phylink_config.lpi_timer_default would do this, but there is a 1363 * chance that userspace could change the eee_timer setting via sysfs 1364 * before the first open. Thus, preserve existing behaviour. 1365 */ 1366 if (!phylink_ethtool_get_eee(priv->phylink, &eee)) { 1367 eee.tx_lpi_timer = priv->tx_lpi_timer; 1368 phylink_ethtool_set_eee(priv->phylink, &eee); 1369 } 1370 1371 return 0; 1372 } 1373 1374 static int stmmac_phylink_setup(struct stmmac_priv *priv) 1375 { 1376 struct phylink_config *config; 1377 struct phylink_pcs *pcs; 1378 struct phylink *phylink; 1379 1380 config = &priv->phylink_config; 1381 1382 config->dev = &priv->dev->dev; 1383 config->type = PHYLINK_NETDEV; 1384 config->mac_managed_pm = true; 1385 1386 /* Stmmac always requires an RX clock for hardware initialization */ 1387 config->mac_requires_rxc = true; 1388 1389 /* Disable EEE RX clock stop to ensure VLAN register access works 1390 * correctly. 1391 */ 1392 if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI) && 1393 !(priv->dev->features & NETIF_F_VLAN_FEATURES)) 1394 config->eee_rx_clk_stop_enable = true; 1395 1396 /* Set the default transmit clock stop bit based on the platform glue */ 1397 priv->tx_lpi_clk_stop = priv->plat->flags & 1398 STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; 1399 1400 /* Get the PHY interface modes (at the PHY end of the link) that 1401 * are supported by the platform. 1402 */ 1403 if (priv->plat->get_interfaces) 1404 priv->plat->get_interfaces(priv, priv->plat->bsp_priv, 1405 config->supported_interfaces); 1406 1407 config->default_an_inband = priv->plat->default_an_inband; 1408 1409 /* Set the platform/firmware specified interface mode if the 1410 * supported interfaces have not already been provided using 1411 * phy_interface as a last resort. 1412 */ 1413 if (phy_interface_empty(config->supported_interfaces)) 1414 __set_bit(priv->plat->phy_interface, 1415 config->supported_interfaces); 1416 1417 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1418 if (priv->hw->xpcs) 1419 pcs = xpcs_to_phylink_pcs(priv->hw->xpcs); 1420 else 1421 pcs = priv->hw->phylink_pcs; 1422 1423 if (pcs) 1424 phy_interface_or(config->supported_interfaces, 1425 config->supported_interfaces, 1426 pcs->supported_interfaces); 1427 1428 /* Some platforms, e.g. iMX8MP, wire lpi_intr_o to the same interrupt 1429 * used for stmmac's main interrupts, which leads to interrupt storms. 1430 * STMMAC_FLAG_EEE_DISABLE allows EEE to be disabled on such platforms. 1431 */ 1432 if (priv->dma_cap.eee && 1433 !(priv->plat->flags & STMMAC_FLAG_EEE_DISABLE)) { 1434 /* The GMAC 3.74a databook states that EEE is only supported 1435 * in MII, GMII, and RGMII interfaces. 1436 */ 1437 __set_bit(PHY_INTERFACE_MODE_MII, config->lpi_interfaces); 1438 __set_bit(PHY_INTERFACE_MODE_GMII, config->lpi_interfaces); 1439 phy_interface_set_rgmii(config->lpi_interfaces); 1440 1441 /* If we have a non-integrated PCS, assume that it is connected 1442 * to the GMAC using GMII or another EEE compatible interface, 1443 * and thus all PCS-supported interfaces support LPI. 1444 */ 1445 if (pcs) 1446 phy_interface_or(config->lpi_interfaces, 1447 config->lpi_interfaces, 1448 pcs->supported_interfaces); 1449 1450 /* All full duplex speeds above 100Mbps are supported */ 1451 config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD; 1452 config->lpi_timer_default = eee_timer * 1000; 1453 config->eee_enabled_default = true; 1454 } 1455 1456 config->wol_phy_speed_ctrl = true; 1457 if (priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL) { 1458 config->wol_phy_legacy = true; 1459 } else { 1460 if (priv->dma_cap.pmt_remote_wake_up) 1461 config->wol_mac_support |= WAKE_UCAST; 1462 if (priv->dma_cap.pmt_magic_frame) 1463 config->wol_mac_support |= WAKE_MAGIC; 1464 } 1465 1466 phylink = phylink_create(config, dev_fwnode(priv->device), 1467 priv->plat->phy_interface, 1468 &stmmac_phylink_mac_ops); 1469 if (IS_ERR(phylink)) 1470 return PTR_ERR(phylink); 1471 1472 priv->phylink = phylink; 1473 return 0; 1474 } 1475 1476 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1477 struct stmmac_dma_conf *dma_conf) 1478 { 1479 u8 rx_cnt = priv->plat->rx_queues_to_use; 1480 unsigned int desc_size; 1481 void *head_rx; 1482 u8 queue; 1483 1484 /* Display RX rings */ 1485 for (queue = 0; queue < rx_cnt; queue++) { 1486 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1487 1488 pr_info("\tRX Queue %u rings\n", queue); 1489 1490 head_rx = stmmac_get_rx_desc(priv, rx_q, 0); 1491 desc_size = stmmac_get_rx_desc_size(priv); 1492 1493 /* Display RX ring */ 1494 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1495 rx_q->dma_rx_phy, desc_size); 1496 } 1497 } 1498 1499 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1500 struct stmmac_dma_conf *dma_conf) 1501 { 1502 u8 tx_cnt = priv->plat->tx_queues_to_use; 1503 unsigned int desc_size; 1504 void *head_tx; 1505 u8 queue; 1506 1507 /* Display TX rings */ 1508 for (queue = 0; queue < tx_cnt; queue++) { 1509 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1510 1511 pr_info("\tTX Queue %d rings\n", queue); 1512 1513 head_tx = stmmac_get_tx_desc(priv, tx_q, 0); 1514 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 1515 1516 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1517 tx_q->dma_tx_phy, desc_size); 1518 } 1519 } 1520 1521 static void stmmac_display_rings(struct stmmac_priv *priv, 1522 struct stmmac_dma_conf *dma_conf) 1523 { 1524 /* Display RX ring */ 1525 stmmac_display_rx_rings(priv, dma_conf); 1526 1527 /* Display TX ring */ 1528 stmmac_display_tx_rings(priv, dma_conf); 1529 } 1530 1531 static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) 1532 { 1533 if (stmmac_xdp_is_enabled(priv)) 1534 return XDP_PACKET_HEADROOM + NET_IP_ALIGN; 1535 1536 return NET_SKB_PAD + NET_IP_ALIGN; 1537 } 1538 1539 static int stmmac_set_bfsize(int len) 1540 { 1541 int ret; 1542 1543 if (len > BUF_SIZE_8KiB) 1544 ret = BUF_SIZE_16KiB; 1545 else if (len > BUF_SIZE_4KiB) 1546 ret = BUF_SIZE_8KiB; 1547 else if (len > BUF_SIZE_2KiB) 1548 ret = BUF_SIZE_4KiB; 1549 else if (len > DEFAULT_BUFSIZE) 1550 ret = BUF_SIZE_2KiB; 1551 else 1552 ret = DEFAULT_BUFSIZE; 1553 1554 return ret; 1555 } 1556 1557 /** 1558 * stmmac_clear_rx_descriptors - clear RX descriptors 1559 * @priv: driver private structure 1560 * @dma_conf: structure to take the dma data 1561 * @queue: RX queue index 1562 * Description: this function is called to clear the RX descriptors 1563 * in case of both basic and extended descriptors are used. 1564 */ 1565 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1566 struct stmmac_dma_conf *dma_conf, 1567 u32 queue) 1568 { 1569 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1570 struct dma_desc *desc; 1571 int i; 1572 1573 /* Clear the RX descriptors */ 1574 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1575 desc = stmmac_get_rx_desc(priv, rx_q, i); 1576 1577 stmmac_init_rx_desc(priv, desc, priv->use_riwt, 1578 priv->descriptor_mode, 1579 (i == dma_conf->dma_rx_size - 1), 1580 dma_conf->dma_buf_sz); 1581 } 1582 } 1583 1584 /** 1585 * stmmac_clear_tx_descriptors - clear tx descriptors 1586 * @priv: driver private structure 1587 * @dma_conf: structure to take the dma data 1588 * @queue: TX queue index. 1589 * Description: this function is called to clear the TX descriptors 1590 * in case of both basic and extended descriptors are used. 1591 */ 1592 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1593 struct stmmac_dma_conf *dma_conf, 1594 u32 queue) 1595 { 1596 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1597 int i; 1598 1599 /* Clear the TX descriptors */ 1600 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1601 int last = (i == (dma_conf->dma_tx_size - 1)); 1602 struct dma_desc *p; 1603 1604 p = stmmac_get_tx_desc(priv, tx_q, i); 1605 stmmac_init_tx_desc(priv, p, priv->descriptor_mode, last); 1606 } 1607 } 1608 1609 /** 1610 * stmmac_clear_descriptors - clear descriptors 1611 * @priv: driver private structure 1612 * @dma_conf: structure to take the dma data 1613 * Description: this function is called to clear the TX and RX descriptors 1614 * in case of both basic and extended descriptors are used. 1615 */ 1616 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1617 struct stmmac_dma_conf *dma_conf) 1618 { 1619 u8 rx_queue_cnt = priv->plat->rx_queues_to_use; 1620 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 1621 u8 queue; 1622 1623 /* Clear the RX descriptors */ 1624 for (queue = 0; queue < rx_queue_cnt; queue++) 1625 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1626 1627 /* Clear the TX descriptors */ 1628 for (queue = 0; queue < tx_queue_cnt; queue++) 1629 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1630 } 1631 1632 static bool stmmac_rx_check_buf2_cap(struct stmmac_priv *priv) 1633 { 1634 /* Only cores that can program an independent secondary RX buffer 1635 * (used for scatter-gather overflow or split-header payload) back 1636 * buffer2. Legacy cores have no set_sec_addr op, so buffer2 is 1637 * never handed to the hardware there. 1638 */ 1639 return priv->hw->desc && priv->hw->desc->set_sec_addr; 1640 } 1641 1642 /** 1643 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1644 * @priv: driver private structure 1645 * @dma_conf: structure to take the dma data 1646 * @p: descriptor pointer 1647 * @i: descriptor index 1648 * @flags: gfp flag 1649 * @queue: RX queue index 1650 * Description: this function is called to allocate a receive buffer, perform 1651 * the DMA mapping and init the descriptor. 1652 */ 1653 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1654 struct stmmac_dma_conf *dma_conf, 1655 struct dma_desc *p, 1656 int i, gfp_t flags, u32 queue) 1657 { 1658 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1659 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1660 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1661 1662 if (priv->dma_cap.host_dma_width <= 32) 1663 gfp |= GFP_DMA32; 1664 1665 if (!buf->page) { 1666 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1667 if (!buf->page) 1668 return -ENOMEM; 1669 buf->page_offset = stmmac_rx_offset(priv); 1670 } 1671 1672 if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) { 1673 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1674 if (!buf->sec_page) 1675 return -ENOMEM; 1676 1677 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1678 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1679 } 1680 1681 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1682 1683 stmmac_set_desc_addr(priv, p, buf->addr); 1684 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1685 stmmac_init_desc3(priv, p); 1686 1687 return 0; 1688 } 1689 1690 /** 1691 * stmmac_free_rx_buffer - free RX dma buffers 1692 * @priv: private structure 1693 * @rx_q: RX queue 1694 * @i: buffer index. 1695 */ 1696 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1697 struct stmmac_rx_queue *rx_q, 1698 int i) 1699 { 1700 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1701 1702 if (buf->page) 1703 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1704 buf->page = NULL; 1705 1706 if (buf->sec_page) 1707 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1708 buf->sec_page = NULL; 1709 } 1710 1711 /** 1712 * stmmac_free_tx_buffer - free RX dma buffers 1713 * @priv: private structure 1714 * @dma_conf: structure to take the dma data 1715 * @queue: RX queue index 1716 * @i: buffer index. 1717 */ 1718 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1719 struct stmmac_dma_conf *dma_conf, 1720 u32 queue, int i) 1721 { 1722 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1723 1724 if (tx_q->tx_skbuff_dma[i].buf && 1725 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1726 if (tx_q->tx_skbuff_dma[i].map_as_page) 1727 dma_unmap_page(priv->device, 1728 tx_q->tx_skbuff_dma[i].buf, 1729 tx_q->tx_skbuff_dma[i].len, 1730 DMA_TO_DEVICE); 1731 else 1732 dma_unmap_single(priv->device, 1733 tx_q->tx_skbuff_dma[i].buf, 1734 tx_q->tx_skbuff_dma[i].len, 1735 DMA_TO_DEVICE); 1736 } 1737 1738 if (tx_q->xdpf[i] && 1739 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1740 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1741 xdp_return_frame(tx_q->xdpf[i]); 1742 tx_q->xdpf[i] = NULL; 1743 } 1744 1745 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1746 tx_q->xsk_frames_done++; 1747 1748 if (tx_q->tx_skbuff[i] && 1749 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1750 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1751 tx_q->tx_skbuff[i] = NULL; 1752 } 1753 1754 tx_q->tx_skbuff_dma[i].buf = 0; 1755 tx_q->tx_skbuff_dma[i].map_as_page = false; 1756 } 1757 1758 /** 1759 * dma_free_rx_skbufs - free RX dma buffers 1760 * @priv: private structure 1761 * @dma_conf: structure to take the dma data 1762 * @queue: RX queue index 1763 */ 1764 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1765 struct stmmac_dma_conf *dma_conf, 1766 u32 queue) 1767 { 1768 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1769 int i; 1770 1771 for (i = 0; i < dma_conf->dma_rx_size; i++) 1772 stmmac_free_rx_buffer(priv, rx_q, i); 1773 } 1774 1775 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1776 struct stmmac_dma_conf *dma_conf, 1777 u32 queue, gfp_t flags) 1778 { 1779 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1780 int i; 1781 1782 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1783 struct dma_desc *p; 1784 int ret; 1785 1786 p = stmmac_get_rx_desc(priv, rx_q, i); 1787 1788 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1789 queue); 1790 if (ret) 1791 return ret; 1792 1793 rx_q->buf_alloc_num++; 1794 } 1795 1796 return 0; 1797 } 1798 1799 /** 1800 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1801 * @priv: private structure 1802 * @dma_conf: structure to take the dma data 1803 * @queue: RX queue index 1804 */ 1805 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1806 struct stmmac_dma_conf *dma_conf, 1807 u32 queue) 1808 { 1809 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1810 int i; 1811 1812 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1813 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1814 1815 if (!buf->xdp) 1816 continue; 1817 1818 xsk_buff_free(buf->xdp); 1819 buf->xdp = NULL; 1820 } 1821 } 1822 1823 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1824 struct stmmac_dma_conf *dma_conf, 1825 u32 queue) 1826 { 1827 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1828 int i; 1829 1830 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1831 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1832 * use this macro to make sure no size violations. 1833 */ 1834 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1835 1836 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1837 struct stmmac_rx_buffer *buf; 1838 dma_addr_t dma_addr; 1839 struct dma_desc *p; 1840 1841 p = stmmac_get_rx_desc(priv, rx_q, i); 1842 1843 buf = &rx_q->buf_pool[i]; 1844 1845 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1846 if (!buf->xdp) 1847 return -ENOMEM; 1848 1849 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1850 stmmac_set_desc_addr(priv, p, dma_addr); 1851 rx_q->buf_alloc_num++; 1852 } 1853 1854 return 0; 1855 } 1856 1857 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1858 { 1859 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1860 return NULL; 1861 1862 return xsk_get_pool_from_qid(priv->dev, queue); 1863 } 1864 1865 /** 1866 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1867 * @priv: driver private structure 1868 * @dma_conf: structure to take the dma data 1869 * @queue: RX queue index 1870 * @flags: gfp flag. 1871 * Description: this function initializes the DMA RX descriptors 1872 * and allocates the socket buffers. It supports the chained and ring 1873 * modes. 1874 */ 1875 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1876 struct stmmac_dma_conf *dma_conf, 1877 u32 queue, gfp_t flags) 1878 { 1879 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1880 void *des; 1881 int ret; 1882 1883 netif_dbg(priv, probe, priv->dev, 1884 "(%s) dma_rx_phy=0x%08x\n", __func__, 1885 (u32)rx_q->dma_rx_phy); 1886 1887 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1888 1889 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1890 1891 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1892 1893 if (rx_q->xsk_pool) { 1894 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1895 MEM_TYPE_XSK_BUFF_POOL, 1896 NULL)); 1897 netdev_info(priv->dev, 1898 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1899 queue); 1900 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1901 } else { 1902 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1903 MEM_TYPE_PAGE_POOL, 1904 rx_q->page_pool)); 1905 netdev_info(priv->dev, 1906 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1907 queue); 1908 } 1909 1910 if (rx_q->xsk_pool) { 1911 /* RX XDP ZC buffer pool may not be populated, e.g. 1912 * xdpsock TX-only. 1913 */ 1914 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1915 } else { 1916 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1917 if (ret < 0) 1918 return -ENOMEM; 1919 } 1920 1921 /* Setup the chained descriptor addresses */ 1922 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 1923 if (priv->extend_desc) 1924 des = rx_q->dma_erx; 1925 else 1926 des = rx_q->dma_rx; 1927 1928 stmmac_mode_init(priv, des, rx_q->dma_rx_phy, 1929 dma_conf->dma_rx_size, priv->extend_desc); 1930 } 1931 1932 return 0; 1933 } 1934 1935 static int init_dma_rx_desc_rings(struct net_device *dev, 1936 struct stmmac_dma_conf *dma_conf, 1937 gfp_t flags) 1938 { 1939 struct stmmac_priv *priv = netdev_priv(dev); 1940 u8 rx_count = priv->plat->rx_queues_to_use; 1941 int queue; 1942 int ret; 1943 1944 /* RX INITIALIZATION */ 1945 netif_dbg(priv, probe, priv->dev, 1946 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1947 1948 for (queue = 0; queue < rx_count; queue++) { 1949 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1950 if (ret) 1951 goto err_init_rx_buffers; 1952 } 1953 1954 return 0; 1955 1956 err_init_rx_buffers: 1957 while (queue >= 0) { 1958 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1959 1960 if (rx_q->xsk_pool) 1961 dma_free_rx_xskbufs(priv, dma_conf, queue); 1962 else 1963 dma_free_rx_skbufs(priv, dma_conf, queue); 1964 1965 rx_q->buf_alloc_num = 0; 1966 rx_q->xsk_pool = NULL; 1967 1968 queue--; 1969 } 1970 1971 return ret; 1972 } 1973 1974 static void stmmac_set_tx_dma_entry(struct stmmac_tx_queue *tx_q, 1975 unsigned int entry, 1976 enum stmmac_txbuf_type type, 1977 dma_addr_t addr, size_t len, 1978 bool map_as_page) 1979 { 1980 tx_q->tx_skbuff_dma[entry].buf = addr; 1981 tx_q->tx_skbuff_dma[entry].len = len; 1982 tx_q->tx_skbuff_dma[entry].buf_type = type; 1983 tx_q->tx_skbuff_dma[entry].map_as_page = map_as_page; 1984 tx_q->tx_skbuff_dma[entry].last_segment = false; 1985 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1986 } 1987 1988 static void stmmac_set_tx_skb_dma_entry(struct stmmac_tx_queue *tx_q, 1989 unsigned int entry, dma_addr_t addr, 1990 size_t len, bool map_as_page) 1991 { 1992 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_SKB, addr, len, 1993 map_as_page); 1994 } 1995 1996 static void stmmac_set_tx_dma_last_segment(struct stmmac_tx_queue *tx_q, 1997 unsigned int entry) 1998 { 1999 tx_q->tx_skbuff_dma[entry].last_segment = true; 2000 } 2001 2002 /** 2003 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 2004 * @priv: driver private structure 2005 * @dma_conf: structure to take the dma data 2006 * @queue: TX queue index 2007 * Description: this function initializes the DMA TX descriptors 2008 * and allocates the socket buffers. It supports the chained and ring 2009 * modes. 2010 */ 2011 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 2012 struct stmmac_dma_conf *dma_conf, 2013 u32 queue) 2014 { 2015 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2016 int i; 2017 2018 netif_dbg(priv, probe, priv->dev, 2019 "(%s) dma_tx_phy=0x%08x\n", __func__, 2020 (u32)tx_q->dma_tx_phy); 2021 2022 /* Setup the chained descriptor addresses */ 2023 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 2024 if (priv->extend_desc) 2025 stmmac_mode_init(priv, tx_q->dma_etx, 2026 tx_q->dma_tx_phy, 2027 dma_conf->dma_tx_size, 1); 2028 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 2029 stmmac_mode_init(priv, tx_q->dma_tx, 2030 tx_q->dma_tx_phy, 2031 dma_conf->dma_tx_size, 0); 2032 } 2033 2034 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 2035 2036 for (i = 0; i < dma_conf->dma_tx_size; i++) { 2037 struct dma_desc *p; 2038 2039 p = stmmac_get_tx_desc(priv, tx_q, i); 2040 stmmac_clear_desc(priv, p); 2041 stmmac_set_tx_skb_dma_entry(tx_q, i, 0, 0, false); 2042 2043 tx_q->tx_skbuff[i] = NULL; 2044 } 2045 2046 return 0; 2047 } 2048 2049 static int init_dma_tx_desc_rings(struct net_device *dev, 2050 struct stmmac_dma_conf *dma_conf) 2051 { 2052 struct stmmac_priv *priv = netdev_priv(dev); 2053 u8 tx_queue_cnt; 2054 u8 queue; 2055 2056 tx_queue_cnt = priv->plat->tx_queues_to_use; 2057 2058 for (queue = 0; queue < tx_queue_cnt; queue++) 2059 __init_dma_tx_desc_rings(priv, dma_conf, queue); 2060 2061 return 0; 2062 } 2063 2064 /** 2065 * init_dma_desc_rings - init the RX/TX descriptor rings 2066 * @dev: net device structure 2067 * @dma_conf: structure to take the dma data 2068 * @flags: gfp flag. 2069 * Description: this function initializes the DMA RX/TX descriptors 2070 * and allocates the socket buffers. It supports the chained and ring 2071 * modes. 2072 */ 2073 static int init_dma_desc_rings(struct net_device *dev, 2074 struct stmmac_dma_conf *dma_conf, 2075 gfp_t flags) 2076 { 2077 struct stmmac_priv *priv = netdev_priv(dev); 2078 int ret; 2079 2080 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 2081 if (ret) 2082 return ret; 2083 2084 ret = init_dma_tx_desc_rings(dev, dma_conf); 2085 2086 stmmac_clear_descriptors(priv, dma_conf); 2087 2088 if (netif_msg_hw(priv)) 2089 stmmac_display_rings(priv, dma_conf); 2090 2091 return ret; 2092 } 2093 2094 /** 2095 * dma_free_tx_skbufs - free TX dma buffers 2096 * @priv: private structure 2097 * @dma_conf: structure to take the dma data 2098 * @queue: TX queue index 2099 */ 2100 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 2101 struct stmmac_dma_conf *dma_conf, 2102 u32 queue) 2103 { 2104 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2105 int i; 2106 2107 tx_q->xsk_frames_done = 0; 2108 2109 for (i = 0; i < dma_conf->dma_tx_size; i++) 2110 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 2111 2112 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 2113 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2114 tx_q->xsk_frames_done = 0; 2115 tx_q->xsk_pool = NULL; 2116 } 2117 } 2118 2119 /** 2120 * stmmac_free_tx_skbufs - free TX skb buffers 2121 * @priv: private structure 2122 */ 2123 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 2124 { 2125 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 2126 u8 queue; 2127 2128 for (queue = 0; queue < tx_queue_cnt; queue++) 2129 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 2130 } 2131 2132 /** 2133 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 2134 * @priv: private structure 2135 * @dma_conf: structure to take the dma data 2136 * @queue: RX queue index 2137 */ 2138 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 2139 struct stmmac_dma_conf *dma_conf, 2140 u32 queue) 2141 { 2142 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2143 size_t size; 2144 void *addr; 2145 2146 /* Release the DMA RX socket buffers */ 2147 if (rx_q->xsk_pool) 2148 dma_free_rx_xskbufs(priv, dma_conf, queue); 2149 else 2150 dma_free_rx_skbufs(priv, dma_conf, queue); 2151 2152 rx_q->buf_alloc_num = 0; 2153 rx_q->xsk_pool = NULL; 2154 2155 /* Free DMA regions of consistent memory previously allocated */ 2156 if (priv->extend_desc) 2157 addr = rx_q->dma_erx; 2158 else 2159 addr = rx_q->dma_rx; 2160 2161 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2162 2163 dma_free_coherent(priv->device, size, addr, rx_q->dma_rx_phy); 2164 2165 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 2166 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 2167 2168 kfree(rx_q->buf_pool); 2169 if (rx_q->page_pool) 2170 page_pool_destroy(rx_q->page_pool); 2171 } 2172 2173 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 2174 struct stmmac_dma_conf *dma_conf) 2175 { 2176 u8 rx_count = priv->plat->rx_queues_to_use; 2177 u8 queue; 2178 2179 /* Free RX queue resources */ 2180 for (queue = 0; queue < rx_count; queue++) 2181 __free_dma_rx_desc_resources(priv, dma_conf, queue); 2182 } 2183 2184 /** 2185 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 2186 * @priv: private structure 2187 * @dma_conf: structure to take the dma data 2188 * @queue: TX queue index 2189 */ 2190 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 2191 struct stmmac_dma_conf *dma_conf, 2192 u32 queue) 2193 { 2194 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2195 size_t size; 2196 void *addr; 2197 2198 /* Release the DMA TX socket buffers */ 2199 dma_free_tx_skbufs(priv, dma_conf, queue); 2200 2201 if (priv->extend_desc) { 2202 addr = tx_q->dma_etx; 2203 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 2204 addr = tx_q->dma_entx; 2205 } else { 2206 addr = tx_q->dma_tx; 2207 } 2208 2209 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2210 2211 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 2212 2213 kfree(tx_q->tx_skbuff_dma); 2214 kfree(tx_q->tx_skbuff); 2215 } 2216 2217 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2218 struct stmmac_dma_conf *dma_conf) 2219 { 2220 u8 tx_count = priv->plat->tx_queues_to_use; 2221 u8 queue; 2222 2223 /* Free TX queue resources */ 2224 for (queue = 0; queue < tx_count; queue++) 2225 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2226 } 2227 2228 /** 2229 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2230 * @priv: private structure 2231 * @dma_conf: structure to take the dma data 2232 * @queue: RX queue index 2233 * Description: according to which descriptor can be used (extend or basic) 2234 * this function allocates the resources for TX and RX paths. In case of 2235 * reception, for example, it pre-allocated the RX socket buffer in order to 2236 * allow zero-copy mechanism. 2237 */ 2238 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2239 struct stmmac_dma_conf *dma_conf, 2240 u32 queue) 2241 { 2242 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2243 struct stmmac_channel *ch = &priv->channel[queue]; 2244 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2245 struct page_pool_params pp_params = { 0 }; 2246 unsigned int dma_buf_sz_pad, num_pages; 2247 unsigned int napi_id; 2248 size_t size; 2249 void *addr; 2250 int ret; 2251 2252 dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + 2253 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2254 num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE); 2255 2256 rx_q->queue_index = queue; 2257 rx_q->priv_data = priv; 2258 rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE; 2259 2260 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2261 pp_params.pool_size = dma_conf->dma_rx_size; 2262 pp_params.order = order_base_2(num_pages); 2263 pp_params.nid = dev_to_node(priv->device); 2264 pp_params.dev = priv->device; 2265 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2266 pp_params.offset = 0; 2267 pp_params.max_len = dma_conf->dma_buf_sz + stmmac_rx_offset(priv); 2268 2269 rx_q->page_pool = page_pool_create(&pp_params); 2270 if (IS_ERR(rx_q->page_pool)) { 2271 ret = PTR_ERR(rx_q->page_pool); 2272 rx_q->page_pool = NULL; 2273 return ret; 2274 } 2275 2276 rx_q->buf_pool = kzalloc_objs(*rx_q->buf_pool, dma_conf->dma_rx_size); 2277 if (!rx_q->buf_pool) 2278 return -ENOMEM; 2279 2280 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2281 2282 addr = dma_alloc_coherent(priv->device, size, &rx_q->dma_rx_phy, 2283 GFP_KERNEL); 2284 if (!addr) 2285 return -ENOMEM; 2286 2287 if (priv->extend_desc) 2288 rx_q->dma_erx = addr; 2289 else 2290 rx_q->dma_rx = addr; 2291 2292 if (stmmac_xdp_is_enabled(priv) && 2293 test_bit(queue, priv->af_xdp_zc_qps)) 2294 napi_id = ch->rxtx_napi.napi_id; 2295 else 2296 napi_id = ch->rx_napi.napi_id; 2297 2298 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, queue, napi_id); 2299 if (ret) { 2300 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2301 return -EINVAL; 2302 } 2303 2304 return 0; 2305 } 2306 2307 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2308 struct stmmac_dma_conf *dma_conf) 2309 { 2310 u8 rx_count = priv->plat->rx_queues_to_use; 2311 u8 queue; 2312 int ret; 2313 2314 /* RX queues buffers and DMA */ 2315 for (queue = 0; queue < rx_count; queue++) { 2316 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2317 if (ret) 2318 goto err_dma; 2319 } 2320 2321 return 0; 2322 2323 err_dma: 2324 free_dma_rx_desc_resources(priv, dma_conf); 2325 2326 return ret; 2327 } 2328 2329 /** 2330 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2331 * @priv: private structure 2332 * @dma_conf: structure to take the dma data 2333 * @queue: TX queue index 2334 * Description: according to which descriptor can be used (extend or basic) 2335 * this function allocates the resources for TX and RX paths. In case of 2336 * reception, for example, it pre-allocated the RX socket buffer in order to 2337 * allow zero-copy mechanism. 2338 */ 2339 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2340 struct stmmac_dma_conf *dma_conf, 2341 u32 queue) 2342 { 2343 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2344 size_t size; 2345 void *addr; 2346 2347 tx_q->queue_index = queue; 2348 tx_q->priv_data = priv; 2349 2350 tx_q->tx_skbuff_dma = kzalloc_objs(*tx_q->tx_skbuff_dma, 2351 dma_conf->dma_tx_size); 2352 if (!tx_q->tx_skbuff_dma) 2353 return -ENOMEM; 2354 2355 tx_q->tx_skbuff = kzalloc_objs(struct sk_buff *, dma_conf->dma_tx_size); 2356 if (!tx_q->tx_skbuff) 2357 return -ENOMEM; 2358 2359 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2360 2361 addr = dma_alloc_coherent(priv->device, size, 2362 &tx_q->dma_tx_phy, GFP_KERNEL); 2363 if (!addr) 2364 return -ENOMEM; 2365 2366 if (priv->extend_desc) 2367 tx_q->dma_etx = addr; 2368 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2369 tx_q->dma_entx = addr; 2370 else 2371 tx_q->dma_tx = addr; 2372 2373 return 0; 2374 } 2375 2376 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2377 struct stmmac_dma_conf *dma_conf) 2378 { 2379 u8 tx_count = priv->plat->tx_queues_to_use; 2380 u8 queue; 2381 int ret; 2382 2383 /* TX queues buffers and DMA */ 2384 for (queue = 0; queue < tx_count; queue++) { 2385 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2386 if (ret) 2387 goto err_dma; 2388 } 2389 2390 return 0; 2391 2392 err_dma: 2393 free_dma_tx_desc_resources(priv, dma_conf); 2394 return ret; 2395 } 2396 2397 /** 2398 * alloc_dma_desc_resources - alloc TX/RX resources. 2399 * @priv: private structure 2400 * @dma_conf: structure to take the dma data 2401 * Description: according to which descriptor can be used (extend or basic) 2402 * this function allocates the resources for TX and RX paths. In case of 2403 * reception, for example, it pre-allocated the RX socket buffer in order to 2404 * allow zero-copy mechanism. 2405 */ 2406 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2407 struct stmmac_dma_conf *dma_conf) 2408 { 2409 /* RX Allocation */ 2410 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2411 2412 if (ret) 2413 return ret; 2414 2415 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2416 2417 return ret; 2418 } 2419 2420 /** 2421 * free_dma_desc_resources - free dma desc resources 2422 * @priv: private structure 2423 * @dma_conf: structure to take the dma data 2424 */ 2425 static void free_dma_desc_resources(struct stmmac_priv *priv, 2426 struct stmmac_dma_conf *dma_conf) 2427 { 2428 /* Release the DMA TX socket buffers */ 2429 free_dma_tx_desc_resources(priv, dma_conf); 2430 2431 /* Release the DMA RX socket buffers later 2432 * to ensure all pending XDP_TX buffers are returned. 2433 */ 2434 free_dma_rx_desc_resources(priv, dma_conf); 2435 } 2436 2437 /** 2438 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2439 * @priv: driver private structure 2440 * Description: It is used for enabling the rx queues in the MAC 2441 */ 2442 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2443 { 2444 u8 rx_queues_count = priv->plat->rx_queues_to_use; 2445 u8 queue; 2446 u8 mode; 2447 2448 for (queue = 0; queue < rx_queues_count; queue++) { 2449 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2450 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2451 } 2452 } 2453 2454 /** 2455 * stmmac_start_rx_dma - start RX DMA channel 2456 * @priv: driver private structure 2457 * @chan: RX channel index 2458 * Description: 2459 * This starts a RX DMA channel 2460 */ 2461 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2462 { 2463 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2464 stmmac_start_rx(priv, priv->ioaddr, chan); 2465 } 2466 2467 /** 2468 * stmmac_start_tx_dma - start TX DMA channel 2469 * @priv: driver private structure 2470 * @chan: TX channel index 2471 * Description: 2472 * This starts a TX DMA channel 2473 */ 2474 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2475 { 2476 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2477 stmmac_start_tx(priv, priv->ioaddr, chan); 2478 } 2479 2480 /** 2481 * stmmac_stop_rx_dma - stop RX DMA channel 2482 * @priv: driver private structure 2483 * @chan: RX channel index 2484 * Description: 2485 * This stops a RX DMA channel 2486 */ 2487 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2488 { 2489 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2490 stmmac_stop_rx(priv, priv->ioaddr, chan); 2491 } 2492 2493 /** 2494 * stmmac_stop_tx_dma - stop TX DMA channel 2495 * @priv: driver private structure 2496 * @chan: TX channel index 2497 * Description: 2498 * This stops a TX DMA channel 2499 */ 2500 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2501 { 2502 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2503 stmmac_stop_tx(priv, priv->ioaddr, chan); 2504 } 2505 2506 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2507 { 2508 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2509 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2510 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2511 u8 chan; 2512 2513 for (chan = 0; chan < dma_csr_ch; chan++) { 2514 struct stmmac_channel *ch = &priv->channel[chan]; 2515 unsigned long flags; 2516 2517 spin_lock_irqsave(&ch->lock, flags); 2518 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2519 spin_unlock_irqrestore(&ch->lock, flags); 2520 } 2521 } 2522 2523 /** 2524 * stmmac_start_all_dma - start all RX and TX DMA channels 2525 * @priv: driver private structure 2526 * Description: 2527 * This starts all the RX and TX DMA channels 2528 */ 2529 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2530 { 2531 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2532 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2533 u8 chan; 2534 2535 for (chan = 0; chan < rx_channels_count; chan++) 2536 stmmac_start_rx_dma(priv, chan); 2537 2538 for (chan = 0; chan < tx_channels_count; chan++) 2539 stmmac_start_tx_dma(priv, chan); 2540 } 2541 2542 /** 2543 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2544 * @priv: driver private structure 2545 * Description: 2546 * This stops the RX and TX DMA channels 2547 */ 2548 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2549 { 2550 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2551 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2552 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2553 u8 chan; 2554 2555 for (chan = 0; chan < rx_channels_count; chan++) 2556 stmmac_stop_rx_dma(priv, chan); 2557 2558 for (chan = 0; chan < tx_channels_count; chan++) 2559 stmmac_stop_tx_dma(priv, chan); 2560 2561 for (chan = 0; chan < dma_csr_ch; chan++) 2562 stmmac_deinit_chan(priv, priv->ioaddr, chan); 2563 } 2564 2565 /** 2566 * stmmac_dma_operation_mode - HW DMA operation mode 2567 * @priv: driver private structure 2568 * Description: it is used for configuring the DMA operation mode register in 2569 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2570 */ 2571 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2572 { 2573 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2574 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2575 int rxfifosz = priv->plat->rx_fifo_size; 2576 int txfifosz = priv->plat->tx_fifo_size; 2577 u32 txmode = 0; 2578 u32 rxmode = 0; 2579 u8 qmode = 0; 2580 u8 chan; 2581 2582 if (rxfifosz == 0) 2583 rxfifosz = priv->dma_cap.rx_fifo_size; 2584 if (txfifosz == 0) 2585 txfifosz = priv->dma_cap.tx_fifo_size; 2586 2587 /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ 2588 if (dwmac_is_xmac(priv->plat->core_type)) { 2589 rxfifosz /= rx_channels_count; 2590 txfifosz /= tx_channels_count; 2591 } 2592 2593 if (priv->plat->force_thresh_dma_mode) { 2594 txmode = tc; 2595 rxmode = tc; 2596 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2597 /* 2598 * In case of GMAC, SF mode can be enabled 2599 * to perform the TX COE in HW. This depends on: 2600 * 1) TX COE if actually supported 2601 * 2) There is no bugged Jumbo frame support 2602 * that needs to not insert csum in the TDES. 2603 */ 2604 txmode = SF_DMA_MODE; 2605 rxmode = SF_DMA_MODE; 2606 priv->xstats.threshold = SF_DMA_MODE; 2607 } else { 2608 txmode = tc; 2609 rxmode = SF_DMA_MODE; 2610 } 2611 2612 /* configure all channels */ 2613 for (chan = 0; chan < rx_channels_count; chan++) { 2614 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2615 2616 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2617 2618 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2619 rxfifosz, qmode); 2620 2621 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 2622 } 2623 2624 for (chan = 0; chan < tx_channels_count; chan++) { 2625 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2626 2627 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2628 txfifosz, qmode); 2629 } 2630 } 2631 2632 static void stmmac_xsk_request_timestamp(void *_priv) 2633 { 2634 struct stmmac_metadata_request *meta_req = _priv; 2635 2636 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2637 *meta_req->set_ic = true; 2638 } 2639 2640 static u64 stmmac_xsk_fill_timestamp(void *_priv) 2641 { 2642 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2643 struct stmmac_priv *priv = tx_compl->priv; 2644 struct dma_desc *desc = tx_compl->desc; 2645 bool found = false; 2646 u64 ns = 0; 2647 2648 if (!priv->hwts_tx_en) 2649 return 0; 2650 2651 /* check tx tstamp status */ 2652 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2653 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2654 found = true; 2655 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2656 found = true; 2657 } 2658 2659 if (found) { 2660 ns -= priv->plat->cdc_error_adj; 2661 return ns_to_ktime(ns); 2662 } 2663 2664 return 0; 2665 } 2666 2667 static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv) 2668 { 2669 struct timespec64 ts = ns_to_timespec64(launch_time); 2670 struct stmmac_metadata_request *meta_req = _priv; 2671 2672 if (meta_req->tbs & STMMAC_TBS_EN) 2673 stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec, 2674 ts.tv_nsec); 2675 } 2676 2677 static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2678 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2679 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2680 .tmo_request_launch_time = stmmac_xsk_request_launch_time, 2681 }; 2682 2683 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2684 { 2685 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2686 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2687 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2688 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 2689 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2690 unsigned int entry = tx_q->cur_tx; 2691 struct dma_desc *tx_desc = NULL; 2692 struct xdp_desc xdp_desc; 2693 bool work_done = true; 2694 u32 tx_set_ic_bit = 0; 2695 2696 /* Avoids TX time-out as we are sharing with slow path */ 2697 txq_trans_cond_update(nq); 2698 2699 budget = min(budget, stmmac_tx_avail(priv, queue)); 2700 2701 for (; budget > 0; budget--) { 2702 struct stmmac_metadata_request meta_req; 2703 struct xsk_tx_metadata *meta = NULL; 2704 dma_addr_t dma_addr; 2705 bool set_ic; 2706 2707 /* We are sharing with slow path and stop XSK TX desc submission when 2708 * available TX ring is less than threshold. 2709 */ 2710 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2711 !netif_carrier_ok(priv->dev)) { 2712 work_done = false; 2713 break; 2714 } 2715 2716 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2717 break; 2718 2719 if (priv->est && priv->est->enable && 2720 priv->est->max_sdu[queue] && 2721 xdp_desc.len > priv->est->max_sdu[queue]) { 2722 priv->xstats.max_sdu_txq_drop[queue]++; 2723 continue; 2724 } 2725 2726 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 2727 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2728 meta = xsk_buff_get_metadata(pool, xdp_desc.addr, 2729 xdp_desc.options); 2730 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2731 2732 /* To return XDP buffer to XSK pool, we simple call 2733 * xsk_tx_completed(), so we don't need to fill up 2734 * 'buf' and 'xdpf'. 2735 */ 2736 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_XSK_TX, 2737 0, xdp_desc.len, false); 2738 stmmac_set_tx_dma_last_segment(tx_q, entry); 2739 2740 tx_q->xdpf[entry] = NULL; 2741 2742 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2743 2744 tx_q->tx_count_frames++; 2745 2746 if (!priv->tx_coal_frames[queue]) 2747 set_ic = false; 2748 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2749 set_ic = true; 2750 else 2751 set_ic = false; 2752 2753 meta_req.priv = priv; 2754 meta_req.tx_desc = tx_desc; 2755 meta_req.set_ic = &set_ic; 2756 meta_req.tbs = tx_q->tbs; 2757 meta_req.edesc = &tx_q->dma_entx[entry]; 2758 xsk_tx_metadata_request(pool, &meta, 2759 &stmmac_xsk_tx_metadata_ops, &meta_req); 2760 if (set_ic) { 2761 tx_q->tx_count_frames = 0; 2762 stmmac_set_tx_ic(priv, tx_desc); 2763 tx_set_ic_bit++; 2764 } 2765 2766 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2767 csum, priv->descriptor_mode, true, true, 2768 xdp_desc.len); 2769 2770 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 2771 2772 xsk_tx_metadata_to_compl(meta, 2773 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2774 2775 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2776 entry = tx_q->cur_tx; 2777 } 2778 u64_stats_update_begin(&txq_stats->napi_syncp); 2779 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2780 u64_stats_update_end(&txq_stats->napi_syncp); 2781 2782 if (tx_desc) { 2783 stmmac_flush_tx_descriptors(priv, queue); 2784 xsk_tx_release(pool); 2785 } 2786 2787 /* Return true if all of the 3 conditions are met 2788 * a) TX Budget is still available 2789 * b) work_done = true when XSK TX desc peek is empty (no more 2790 * pending XSK TX for transmission) 2791 */ 2792 return !!budget && work_done; 2793 } 2794 2795 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2796 { 2797 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2798 tc += 64; 2799 2800 if (priv->plat->force_thresh_dma_mode) 2801 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2802 else 2803 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2804 chan); 2805 2806 priv->xstats.threshold = tc; 2807 } 2808 } 2809 2810 /** 2811 * stmmac_tx_clean - to manage the transmission completion 2812 * @priv: driver private structure 2813 * @budget: napi budget limiting this functions packet handling 2814 * @queue: TX queue index 2815 * @pending_packets: signal to arm the TX coal timer 2816 * Description: it reclaims the transmit resources after transmission completes. 2817 * If some packets still needs to be handled, due to TX coalesce, set 2818 * pending_packets to true to make NAPI arm the TX coal timer. 2819 */ 2820 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2821 bool *pending_packets) 2822 { 2823 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2824 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2825 unsigned int bytes_compl = 0, pkts_compl = 0; 2826 unsigned int entry, xmits = 0, count = 0; 2827 u32 tx_packets = 0, tx_errors = 0; 2828 2829 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2830 2831 tx_q->xsk_frames_done = 0; 2832 2833 entry = tx_q->dirty_tx; 2834 2835 /* Try to clean all TX complete frame in 1 shot */ 2836 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2837 struct xdp_frame *xdpf; 2838 struct sk_buff *skb; 2839 struct dma_desc *p; 2840 int status; 2841 2842 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2843 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2844 xdpf = tx_q->xdpf[entry]; 2845 skb = NULL; 2846 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2847 xdpf = NULL; 2848 skb = tx_q->tx_skbuff[entry]; 2849 } else { 2850 xdpf = NULL; 2851 skb = NULL; 2852 } 2853 2854 p = stmmac_get_tx_desc(priv, tx_q, entry); 2855 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2856 /* Check if the descriptor is owned by the DMA */ 2857 if (unlikely(status & tx_dma_own)) 2858 break; 2859 2860 count++; 2861 2862 /* Make sure descriptor fields are read after reading 2863 * the own bit. 2864 */ 2865 dma_rmb(); 2866 2867 /* Just consider the last segment and ...*/ 2868 if (likely(!(status & tx_not_ls))) { 2869 /* ... verify the status error condition */ 2870 if (unlikely(status & tx_err)) { 2871 tx_errors++; 2872 if (unlikely(status & tx_err_bump_tc)) 2873 stmmac_bump_dma_threshold(priv, queue); 2874 } else { 2875 tx_packets++; 2876 } 2877 if (skb) { 2878 stmmac_get_tx_hwtstamp(priv, p, skb); 2879 } else if (tx_q->xsk_pool && 2880 xp_tx_metadata_enabled(tx_q->xsk_pool)) { 2881 struct stmmac_xsk_tx_complete tx_compl = { 2882 .priv = priv, 2883 .desc = p, 2884 }; 2885 2886 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2887 &stmmac_xsk_tx_metadata_ops, 2888 &tx_compl); 2889 } 2890 } 2891 2892 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2893 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2894 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2895 dma_unmap_page(priv->device, 2896 tx_q->tx_skbuff_dma[entry].buf, 2897 tx_q->tx_skbuff_dma[entry].len, 2898 DMA_TO_DEVICE); 2899 else 2900 dma_unmap_single(priv->device, 2901 tx_q->tx_skbuff_dma[entry].buf, 2902 tx_q->tx_skbuff_dma[entry].len, 2903 DMA_TO_DEVICE); 2904 tx_q->tx_skbuff_dma[entry].buf = 0; 2905 tx_q->tx_skbuff_dma[entry].len = 0; 2906 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2907 } 2908 2909 /* This looks at tx_q->tx_skbuff_dma[tx_q->dirty_tx].is_jumbo 2910 * and tx_q->tx_skbuff_dma[tx_q->dirty_tx].last_segment 2911 */ 2912 stmmac_clean_desc3(priv, tx_q, p); 2913 2914 tx_q->tx_skbuff_dma[entry].last_segment = false; 2915 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2916 2917 if (xdpf && 2918 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2919 xdp_return_frame_rx_napi(xdpf); 2920 tx_q->xdpf[entry] = NULL; 2921 } 2922 2923 if (xdpf && 2924 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2925 xdp_return_frame(xdpf); 2926 tx_q->xdpf[entry] = NULL; 2927 } 2928 2929 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2930 tx_q->xsk_frames_done++; 2931 2932 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2933 if (likely(skb)) { 2934 pkts_compl++; 2935 bytes_compl += skb->len; 2936 dev_consume_skb_any(skb); 2937 tx_q->tx_skbuff[entry] = NULL; 2938 } 2939 } 2940 2941 stmmac_release_tx_desc(priv, p, priv->descriptor_mode); 2942 2943 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 2944 } 2945 tx_q->dirty_tx = entry; 2946 2947 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2948 pkts_compl, bytes_compl); 2949 2950 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2951 queue))) && 2952 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2953 2954 netif_dbg(priv, tx_done, priv->dev, 2955 "%s: restart transmit\n", __func__); 2956 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2957 } 2958 2959 if (tx_q->xsk_pool) { 2960 bool work_done; 2961 2962 if (tx_q->xsk_frames_done) 2963 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2964 2965 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2966 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2967 2968 /* For XSK TX, we try to send as many as possible. 2969 * If XSK work done (XSK TX desc empty and budget still 2970 * available), return "budget - 1" to reenable TX IRQ. 2971 * Else, return "budget" to make NAPI continue polling. 2972 */ 2973 work_done = stmmac_xdp_xmit_zc(priv, queue, 2974 STMMAC_XSK_TX_BUDGET_MAX); 2975 if (work_done) 2976 xmits = budget - 1; 2977 else 2978 xmits = budget; 2979 } 2980 2981 if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode) 2982 stmmac_restart_sw_lpi_timer(priv); 2983 2984 /* We still have pending packets, let's call for a new scheduling */ 2985 if (tx_q->dirty_tx != tx_q->cur_tx) 2986 *pending_packets = true; 2987 2988 u64_stats_update_begin(&txq_stats->napi_syncp); 2989 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2990 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2991 u64_stats_inc(&txq_stats->napi.tx_clean); 2992 u64_stats_update_end(&txq_stats->napi_syncp); 2993 2994 priv->xstats.tx_errors += tx_errors; 2995 2996 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2997 2998 /* Combine decisions from TX clean and XSK TX */ 2999 return max(count, xmits); 3000 } 3001 3002 /** 3003 * stmmac_tx_err - to manage the tx error 3004 * @priv: driver private structure 3005 * @chan: channel index 3006 * Description: it cleans the descriptors and restarts the transmission 3007 * in case of transmission errors. 3008 */ 3009 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 3010 { 3011 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3012 3013 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 3014 3015 stmmac_stop_tx_dma(priv, chan); 3016 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 3017 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 3018 stmmac_reset_tx_queue(priv, chan); 3019 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3020 tx_q->dma_tx_phy, chan); 3021 stmmac_start_tx_dma(priv, chan); 3022 3023 priv->xstats.tx_errors++; 3024 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 3025 } 3026 3027 /** 3028 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 3029 * @priv: driver private structure 3030 * @txmode: TX operating mode 3031 * @rxmode: RX operating mode 3032 * @chan: channel index 3033 * Description: it is used for configuring of the DMA operation mode in 3034 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 3035 * mode. 3036 */ 3037 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 3038 u32 rxmode, u32 chan) 3039 { 3040 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 3041 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 3042 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3043 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3044 int rxfifosz = priv->plat->rx_fifo_size; 3045 int txfifosz = priv->plat->tx_fifo_size; 3046 3047 if (rxfifosz == 0) 3048 rxfifosz = priv->dma_cap.rx_fifo_size; 3049 if (txfifosz == 0) 3050 txfifosz = priv->dma_cap.tx_fifo_size; 3051 3052 /* Adjust for real per queue fifo size */ 3053 rxfifosz /= rx_channels_count; 3054 txfifosz /= tx_channels_count; 3055 3056 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 3057 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 3058 } 3059 3060 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 3061 { 3062 int ret; 3063 3064 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 3065 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 3066 if (ret && (ret != -EINVAL)) { 3067 stmmac_global_err(priv); 3068 return true; 3069 } 3070 3071 return false; 3072 } 3073 3074 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 3075 { 3076 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 3077 &priv->xstats, chan, dir); 3078 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 3079 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3080 struct stmmac_channel *ch = &priv->channel[chan]; 3081 struct napi_struct *rx_napi; 3082 struct napi_struct *tx_napi; 3083 unsigned long flags; 3084 3085 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 3086 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3087 3088 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 3089 if (napi_schedule_prep(rx_napi)) { 3090 spin_lock_irqsave(&ch->lock, flags); 3091 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 3092 spin_unlock_irqrestore(&ch->lock, flags); 3093 __napi_schedule(rx_napi); 3094 } 3095 } 3096 3097 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 3098 if (napi_schedule_prep(tx_napi)) { 3099 spin_lock_irqsave(&ch->lock, flags); 3100 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 3101 spin_unlock_irqrestore(&ch->lock, flags); 3102 __napi_schedule(tx_napi); 3103 } 3104 } 3105 3106 return status; 3107 } 3108 3109 /** 3110 * stmmac_dma_interrupt - DMA ISR 3111 * @priv: driver private structure 3112 * Description: this is the DMA ISR. It is called by the main ISR. 3113 * It calls the dwmac dma routine and schedule poll method in case of some 3114 * work can be done. 3115 */ 3116 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 3117 { 3118 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3119 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3120 u8 channels_to_check = tx_channel_count > rx_channel_count ? 3121 tx_channel_count : rx_channel_count; 3122 int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 3123 u8 chan; 3124 3125 /* Make sure we never check beyond our status buffer. */ 3126 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 3127 channels_to_check = ARRAY_SIZE(status); 3128 3129 for (chan = 0; chan < channels_to_check; chan++) 3130 status[chan] = stmmac_napi_check(priv, chan, 3131 DMA_DIR_RXTX); 3132 3133 for (chan = 0; chan < tx_channel_count; chan++) { 3134 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 3135 /* Try to bump up the dma threshold on this failure */ 3136 stmmac_bump_dma_threshold(priv, chan); 3137 } else if (unlikely(status[chan] == tx_hard_error)) { 3138 stmmac_tx_err(priv, chan); 3139 } 3140 } 3141 } 3142 3143 /** 3144 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 3145 * @priv: driver private structure 3146 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 3147 */ 3148 static void stmmac_mmc_setup(struct stmmac_priv *priv) 3149 { 3150 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 3151 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 3152 3153 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 3154 3155 if (priv->dma_cap.rmon) { 3156 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 3157 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 3158 } else 3159 netdev_info(priv->dev, "No MAC Management Counters available\n"); 3160 } 3161 3162 /** 3163 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 3164 * @priv: driver private structure 3165 * Description: 3166 * new GMAC chip generations have a new register to indicate the 3167 * presence of the optional feature/functions. 3168 * This can be also used to override the value passed through the 3169 * platform and necessary for old MAC10/100 and GMAC chips. 3170 */ 3171 static int stmmac_get_hw_features(struct stmmac_priv *priv) 3172 { 3173 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 3174 } 3175 3176 /** 3177 * stmmac_check_ether_addr - check if the MAC addr is valid 3178 * @priv: driver private structure 3179 * Description: 3180 * it is to verify if the MAC address is valid, in case of failures it 3181 * generates a random MAC address 3182 */ 3183 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 3184 { 3185 u8 addr[ETH_ALEN]; 3186 3187 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 3188 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 3189 if (is_valid_ether_addr(addr)) 3190 eth_hw_addr_set(priv->dev, addr); 3191 else 3192 eth_hw_addr_random(priv->dev); 3193 dev_info(priv->device, "device MAC address %pM\n", 3194 priv->dev->dev_addr); 3195 } 3196 } 3197 3198 int stmmac_get_phy_intf_sel(phy_interface_t interface) 3199 { 3200 int phy_intf_sel = -EINVAL; 3201 3202 if (interface == PHY_INTERFACE_MODE_MII || 3203 interface == PHY_INTERFACE_MODE_GMII) 3204 phy_intf_sel = PHY_INTF_SEL_GMII_MII; 3205 else if (phy_interface_mode_is_rgmii(interface)) 3206 phy_intf_sel = PHY_INTF_SEL_RGMII; 3207 else if (interface == PHY_INTERFACE_MODE_RMII) 3208 phy_intf_sel = PHY_INTF_SEL_RMII; 3209 else if (interface == PHY_INTERFACE_MODE_REVMII) 3210 phy_intf_sel = PHY_INTF_SEL_REVMII; 3211 3212 return phy_intf_sel; 3213 } 3214 EXPORT_SYMBOL_GPL(stmmac_get_phy_intf_sel); 3215 3216 static int stmmac_prereset_configure(struct stmmac_priv *priv) 3217 { 3218 struct plat_stmmacenet_data *plat_dat = priv->plat; 3219 phy_interface_t interface; 3220 struct phylink_pcs *pcs; 3221 int phy_intf_sel, ret; 3222 3223 if (!plat_dat->set_phy_intf_sel) 3224 return 0; 3225 3226 interface = plat_dat->phy_interface; 3227 3228 /* Check whether this mode uses a PCS */ 3229 pcs = stmmac_mac_select_pcs(&priv->phylink_config, interface); 3230 if (priv->integrated_pcs && pcs == &priv->integrated_pcs->pcs) { 3231 /* Request the phy_intf_sel from the integrated PCS */ 3232 phy_intf_sel = stmmac_integrated_pcs_get_phy_intf_sel(pcs, 3233 interface); 3234 } else { 3235 phy_intf_sel = stmmac_get_phy_intf_sel(interface); 3236 } 3237 3238 if (phy_intf_sel < 0) { 3239 netdev_err(priv->dev, 3240 "failed to get phy_intf_sel for %s: %pe\n", 3241 phy_modes(interface), ERR_PTR(phy_intf_sel)); 3242 return phy_intf_sel; 3243 } 3244 3245 ret = plat_dat->set_phy_intf_sel(plat_dat->bsp_priv, phy_intf_sel); 3246 if (ret == -EINVAL) 3247 netdev_err(priv->dev, "platform does not support %s\n", 3248 phy_modes(interface)); 3249 else if (ret < 0) 3250 netdev_err(priv->dev, 3251 "platform failed to set interface %s: %pe\n", 3252 phy_modes(interface), ERR_PTR(ret)); 3253 3254 return ret; 3255 } 3256 3257 /** 3258 * stmmac_init_dma_engine - DMA init. 3259 * @priv: driver private structure 3260 * Description: 3261 * It inits the DMA invoking the specific MAC/GMAC callback. 3262 * Some DMA parameters can be passed from the platform; 3263 * in case of these are not passed a default is kept for the MAC or GMAC. 3264 */ 3265 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 3266 { 3267 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3268 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3269 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 3270 struct stmmac_rx_queue *rx_q; 3271 struct stmmac_tx_queue *tx_q; 3272 int ret = 0; 3273 u8 chan; 3274 3275 ret = stmmac_prereset_configure(priv); 3276 if (ret) 3277 return ret; 3278 3279 ret = stmmac_reset(priv); 3280 if (ret) { 3281 netdev_err(priv->dev, "Failed to reset the dma\n"); 3282 return ret; 3283 } 3284 3285 /* DMA Configuration */ 3286 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); 3287 3288 if (priv->plat->axi) 3289 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3290 3291 /* DMA CSR Channel configuration */ 3292 for (chan = 0; chan < dma_csr_ch; chan++) { 3293 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3294 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3295 } 3296 3297 /* DMA RX Channel Configuration */ 3298 for (chan = 0; chan < rx_channels_count; chan++) { 3299 rx_q = &priv->dma_conf.rx_queue[chan]; 3300 3301 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3302 rx_q->dma_rx_phy, chan); 3303 3304 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 3305 rx_q->buf_alloc_num); 3306 } 3307 3308 /* DMA TX Channel Configuration */ 3309 for (chan = 0; chan < tx_channels_count; chan++) { 3310 tx_q = &priv->dma_conf.tx_queue[chan]; 3311 3312 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3313 tx_q->dma_tx_phy, chan); 3314 3315 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 3316 } 3317 3318 return ret; 3319 } 3320 3321 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3322 { 3323 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3324 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3325 struct stmmac_channel *ch; 3326 struct napi_struct *napi; 3327 3328 if (!tx_coal_timer) 3329 return; 3330 3331 ch = &priv->channel[queue]; 3332 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3333 3334 /* Arm timer only if napi is not already scheduled. 3335 * Try to cancel any timer if napi is scheduled, timer will be armed 3336 * again in the next scheduled napi. 3337 */ 3338 if (unlikely(!napi_is_scheduled(napi))) { 3339 if (unlikely(!(hrtimer_active(&tx_q->txtimer)))) 3340 hrtimer_start(&tx_q->txtimer, 3341 STMMAC_COAL_TIMER(tx_coal_timer), 3342 HRTIMER_MODE_REL); 3343 } else { 3344 hrtimer_try_to_cancel(&tx_q->txtimer); 3345 } 3346 } 3347 3348 /** 3349 * stmmac_tx_timer - mitigation sw timer for tx. 3350 * @t: data pointer 3351 * Description: 3352 * This is the timer handler to directly invoke the stmmac_tx_clean. 3353 */ 3354 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3355 { 3356 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3357 struct stmmac_priv *priv = tx_q->priv_data; 3358 struct stmmac_channel *ch; 3359 struct napi_struct *napi; 3360 3361 ch = &priv->channel[tx_q->queue_index]; 3362 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3363 3364 if (likely(napi_schedule_prep(napi))) { 3365 unsigned long flags; 3366 3367 spin_lock_irqsave(&ch->lock, flags); 3368 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3369 spin_unlock_irqrestore(&ch->lock, flags); 3370 __napi_schedule(napi); 3371 } 3372 3373 return HRTIMER_NORESTART; 3374 } 3375 3376 /** 3377 * stmmac_init_coalesce - init mitigation options. 3378 * @priv: driver private structure 3379 * Description: 3380 * This inits the coalesce parameters: i.e. timer rate, 3381 * timer handler and default threshold used for enabling the 3382 * interrupt on completion bit. 3383 */ 3384 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3385 { 3386 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3387 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3388 u8 chan; 3389 3390 for (chan = 0; chan < tx_channel_count; chan++) { 3391 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3392 3393 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3394 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3395 3396 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3397 } 3398 3399 for (chan = 0; chan < rx_channel_count; chan++) 3400 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3401 } 3402 3403 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3404 { 3405 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3406 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3407 u8 chan; 3408 3409 /* set TX ring length */ 3410 for (chan = 0; chan < tx_channels_count; chan++) 3411 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3412 (priv->dma_conf.dma_tx_size - 1), chan); 3413 3414 /* set RX ring length */ 3415 for (chan = 0; chan < rx_channels_count; chan++) 3416 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3417 (priv->dma_conf.dma_rx_size - 1), chan); 3418 } 3419 3420 /** 3421 * stmmac_set_tx_queue_weight - Set TX queue weight 3422 * @priv: driver private structure 3423 * Description: It is used for setting TX queues weight 3424 */ 3425 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3426 { 3427 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3428 u32 weight; 3429 u8 queue; 3430 3431 for (queue = 0; queue < tx_queues_count; queue++) { 3432 weight = priv->plat->tx_queues_cfg[queue].weight; 3433 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3434 } 3435 } 3436 3437 /** 3438 * stmmac_configure_cbs - Configure CBS in TX queue 3439 * @priv: driver private structure 3440 * Description: It is used for configuring CBS in AVB TX queues 3441 */ 3442 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3443 { 3444 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3445 u32 mode_to_use; 3446 u8 queue; 3447 3448 /* queue 0 is reserved for legacy traffic */ 3449 for (queue = 1; queue < tx_queues_count; queue++) { 3450 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3451 if (mode_to_use == MTL_QUEUE_DCB) 3452 continue; 3453 3454 stmmac_config_cbs(priv, priv->hw, 3455 priv->plat->tx_queues_cfg[queue].send_slope, 3456 priv->plat->tx_queues_cfg[queue].idle_slope, 3457 priv->plat->tx_queues_cfg[queue].high_credit, 3458 priv->plat->tx_queues_cfg[queue].low_credit, 3459 queue); 3460 } 3461 } 3462 3463 /** 3464 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3465 * @priv: driver private structure 3466 * Description: It is used for mapping RX queues to RX dma channels 3467 */ 3468 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3469 { 3470 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3471 u8 queue; 3472 u32 chan; 3473 3474 for (queue = 0; queue < rx_queues_count; queue++) { 3475 chan = priv->plat->rx_queues_cfg[queue].chan; 3476 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3477 } 3478 } 3479 3480 /** 3481 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3482 * @priv: driver private structure 3483 * Description: It is used for configuring the RX Queue Priority 3484 */ 3485 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3486 { 3487 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3488 u8 queue; 3489 u32 prio; 3490 3491 for (queue = 0; queue < rx_queues_count; queue++) { 3492 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3493 continue; 3494 3495 prio = priv->plat->rx_queues_cfg[queue].prio; 3496 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3497 } 3498 } 3499 3500 /** 3501 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3502 * @priv: driver private structure 3503 * Description: It is used for configuring the TX Queue Priority 3504 */ 3505 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3506 { 3507 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3508 u8 queue; 3509 u32 prio; 3510 3511 for (queue = 0; queue < tx_queues_count; queue++) { 3512 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3513 continue; 3514 3515 prio = priv->plat->tx_queues_cfg[queue].prio; 3516 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3517 } 3518 } 3519 3520 /** 3521 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3522 * @priv: driver private structure 3523 * Description: It is used for configuring the RX queue routing 3524 */ 3525 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3526 { 3527 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3528 u8 packet; 3529 u8 queue; 3530 3531 for (queue = 0; queue < rx_queues_count; queue++) { 3532 /* no specific packet type routing specified for the queue */ 3533 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3534 continue; 3535 3536 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3537 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3538 } 3539 } 3540 3541 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3542 { 3543 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3544 priv->rss.enable = false; 3545 return; 3546 } 3547 3548 if (priv->dev->features & NETIF_F_RXHASH) 3549 priv->rss.enable = true; 3550 else 3551 priv->rss.enable = false; 3552 3553 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3554 priv->plat->rx_queues_to_use); 3555 } 3556 3557 /** 3558 * stmmac_mtl_configuration - Configure MTL 3559 * @priv: driver private structure 3560 * Description: It is used for configuring MTL 3561 */ 3562 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3563 { 3564 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3565 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3566 3567 if (tx_queues_count > 1) 3568 stmmac_set_tx_queue_weight(priv); 3569 3570 /* Configure MTL RX algorithms */ 3571 if (rx_queues_count > 1) 3572 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3573 priv->plat->rx_sched_algorithm); 3574 3575 /* Configure MTL TX algorithms */ 3576 if (tx_queues_count > 1) 3577 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3578 priv->plat->tx_sched_algorithm); 3579 3580 /* Configure CBS in AVB TX queues */ 3581 if (tx_queues_count > 1) 3582 stmmac_configure_cbs(priv); 3583 3584 /* Map RX MTL to DMA channels */ 3585 stmmac_rx_queue_dma_chan_map(priv); 3586 3587 /* Enable MAC RX Queues */ 3588 stmmac_mac_enable_rx_queues(priv); 3589 3590 /* Set RX priorities */ 3591 if (rx_queues_count > 1) 3592 stmmac_mac_config_rx_queues_prio(priv); 3593 3594 /* Set TX priorities */ 3595 if (tx_queues_count > 1) 3596 stmmac_mac_config_tx_queues_prio(priv); 3597 3598 /* Set RX routing */ 3599 if (rx_queues_count > 1) 3600 stmmac_mac_config_rx_queues_routing(priv); 3601 3602 /* Receive Side Scaling */ 3603 if (rx_queues_count > 1) 3604 stmmac_mac_config_rss(priv); 3605 } 3606 3607 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3608 { 3609 if (priv->dma_cap.asp) { 3610 netdev_info(priv->dev, "Enabling Safety Features\n"); 3611 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3612 priv->plat->safety_feat_cfg); 3613 } else { 3614 netdev_info(priv->dev, "No Safety Features support found\n"); 3615 } 3616 } 3617 3618 /* STM32MP25xx (dwmac v5.3) states "Do not enable time-based scheduling for 3619 * channels on which the TSO feature is enabled." If we have a skb for a 3620 * channel which has TBS enabled, fall back to software GSO. 3621 */ 3622 static bool stmmac_tso_channel_permitted(struct stmmac_priv *priv, 3623 unsigned int chan) 3624 { 3625 /* TSO and TBS cannot co-exist */ 3626 return !(priv->dma_conf.tx_queue[chan].tbs & STMMAC_TBS_AVAIL); 3627 } 3628 3629 /** 3630 * stmmac_hw_setup - setup mac in a usable state. 3631 * @dev : pointer to the device structure. 3632 * Description: 3633 * this is the main function to setup the HW in a usable state because the 3634 * dma engine is reset, the core registers are configured (e.g. AXI, 3635 * Checksum features, timers). The DMA is ready to start receiving and 3636 * transmitting. 3637 * Return value: 3638 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3639 * file on failure. 3640 */ 3641 static int stmmac_hw_setup(struct net_device *dev) 3642 { 3643 struct stmmac_priv *priv = netdev_priv(dev); 3644 u8 rx_cnt = priv->plat->rx_queues_to_use; 3645 u8 tx_cnt = priv->plat->tx_queues_to_use; 3646 bool sph_en; 3647 u8 chan; 3648 int ret; 3649 3650 /* Make sure RX clock is enabled */ 3651 if (priv->hw->phylink_pcs) 3652 phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs); 3653 3654 /* Note that clk_rx_i must be running for reset to complete. This 3655 * clock may also be required when setting the MAC address. 3656 * 3657 * Block the receive clock stop for LPI mode at the PHY in case 3658 * the link is established with EEE mode active. 3659 */ 3660 phylink_rx_clk_stop_block(priv->phylink); 3661 3662 /* DMA initialization and SW reset */ 3663 ret = stmmac_init_dma_engine(priv); 3664 if (ret < 0) { 3665 phylink_rx_clk_stop_unblock(priv->phylink); 3666 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3667 __func__); 3668 return ret; 3669 } 3670 3671 /* Copy the MAC addr into the HW */ 3672 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3673 phylink_rx_clk_stop_unblock(priv->phylink); 3674 3675 /* Initialize the MAC Core */ 3676 stmmac_core_init(priv, priv->hw, dev); 3677 3678 /* Initialize MTL*/ 3679 stmmac_mtl_configuration(priv); 3680 3681 /* Apply the RX packet parser table */ 3682 if (priv->tc_entries) { 3683 ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries, 3684 priv->tc_entries_max); 3685 if (ret) 3686 return ret; 3687 } 3688 3689 /* Initialize Safety Features */ 3690 stmmac_safety_feat_configuration(priv); 3691 3692 ret = stmmac_rx_ipc(priv, priv->hw); 3693 if (!ret) { 3694 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3695 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3696 priv->hw->rx_csum = 0; 3697 } 3698 3699 /* Enable the MAC Rx/Tx */ 3700 stmmac_mac_set(priv, priv->ioaddr, true); 3701 3702 /* Set the HW DMA mode and the COE */ 3703 stmmac_dma_operation_mode(priv); 3704 3705 stmmac_mmc_setup(priv); 3706 3707 if (priv->use_riwt) { 3708 u32 queue; 3709 3710 for (queue = 0; queue < rx_cnt; queue++) { 3711 if (!priv->rx_riwt[queue]) 3712 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3713 3714 stmmac_rx_watchdog(priv, priv->ioaddr, 3715 priv->rx_riwt[queue], queue); 3716 } 3717 } 3718 3719 /* set TX and RX rings length */ 3720 stmmac_set_rings_length(priv); 3721 3722 /* Enable TSO */ 3723 if (priv->dma_cap.tsoen && priv->plat->flags & STMMAC_FLAG_TSO_EN) { 3724 for (chan = 0; chan < tx_cnt; chan++) { 3725 if (!stmmac_tso_channel_permitted(priv, chan)) 3726 continue; 3727 3728 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3729 } 3730 } 3731 3732 /* Enable Split Header */ 3733 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 3734 for (chan = 0; chan < rx_cnt; chan++) 3735 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3736 3737 3738 /* VLAN Tag Insertion */ 3739 if (priv->dma_cap.vlins) 3740 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3741 3742 /* TBS */ 3743 for (chan = 0; chan < tx_cnt; chan++) { 3744 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3745 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3746 3747 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3748 } 3749 3750 /* Configure real RX and TX queues */ 3751 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3752 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3753 3754 /* Start the ball rolling... */ 3755 stmmac_start_all_dma(priv); 3756 3757 phylink_rx_clk_stop_block(priv->phylink); 3758 stmmac_set_hw_vlan_mode(priv, priv->hw); 3759 phylink_rx_clk_stop_unblock(priv->phylink); 3760 3761 return 0; 3762 } 3763 3764 static void stmmac_free_irq(struct net_device *dev, 3765 enum request_irq_err irq_err, int irq_idx) 3766 { 3767 struct stmmac_priv *priv = netdev_priv(dev); 3768 struct stmmac_msi *msi = priv->msi; 3769 int j; 3770 3771 switch (irq_err) { 3772 case REQ_IRQ_ERR_ALL: 3773 irq_idx = priv->plat->tx_queues_to_use; 3774 fallthrough; 3775 case REQ_IRQ_ERR_TX: 3776 for (j = irq_idx - 1; msi && j >= 0; j--) { 3777 if (msi->tx_irq[j] > 0) { 3778 irq_set_affinity_hint(msi->tx_irq[j], NULL); 3779 free_irq(msi->tx_irq[j], 3780 &priv->dma_conf.tx_queue[j]); 3781 } 3782 } 3783 irq_idx = priv->plat->rx_queues_to_use; 3784 fallthrough; 3785 case REQ_IRQ_ERR_RX: 3786 for (j = irq_idx - 1; msi && j >= 0; j--) { 3787 if (msi->rx_irq[j] > 0) { 3788 irq_set_affinity_hint(msi->rx_irq[j], NULL); 3789 free_irq(msi->rx_irq[j], 3790 &priv->dma_conf.rx_queue[j]); 3791 } 3792 } 3793 3794 if (msi && msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) 3795 free_irq(msi->sfty_ue_irq, dev); 3796 fallthrough; 3797 case REQ_IRQ_ERR_SFTY_UE: 3798 if (msi && msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) 3799 free_irq(msi->sfty_ce_irq, dev); 3800 fallthrough; 3801 case REQ_IRQ_ERR_SFTY_CE: 3802 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3803 free_irq(priv->wol_irq, dev); 3804 fallthrough; 3805 case REQ_IRQ_ERR_SFTY: 3806 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) 3807 free_irq(priv->sfty_irq, dev); 3808 fallthrough; 3809 case REQ_IRQ_ERR_WOL: 3810 free_irq(dev->irq, dev); 3811 fallthrough; 3812 case REQ_IRQ_ERR_MAC: 3813 case REQ_IRQ_ERR_NO: 3814 /* If MAC IRQ request error, no more IRQ to free */ 3815 break; 3816 } 3817 } 3818 3819 static int stmmac_msi_init(struct stmmac_priv *priv, 3820 struct stmmac_resources *res) 3821 { 3822 int i; 3823 3824 priv->msi = devm_kmalloc(priv->device, sizeof(*priv->msi), GFP_KERNEL); 3825 if (!priv->msi) 3826 return -ENOMEM; 3827 3828 priv->msi->sfty_ce_irq = res->sfty_ce_irq; 3829 priv->msi->sfty_ue_irq = res->sfty_ue_irq; 3830 3831 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 3832 priv->msi->rx_irq[i] = res->rx_irq[i]; 3833 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 3834 priv->msi->tx_irq[i] = res->tx_irq[i]; 3835 3836 return 0; 3837 } 3838 3839 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3840 { 3841 struct stmmac_priv *priv = netdev_priv(dev); 3842 struct stmmac_msi *msi = priv->msi; 3843 enum request_irq_err irq_err; 3844 int irq_idx = 0; 3845 char *int_name; 3846 int ret; 3847 int i; 3848 3849 /* For common interrupt */ 3850 int_name = msi->int_name_mac; 3851 sprintf(int_name, "%s:%s", dev->name, "mac"); 3852 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3853 0, int_name, dev); 3854 if (unlikely(ret < 0)) { 3855 netdev_err(priv->dev, 3856 "%s: alloc mac MSI %d (error: %d)\n", 3857 __func__, dev->irq, ret); 3858 irq_err = REQ_IRQ_ERR_MAC; 3859 goto irq_error; 3860 } 3861 3862 /* Request the Wake IRQ in case of another line 3863 * is used for WoL 3864 */ 3865 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3866 int_name = msi->int_name_wol; 3867 sprintf(int_name, "%s:%s", dev->name, "wol"); 3868 ret = request_irq(priv->wol_irq, 3869 stmmac_mac_interrupt, 3870 0, int_name, dev); 3871 if (unlikely(ret < 0)) { 3872 netdev_err(priv->dev, 3873 "%s: alloc wol MSI %d (error: %d)\n", 3874 __func__, priv->wol_irq, ret); 3875 irq_err = REQ_IRQ_ERR_WOL; 3876 goto irq_error; 3877 } 3878 } 3879 3880 /* Request the common Safety Feature Correctible/Uncorrectible 3881 * Error line in case of another line is used 3882 */ 3883 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3884 int_name = msi->int_name_sfty; 3885 sprintf(int_name, "%s:%s", dev->name, "safety"); 3886 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3887 0, int_name, dev); 3888 if (unlikely(ret < 0)) { 3889 netdev_err(priv->dev, 3890 "%s: alloc sfty MSI %d (error: %d)\n", 3891 __func__, priv->sfty_irq, ret); 3892 irq_err = REQ_IRQ_ERR_SFTY; 3893 goto irq_error; 3894 } 3895 } 3896 3897 /* Request the Safety Feature Correctible Error line in 3898 * case of another line is used 3899 */ 3900 if (msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) { 3901 int_name = msi->int_name_sfty_ce; 3902 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3903 ret = request_irq(msi->sfty_ce_irq, 3904 stmmac_safety_interrupt, 3905 0, int_name, dev); 3906 if (unlikely(ret < 0)) { 3907 netdev_err(priv->dev, 3908 "%s: alloc sfty ce MSI %d (error: %d)\n", 3909 __func__, msi->sfty_ce_irq, ret); 3910 irq_err = REQ_IRQ_ERR_SFTY_CE; 3911 goto irq_error; 3912 } 3913 } 3914 3915 /* Request the Safety Feature Uncorrectible Error line in 3916 * case of another line is used 3917 */ 3918 if (msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) { 3919 int_name = msi->int_name_sfty_ue; 3920 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3921 ret = request_irq(msi->sfty_ue_irq, 3922 stmmac_safety_interrupt, 3923 0, int_name, dev); 3924 if (unlikely(ret < 0)) { 3925 netdev_err(priv->dev, 3926 "%s: alloc sfty ue MSI %d (error: %d)\n", 3927 __func__, msi->sfty_ue_irq, ret); 3928 irq_err = REQ_IRQ_ERR_SFTY_UE; 3929 goto irq_error; 3930 } 3931 } 3932 3933 /* Request Rx MSI irq */ 3934 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3935 if (i >= MTL_MAX_RX_QUEUES) 3936 break; 3937 if (msi->rx_irq[i] == 0) 3938 continue; 3939 3940 int_name = msi->int_name_rx_irq[i]; 3941 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3942 ret = request_irq(msi->rx_irq[i], 3943 stmmac_msi_intr_rx, 3944 0, int_name, &priv->dma_conf.rx_queue[i]); 3945 if (unlikely(ret < 0)) { 3946 netdev_err(priv->dev, 3947 "%s: alloc rx-%d MSI %d (error: %d)\n", 3948 __func__, i, msi->rx_irq[i], ret); 3949 irq_err = REQ_IRQ_ERR_RX; 3950 irq_idx = i; 3951 goto irq_error; 3952 } 3953 irq_set_affinity_hint(msi->rx_irq[i], 3954 cpumask_of(i % num_online_cpus())); 3955 } 3956 3957 /* Request Tx MSI irq */ 3958 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3959 if (i >= MTL_MAX_TX_QUEUES) 3960 break; 3961 if (msi->tx_irq[i] == 0) 3962 continue; 3963 3964 int_name = msi->int_name_tx_irq[i]; 3965 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3966 ret = request_irq(msi->tx_irq[i], 3967 stmmac_msi_intr_tx, 3968 0, int_name, &priv->dma_conf.tx_queue[i]); 3969 if (unlikely(ret < 0)) { 3970 netdev_err(priv->dev, 3971 "%s: alloc tx-%d MSI %d (error: %d)\n", 3972 __func__, i, msi->tx_irq[i], ret); 3973 irq_err = REQ_IRQ_ERR_TX; 3974 irq_idx = i; 3975 goto irq_error; 3976 } 3977 irq_set_affinity_hint(msi->tx_irq[i], 3978 cpumask_of(i % num_online_cpus())); 3979 } 3980 3981 return 0; 3982 3983 irq_error: 3984 stmmac_free_irq(dev, irq_err, irq_idx); 3985 return ret; 3986 } 3987 3988 static int stmmac_request_irq_single(struct net_device *dev) 3989 { 3990 struct stmmac_priv *priv = netdev_priv(dev); 3991 enum request_irq_err irq_err; 3992 int ret; 3993 3994 ret = request_irq(dev->irq, stmmac_interrupt, 3995 IRQF_SHARED, dev->name, dev); 3996 if (unlikely(ret < 0)) { 3997 netdev_err(priv->dev, 3998 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3999 __func__, dev->irq, ret); 4000 irq_err = REQ_IRQ_ERR_MAC; 4001 goto irq_error; 4002 } 4003 4004 /* Request the Wake IRQ in case of another line 4005 * is used for WoL 4006 */ 4007 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 4008 ret = request_irq(priv->wol_irq, stmmac_interrupt, 4009 IRQF_SHARED, dev->name, dev); 4010 if (unlikely(ret < 0)) { 4011 netdev_err(priv->dev, 4012 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 4013 __func__, priv->wol_irq, ret); 4014 irq_err = REQ_IRQ_ERR_WOL; 4015 goto irq_error; 4016 } 4017 } 4018 4019 /* Request the common Safety Feature Correctible/Uncorrectible 4020 * Error line in case of another line is used 4021 */ 4022 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 4023 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 4024 IRQF_SHARED, dev->name, dev); 4025 if (unlikely(ret < 0)) { 4026 netdev_err(priv->dev, 4027 "%s: ERROR: allocating the sfty IRQ %d (%d)\n", 4028 __func__, priv->sfty_irq, ret); 4029 irq_err = REQ_IRQ_ERR_SFTY; 4030 goto irq_error; 4031 } 4032 } 4033 4034 return 0; 4035 4036 irq_error: 4037 stmmac_free_irq(dev, irq_err, 0); 4038 return ret; 4039 } 4040 4041 static int stmmac_request_irq(struct net_device *dev) 4042 { 4043 struct stmmac_priv *priv = netdev_priv(dev); 4044 int ret; 4045 4046 /* Request the IRQ lines */ 4047 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 4048 ret = stmmac_request_irq_multi_msi(dev); 4049 else 4050 ret = stmmac_request_irq_single(dev); 4051 4052 return ret; 4053 } 4054 4055 /** 4056 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 4057 * @priv: driver private structure 4058 * @mtu: MTU to setup the dma queue and buf with 4059 * Description: Allocate and generate a dma_conf based on the provided MTU. 4060 * Allocate the Tx/Rx DMA queue and init them. 4061 * Return value: 4062 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 4063 */ 4064 static struct stmmac_dma_conf * 4065 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 4066 { 4067 struct stmmac_dma_conf *dma_conf; 4068 int bfsize, len, ret; 4069 u8 chan; 4070 4071 dma_conf = kzalloc_obj(*dma_conf); 4072 if (!dma_conf) { 4073 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 4074 __func__); 4075 return ERR_PTR(-ENOMEM); 4076 } 4077 4078 len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; 4079 4080 /* Returns 0 or BUF_SIZE_16KiB if len > 8KiB and dwmac4 or ring mode */ 4081 bfsize = stmmac_set_16kib_bfsize(priv, len); 4082 if (bfsize < 0) 4083 bfsize = 0; 4084 4085 if (bfsize < BUF_SIZE_16KiB) 4086 bfsize = stmmac_set_bfsize(len); 4087 4088 dma_conf->dma_buf_sz = bfsize; 4089 /* Chose the tx/rx size from the already defined one in the 4090 * priv struct. (if defined) 4091 */ 4092 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 4093 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 4094 4095 if (!dma_conf->dma_tx_size) 4096 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 4097 if (!dma_conf->dma_rx_size) 4098 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 4099 4100 /* Earlier check for TBS */ 4101 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 4102 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 4103 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 4104 4105 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 4106 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 4107 } 4108 4109 ret = alloc_dma_desc_resources(priv, dma_conf); 4110 if (ret < 0) { 4111 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 4112 __func__); 4113 goto alloc_error; 4114 } 4115 4116 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 4117 if (ret < 0) { 4118 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 4119 __func__); 4120 goto init_error; 4121 } 4122 4123 return dma_conf; 4124 4125 init_error: 4126 free_dma_desc_resources(priv, dma_conf); 4127 alloc_error: 4128 kfree(dma_conf); 4129 return ERR_PTR(ret); 4130 } 4131 4132 /** 4133 * __stmmac_open - open entry point of the driver 4134 * @dev : pointer to the device structure. 4135 * @dma_conf : structure to take the dma data 4136 * Description: 4137 * This function is the open entry point of the driver. 4138 * Return value: 4139 * 0 on success and an appropriate (-)ve integer as defined in errno.h 4140 * file on failure. 4141 */ 4142 static int __stmmac_open(struct net_device *dev, 4143 struct stmmac_dma_conf *dma_conf) 4144 { 4145 struct stmmac_priv *priv = netdev_priv(dev); 4146 u8 chan; 4147 int ret; 4148 4149 for (int i = 0; i < priv->plat->tx_queues_to_use; i++) 4150 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 4151 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 4152 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 4153 4154 /* The PHY is suspended when the interface is reopened without 4155 * disconnecting the PHY, e.g. on MTU change. IEEE 802.3 allows PHYs 4156 * to stop their receive clock while powered down, but the DMA 4157 * software reset in stmmac_hw_setup() requires a running receive 4158 * clock, and phylink_start() below resumes the PHY only after the 4159 * hardware setup. Resume a suspended PHY here first. 4160 */ 4161 phylink_prepare_resume(priv->phylink); 4162 4163 stmmac_reset_queues_param(priv); 4164 4165 ret = stmmac_hw_setup(dev); 4166 if (ret < 0) { 4167 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 4168 goto init_error; 4169 } 4170 4171 stmmac_setup_ptp(priv); 4172 4173 stmmac_init_coalesce(priv); 4174 4175 phylink_start(priv->phylink); 4176 4177 stmmac_vlan_restore(priv); 4178 4179 ret = stmmac_request_irq(dev); 4180 if (ret) 4181 goto irq_error; 4182 4183 stmmac_enable_all_queues(priv); 4184 netif_tx_start_all_queues(priv->dev); 4185 stmmac_enable_all_dma_irq(priv); 4186 4187 return 0; 4188 4189 irq_error: 4190 phylink_stop(priv->phylink); 4191 4192 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4193 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4194 4195 stmmac_release_ptp(priv); 4196 init_error: 4197 return ret; 4198 } 4199 4200 static int stmmac_open(struct net_device *dev) 4201 { 4202 struct stmmac_priv *priv = netdev_priv(dev); 4203 struct stmmac_dma_conf *dma_conf; 4204 int ret; 4205 4206 /* Initialise the tx lpi timer, converting from msec to usec */ 4207 if (!priv->tx_lpi_timer) 4208 priv->tx_lpi_timer = eee_timer * 1000; 4209 4210 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 4211 if (IS_ERR(dma_conf)) 4212 return PTR_ERR(dma_conf); 4213 4214 ret = pm_runtime_resume_and_get(priv->device); 4215 if (ret < 0) 4216 goto err_dma_resources; 4217 4218 ret = stmmac_init_phy(dev); 4219 if (ret) 4220 goto err_runtime_pm; 4221 4222 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 4223 ret = stmmac_legacy_serdes_power_up(priv); 4224 if (ret < 0) 4225 goto err_disconnect_phy; 4226 } 4227 4228 ret = __stmmac_open(dev, dma_conf); 4229 if (ret) 4230 goto err_serdes; 4231 4232 kfree(dma_conf); 4233 4234 /* We may have called phylink_speed_down before */ 4235 phylink_speed_up(priv->phylink); 4236 4237 return ret; 4238 4239 err_serdes: 4240 stmmac_legacy_serdes_power_down(priv); 4241 err_disconnect_phy: 4242 phylink_disconnect_phy(priv->phylink); 4243 err_runtime_pm: 4244 pm_runtime_put(priv->device); 4245 err_dma_resources: 4246 free_dma_desc_resources(priv, dma_conf); 4247 kfree(dma_conf); 4248 return ret; 4249 } 4250 4251 static void __stmmac_release(struct net_device *dev) 4252 { 4253 struct stmmac_priv *priv = netdev_priv(dev); 4254 u8 chan; 4255 4256 /* Stop and disconnect the PHY */ 4257 phylink_stop(priv->phylink); 4258 4259 stmmac_disable_all_queues(priv); 4260 4261 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4262 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4263 4264 netif_tx_disable(dev); 4265 4266 /* Free the IRQ lines */ 4267 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4268 4269 /* Stop TX/RX DMA and clear the descriptors */ 4270 stmmac_stop_all_dma(priv); 4271 4272 /* Release and free the Rx/Tx resources */ 4273 free_dma_desc_resources(priv, &priv->dma_conf); 4274 4275 stmmac_release_ptp(priv); 4276 4277 if (stmmac_fpe_supported(priv)) 4278 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 4279 } 4280 4281 /** 4282 * stmmac_release - close entry point of the driver 4283 * @dev : device pointer. 4284 * Description: 4285 * This is the stop entry point of the driver. 4286 */ 4287 static int stmmac_release(struct net_device *dev) 4288 { 4289 struct stmmac_priv *priv = netdev_priv(dev); 4290 4291 /* If the PHY or MAC has WoL enabled, then the PHY will not be 4292 * suspended when phylink_stop() is called below. Set the PHY 4293 * to its slowest speed to save power. 4294 */ 4295 if (device_may_wakeup(priv->device)) 4296 phylink_speed_down(priv->phylink, false); 4297 4298 __stmmac_release(dev); 4299 4300 stmmac_legacy_serdes_power_down(priv); 4301 phylink_disconnect_phy(priv->phylink); 4302 pm_runtime_put(priv->device); 4303 4304 return 0; 4305 } 4306 4307 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4308 struct stmmac_tx_queue *tx_q) 4309 { 4310 struct dma_desc *p; 4311 u16 tag = 0x0; 4312 4313 if (!priv->dma_cap.vlins || !skb_vlan_tag_present(skb)) 4314 return false; 4315 4316 tag = skb_vlan_tag_get(skb); 4317 4318 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4319 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4320 else 4321 p = &tx_q->dma_tx[tx_q->cur_tx]; 4322 4323 if (stmmac_set_desc_vlan_tag(priv, p, tag, 0x0, 0x0)) 4324 return false; 4325 4326 stmmac_set_tx_owner(priv, p); 4327 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4328 return true; 4329 } 4330 4331 /** 4332 * stmmac_tso_allocator - close entry point of the driver 4333 * @priv: driver private structure 4334 * @entry: TX queue buffer index 4335 * @des: buffer start address 4336 * @total_len: total length to fill in descriptors 4337 * @last_segment: condition for the last descriptor 4338 * @queue: TX queue index 4339 * Description: 4340 * This function fills descriptor and request new descriptors according to 4341 * buffer length to fill 4342 */ 4343 static void stmmac_tso_allocator(struct stmmac_priv *priv, u32 *entry, 4344 dma_addr_t des, int total_len, 4345 bool last_segment, u32 queue) 4346 { 4347 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4348 struct dma_desc *desc; 4349 u32 buff_size; 4350 int tmp_len; 4351 4352 tmp_len = total_len; 4353 4354 while (tmp_len > 0) { 4355 dma_addr_t curr_addr; 4356 4357 *entry = STMMAC_NEXT_ENTRY(*entry, priv->dma_conf.dma_tx_size); 4358 WARN_ON(tx_q->tx_skbuff[*entry]); 4359 4360 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4361 desc = &tx_q->dma_entx[*entry].basic; 4362 else 4363 desc = &tx_q->dma_tx[*entry]; 4364 4365 curr_addr = des + (total_len - tmp_len); 4366 stmmac_set_desc_addr(priv, desc, curr_addr); 4367 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4368 TSO_MAX_BUFF_SIZE : tmp_len; 4369 4370 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4371 0, 1, 4372 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4373 0, 0); 4374 4375 tmp_len -= TSO_MAX_BUFF_SIZE; 4376 } 4377 } 4378 4379 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4380 { 4381 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4382 4383 /* The own bit must be the latest setting done when prepare the 4384 * descriptor and then barrier is needed to make sure that 4385 * all is coherent before granting the DMA engine. 4386 */ 4387 wmb(); 4388 4389 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx); 4390 } 4391 4392 static void stmmac_set_gso_features(struct net_device *ndev) 4393 { 4394 struct stmmac_priv *priv = netdev_priv(ndev); 4395 const struct stmmac_dma_cfg *dma_cfg; 4396 int txpbl; 4397 4398 if (priv->dma_cap.tsoen) 4399 dev_info(priv->device, "TSO supported\n"); 4400 4401 if (!(priv->plat->flags & STMMAC_FLAG_TSO_EN)) 4402 return; 4403 4404 if (!priv->dma_cap.tsoen) { 4405 dev_warn(priv->device, "platform requests unsupported TSO\n"); 4406 return; 4407 } 4408 4409 /* FIXME: 4410 * STM32MP151 (v4.2 userver v4.0) states that TxPBL must be >= 4. It 4411 * is not clear whether PBLx8 (which multiplies the PBL value by 8) 4412 * influences this. 4413 */ 4414 dma_cfg = priv->plat->dma_cfg; 4415 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl; 4416 if (txpbl < 4) { 4417 dev_warn(priv->device, "txpbl(%d) is too low for TSO\n", txpbl); 4418 return; 4419 } 4420 4421 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4422 if (priv->plat->core_type == DWMAC_CORE_GMAC4) 4423 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 4424 4425 dev_info(priv->device, "TSO feature enabled\n"); 4426 } 4427 4428 static size_t stmmac_tso_header_size(struct sk_buff *skb) 4429 { 4430 size_t size; 4431 4432 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4433 size = skb_transport_offset(skb) + sizeof(struct udphdr); 4434 else 4435 size = skb_tcp_all_headers(skb); 4436 4437 return size; 4438 } 4439 4440 /* STM32MP151 (dwmac v4.2) and STM32MP25xx (dwmac v5.3) states for TDES2 normal 4441 * (read format) descriptor that the maximum header length supported for the 4442 * TSO feature is 1023 bytes. 4443 * 4444 * While IPv4 is limited to MAC+VLAN+IPv4+ext+TCP+ext = 138 bytes, the IPv6 4445 * extension headers aren't similarly limited. 4446 * 4447 * Fall back to software GSO for these skbs. Also check that the MSS is >= 4448 * the recommended 64 bytes (documented in ETH_DMACxCR register description), 4449 * and that a the header plus MSS is not larger than 16383 (documented in 4450 * "Building the Descriptor and the packet for the TSO feature"). 4451 */ 4452 static bool stmmac_tso_valid_packet(struct sk_buff *skb) 4453 { 4454 size_t header_len = stmmac_tso_header_size(skb); 4455 unsigned int gso_size = skb_shinfo(skb)->gso_size; 4456 4457 return header_len <= 1023 && gso_size >= 64 && 4458 header_len + gso_size < 16383; 4459 } 4460 4461 static int stmmac_tso_get_num_desc(struct stmmac_tx_queue *tx_q, 4462 struct sk_buff *skb, u32 pay_len) 4463 { 4464 int i, ndesc = 1; 4465 4466 /* head payload */ 4467 ndesc += DIV_ROUND_UP(pay_len, TSO_MAX_BUFF_SIZE); 4468 /* frag payload */ 4469 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4470 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4471 4472 ndesc += DIV_ROUND_UP(skb_frag_size(frag), 4473 TSO_MAX_BUFF_SIZE); 4474 } 4475 /* MSS update requires a new descriptor */ 4476 ndesc += !!(skb_shinfo(skb)->gso_size != tx_q->mss); 4477 4478 return ndesc; 4479 } 4480 4481 /** 4482 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4483 * @skb : the socket buffer 4484 * @dev : device pointer 4485 * Description: this is the transmit function that is called on TSO frames 4486 * (support available on GMAC4 and newer chips). 4487 * Diagram below show the ring programming in case of TSO frames: 4488 * 4489 * First Descriptor 4490 * -------- 4491 * | DES0 |---> buffer1 = L2/L3/L4 header 4492 * | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address 4493 * | | width is 32-bit, but we never use it. 4494 * | | Also can be used as the most-significant 8-bits or 16-bits of 4495 * | | buffer1 address pointer if the DMA AXI address width is 40-bit 4496 * | | or 48-bit, and we always use it. 4497 * | DES2 |---> buffer1 len 4498 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4499 * -------- 4500 * -------- 4501 * | DES0 |---> buffer1 = TCP Payload (can continue on next descr...) 4502 * | DES1 |---> same as the First Descriptor 4503 * | DES2 |---> buffer1 len 4504 * | DES3 | 4505 * -------- 4506 * | 4507 * ... 4508 * | 4509 * -------- 4510 * | DES0 |---> buffer1 = Split TCP Payload 4511 * | DES1 |---> same as the First Descriptor 4512 * | DES2 |---> buffer1 len 4513 * | DES3 | 4514 * -------- 4515 * 4516 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4517 */ 4518 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4519 { 4520 unsigned int first_entry, entry, tx_packets, proto_hdr_len; 4521 struct dma_desc *desc, *first, *mss_desc = NULL; 4522 struct stmmac_priv *priv = netdev_priv(dev); 4523 struct stmmac_txq_stats *txq_stats; 4524 int i, first_tx, nfrags, ndesc; 4525 struct stmmac_tx_queue *tx_q; 4526 bool set_ic, is_last_segment; 4527 u32 pay_len, mss, queue; 4528 dma_addr_t des; 4529 u8 hdr; 4530 4531 nfrags = skb_shinfo(skb)->nr_frags; 4532 queue = skb_get_queue_mapping(skb); 4533 4534 tx_q = &priv->dma_conf.tx_queue[queue]; 4535 txq_stats = &priv->xstats.txq_stats[queue]; 4536 first_tx = tx_q->cur_tx; 4537 4538 /* Compute header lengths */ 4539 proto_hdr_len = stmmac_tso_header_size(skb); 4540 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4541 4542 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4543 hdr = sizeof(struct udphdr); 4544 else 4545 hdr = tcp_hdrlen(skb); 4546 4547 ndesc = stmmac_tso_get_num_desc(tx_q, skb, pay_len); 4548 if (unlikely(stmmac_tx_avail(priv, queue) < ndesc)) { 4549 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4550 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4551 queue)); 4552 /* This is a hard error, log it. */ 4553 netdev_err(priv->dev, 4554 "%s: Tx Ring full when queue awake\n", 4555 __func__); 4556 } 4557 return NETDEV_TX_BUSY; 4558 } 4559 4560 mss = skb_shinfo(skb)->gso_size; 4561 4562 /* set new MSS value if needed */ 4563 if (mss != tx_q->mss) { 4564 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4565 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4566 else 4567 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4568 4569 stmmac_set_mss(priv, mss_desc, mss); 4570 tx_q->mss = mss; 4571 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, 4572 priv->dma_conf.dma_tx_size); 4573 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4574 } 4575 4576 if (netif_msg_tx_queued(priv)) { 4577 pr_info("%s: hdrlen %d, hdr_len %u, pay_len %d, mss %d\n", 4578 __func__, hdr, proto_hdr_len, pay_len, mss); 4579 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4580 skb->data_len); 4581 } 4582 4583 first_entry = tx_q->cur_tx; 4584 entry = first_entry; 4585 4586 WARN_ON(tx_q->tx_skbuff[entry]); 4587 4588 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4589 desc = &tx_q->dma_entx[entry].basic; 4590 else 4591 desc = &tx_q->dma_tx[entry]; 4592 first = desc; 4593 4594 /* first descriptor: fill Headers on Buf1 */ 4595 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4596 DMA_TO_DEVICE); 4597 if (dma_mapping_error(priv->device, des)) 4598 goto error; 4599 4600 stmmac_set_desc_addr(priv, first, des); 4601 stmmac_tso_allocator(priv, &entry, des + proto_hdr_len, pay_len, 4602 (nfrags == 0), queue); 4603 4604 /* In case two or more DMA transmit descriptors are allocated for this 4605 * non-paged SKB data, the DMA buffer address should be saved to 4606 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4607 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4608 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4609 * since the tail areas of the DMA buffer can be accessed by DMA engine 4610 * sooner or later. 4611 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4612 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4613 * this DMA buffer right after the DMA engine completely finishes the 4614 * full buffer transmission. 4615 */ 4616 stmmac_set_tx_skb_dma_entry(tx_q, entry, des, skb_headlen(skb), false); 4617 4618 /* Prepare fragments */ 4619 for (i = 0; i < nfrags; i++) { 4620 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4621 4622 des = skb_frag_dma_map(priv->device, frag, 0, 4623 skb_frag_size(frag), 4624 DMA_TO_DEVICE); 4625 if (dma_mapping_error(priv->device, des)) 4626 goto error_dma_unmap; 4627 4628 stmmac_tso_allocator(priv, &entry, des, skb_frag_size(frag), 4629 (i == nfrags - 1), queue); 4630 4631 stmmac_set_tx_skb_dma_entry(tx_q, entry, des, 4632 skb_frag_size(frag), true); 4633 } 4634 tx_q->cur_tx = entry; 4635 4636 stmmac_set_tx_dma_last_segment(tx_q, tx_q->cur_tx); 4637 4638 /* Only the last descriptor gets to point to the skb. */ 4639 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4640 4641 /* Manage tx mitigation */ 4642 tx_packets = CIRC_CNT(tx_q->cur_tx + 1, first_tx, 4643 priv->dma_conf.dma_tx_size); 4644 tx_q->tx_count_frames += tx_packets; 4645 4646 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4647 set_ic = true; 4648 else if (!priv->tx_coal_frames[queue]) 4649 set_ic = false; 4650 else if (!netdev_xmit_more()) 4651 set_ic = true; 4652 else if (tx_packets > priv->tx_coal_frames[queue]) 4653 set_ic = true; 4654 else if ((tx_q->tx_count_frames % 4655 priv->tx_coal_frames[queue]) < tx_packets) 4656 set_ic = true; 4657 else 4658 set_ic = false; 4659 4660 if (set_ic) { 4661 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4662 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4663 else 4664 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4665 4666 tx_q->tx_count_frames = 0; 4667 stmmac_set_tx_ic(priv, desc); 4668 } 4669 4670 /* We've used all descriptors we need for this skb, however, 4671 * advance cur_tx so that it references a fresh descriptor. 4672 * ndo_start_xmit will fill this descriptor the next time it's 4673 * called and stmmac_tx_clean may clean up to this descriptor. 4674 */ 4675 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4676 4677 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4678 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4679 __func__); 4680 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4681 } 4682 4683 u64_stats_update_begin(&txq_stats->q_syncp); 4684 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4685 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4686 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4687 if (set_ic) 4688 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4689 u64_stats_update_end(&txq_stats->q_syncp); 4690 4691 if (priv->sarc_type) 4692 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4693 4694 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4695 priv->hwts_tx_en)) { 4696 /* declare that device is doing timestamping */ 4697 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4698 stmmac_enable_tx_timestamp(priv, first); 4699 } 4700 4701 /* If we only have one entry used, then the first entry is the last 4702 * segment. 4703 */ 4704 is_last_segment = CIRC_CNT(tx_q->cur_tx, first_entry, 4705 priv->dma_conf.dma_tx_size) == 1; 4706 4707 /* Complete the first descriptor before granting the DMA */ 4708 stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, 4709 is_last_segment, hdr / 4, 4710 skb->len - proto_hdr_len); 4711 4712 /* If context desc is used to change MSS */ 4713 if (mss_desc) { 4714 /* Make sure that first descriptor has been completely 4715 * written, including its own bit. This is because MSS is 4716 * actually before first descriptor, so we need to make 4717 * sure that MSS's own bit is the last thing written. 4718 */ 4719 dma_wmb(); 4720 stmmac_set_tx_owner(priv, mss_desc); 4721 } 4722 4723 if (netif_msg_pktdata(priv)) { 4724 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4725 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4726 tx_q->cur_tx, first, nfrags); 4727 pr_info(">>> frame to be transmitted: "); 4728 print_pkt(skb->data, skb_headlen(skb)); 4729 } 4730 4731 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4732 skb_tx_timestamp(skb); 4733 4734 stmmac_flush_tx_descriptors(priv, queue); 4735 stmmac_tx_timer_arm(priv, queue); 4736 4737 return NETDEV_TX_OK; 4738 4739 error_dma_unmap: 4740 for (;;) { 4741 desc = stmmac_get_tx_desc(priv, tx_q, first_entry); 4742 stmmac_release_tx_desc(priv, desc, priv->descriptor_mode); 4743 stmmac_free_tx_buffer(priv, &priv->dma_conf, queue, 4744 first_entry); 4745 if (first_entry == entry) 4746 break; 4747 4748 first_entry = STMMAC_NEXT_ENTRY(first_entry, 4749 priv->dma_conf.dma_tx_size); 4750 } 4751 error: 4752 dev_err(priv->device, "Tx dma map failed\n"); 4753 dev_kfree_skb(skb); 4754 priv->xstats.tx_dropped++; 4755 return NETDEV_TX_OK; 4756 } 4757 4758 /** 4759 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4760 * @skb: socket buffer to check 4761 * 4762 * Check if a packet has an ethertype that will trigger the IP header checks 4763 * and IP/TCP checksum engine of the stmmac core. 4764 * 4765 * Return: true if the ethertype can trigger the checksum engine, false 4766 * otherwise 4767 */ 4768 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4769 { 4770 int depth = 0; 4771 __be16 proto; 4772 4773 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4774 &depth); 4775 4776 return (depth <= ETH_HLEN) && 4777 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4778 } 4779 4780 /** 4781 * stmmac_xmit - Tx entry point of the driver 4782 * @skb : the socket buffer 4783 * @dev : device pointer 4784 * Description : this is the tx entry point of the driver. 4785 * It programs the chain or the ring and supports oversized frames 4786 * and SG feature. 4787 */ 4788 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4789 { 4790 bool enh_desc, has_vlan, set_ic, is_jumbo = false; 4791 struct stmmac_priv *priv = netdev_priv(dev); 4792 unsigned int nopaged_len = skb_headlen(skb); 4793 u32 queue = skb_get_queue_mapping(skb); 4794 int nfrags = skb_shinfo(skb)->nr_frags; 4795 unsigned int first_entry, tx_packets; 4796 struct stmmac_txq_stats *txq_stats; 4797 struct dma_desc *desc, *first_desc; 4798 struct stmmac_tx_queue *tx_q; 4799 int i, csum_insertion = 0; 4800 int entry, first_tx; 4801 dma_addr_t dma_addr; 4802 u32 sdu_len; 4803 4804 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4805 stmmac_stop_sw_lpi(priv); 4806 4807 if (skb_is_gso(skb)) 4808 return stmmac_tso_xmit(skb, dev); 4809 4810 if (priv->est && priv->est->enable && 4811 priv->est->max_sdu[queue]) { 4812 sdu_len = skb->len; 4813 /* Add VLAN tag length if VLAN tag insertion offload is requested */ 4814 if (priv->dma_cap.vlins && skb_vlan_tag_present(skb)) 4815 sdu_len += VLAN_HLEN; 4816 if (sdu_len > priv->est->max_sdu[queue]) { 4817 priv->xstats.max_sdu_txq_drop[queue]++; 4818 goto max_sdu_err; 4819 } 4820 } 4821 4822 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4823 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4824 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4825 queue)); 4826 /* This is a hard error, log it. */ 4827 netdev_err(priv->dev, 4828 "%s: Tx Ring full when queue awake\n", 4829 __func__); 4830 } 4831 return NETDEV_TX_BUSY; 4832 } 4833 4834 tx_q = &priv->dma_conf.tx_queue[queue]; 4835 first_tx = tx_q->cur_tx; 4836 4837 /* Check if VLAN can be inserted by HW */ 4838 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4839 4840 entry = tx_q->cur_tx; 4841 first_entry = entry; 4842 WARN_ON(tx_q->tx_skbuff[first_entry]); 4843 4844 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4845 first_desc = desc; 4846 4847 if (has_vlan) 4848 stmmac_set_desc_vlan(priv, first_desc, STMMAC_VLAN_INSERT); 4849 4850 enh_desc = priv->plat->enh_desc; 4851 /* To program the descriptors according to the size of the frame */ 4852 if (enh_desc) 4853 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4854 4855 csum_insertion = skb->ip_summed == CHECKSUM_PARTIAL; 4856 4857 if (unlikely(is_jumbo)) { 4858 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4859 if (unlikely(entry < 0) && (entry != -EINVAL)) 4860 goto dma_map_err; 4861 } else { 4862 bool last_segment = (nfrags == 0); 4863 4864 dma_addr = dma_map_single(priv->device, skb->data, 4865 nopaged_len, DMA_TO_DEVICE); 4866 if (dma_mapping_error(priv->device, dma_addr)) 4867 goto dma_map_err; 4868 4869 stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr, 4870 nopaged_len, false); 4871 4872 stmmac_set_desc_addr(priv, first_desc, dma_addr); 4873 4874 if (last_segment) 4875 stmmac_set_tx_dma_last_segment(tx_q, first_entry); 4876 4877 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4878 priv->hwts_tx_en)) { 4879 /* declare that device is doing timestamping */ 4880 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4881 stmmac_enable_tx_timestamp(priv, first_desc); 4882 } 4883 4884 /* Prepare the first descriptor without setting the OWN bit */ 4885 stmmac_prepare_tx_desc(priv, first_desc, 1, nopaged_len, 4886 csum_insertion, priv->descriptor_mode, 4887 0, last_segment, skb->len); 4888 } 4889 4890 if (priv->sarc_type) 4891 stmmac_set_desc_sarc(priv, first_desc, priv->sarc_type); 4892 4893 /* STMMAC_TBS_EN can only be set if STMMAC_TBS_AVAIL has already 4894 * been set, which means the underlying type of the descriptors 4895 * will be struct stmmac_edesc. Therefore, it is safe to convert 4896 * the basic descriptor to the enhanced descriptor here. 4897 */ 4898 if (tx_q->tbs & STMMAC_TBS_EN) { 4899 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4900 4901 stmmac_set_desc_tbs(priv, dma_desc_to_edesc(first_desc), 4902 ts.tv_sec, ts.tv_nsec); 4903 } 4904 4905 for (i = 0; i < nfrags; i++) { 4906 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4907 unsigned int frag_size = skb_frag_size(frag); 4908 bool last_segment = (i == (nfrags - 1)); 4909 4910 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4911 WARN_ON(tx_q->tx_skbuff[entry]); 4912 4913 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4914 4915 dma_addr = skb_frag_dma_map(priv->device, frag, 0, frag_size, 4916 DMA_TO_DEVICE); 4917 if (dma_mapping_error(priv->device, dma_addr)) 4918 goto dma_map_err; /* should reuse desc w/o issues */ 4919 4920 stmmac_set_tx_skb_dma_entry(tx_q, entry, dma_addr, frag_size, 4921 true); 4922 stmmac_set_desc_addr(priv, desc, dma_addr); 4923 4924 /* Prepare the descriptor and set the own bit too */ 4925 stmmac_prepare_tx_desc(priv, desc, 0, frag_size, csum_insertion, 4926 priv->descriptor_mode, 1, last_segment, 4927 skb->len); 4928 } 4929 4930 stmmac_set_tx_dma_last_segment(tx_q, entry); 4931 4932 /* Only the last descriptor gets to point to the skb. */ 4933 tx_q->tx_skbuff[entry] = skb; 4934 4935 /* According to the coalesce parameter the IC bit for the latest 4936 * segment is reset and the timer re-started to clean the tx status. 4937 * This approach takes care about the fragments: desc is the first 4938 * element in case of no SG. 4939 */ 4940 tx_packets = CIRC_CNT(entry + 1, first_tx, priv->dma_conf.dma_tx_size); 4941 tx_q->tx_count_frames += tx_packets; 4942 4943 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4944 set_ic = true; 4945 else if (!priv->tx_coal_frames[queue]) 4946 set_ic = false; 4947 else if (!netdev_xmit_more()) 4948 set_ic = true; 4949 else if (tx_packets > priv->tx_coal_frames[queue]) 4950 set_ic = true; 4951 else if ((tx_q->tx_count_frames % 4952 priv->tx_coal_frames[queue]) < tx_packets) 4953 set_ic = true; 4954 else 4955 set_ic = false; 4956 4957 if (set_ic) { 4958 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4959 tx_q->tx_count_frames = 0; 4960 stmmac_set_tx_ic(priv, desc); 4961 } 4962 4963 /* We've used all descriptors we need for this skb, however, 4964 * advance cur_tx so that it references a fresh descriptor. 4965 * ndo_start_xmit will fill this descriptor the next time it's 4966 * called and stmmac_tx_clean may clean up to this descriptor. 4967 */ 4968 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4969 tx_q->cur_tx = entry; 4970 4971 if (netif_msg_pktdata(priv)) { 4972 netdev_dbg(priv->dev, 4973 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4974 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4975 entry, first_desc, nfrags); 4976 4977 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4978 print_pkt(skb->data, skb->len); 4979 } 4980 4981 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4982 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4983 __func__); 4984 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4985 } 4986 4987 txq_stats = &priv->xstats.txq_stats[queue]; 4988 u64_stats_update_begin(&txq_stats->q_syncp); 4989 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4990 if (set_ic) 4991 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4992 u64_stats_update_end(&txq_stats->q_syncp); 4993 4994 /* Set the OWN bit on the first descriptor now that all descriptors 4995 * for this skb are populated. 4996 */ 4997 stmmac_set_tx_owner(priv, first_desc); 4998 4999 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 5000 5001 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5002 skb_tx_timestamp(skb); 5003 stmmac_flush_tx_descriptors(priv, queue); 5004 stmmac_tx_timer_arm(priv, queue); 5005 5006 return NETDEV_TX_OK; 5007 5008 dma_map_err: 5009 netdev_err(priv->dev, "Tx DMA map failed\n"); 5010 max_sdu_err: 5011 dev_kfree_skb(skb); 5012 priv->xstats.tx_dropped++; 5013 return NETDEV_TX_OK; 5014 } 5015 5016 static netdev_features_t stmmac_features_check(struct sk_buff *skb, 5017 struct net_device *dev, 5018 netdev_features_t features) 5019 { 5020 struct stmmac_priv *priv = netdev_priv(dev); 5021 u16 queue = skb_get_queue_mapping(skb); 5022 5023 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 5024 * queues. In that case, checksum offloading for those queues that don't 5025 * support tx coe needs to fallback to software checksum calculation. 5026 * 5027 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 5028 * also have to be checksummed in software. 5029 * 5030 * Note that disabling hardware checksumming also disables TSO. See 5031 * harmonize_features() in net/core/dev.c 5032 */ 5033 if (priv->plat->tx_queues_cfg[queue].coe_unsupported || 5034 !stmmac_has_ip_ethertype(skb)) 5035 features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 5036 5037 if (skb_is_gso(skb)) { 5038 if (!stmmac_tso_channel_permitted(priv, queue) || 5039 !stmmac_tso_valid_packet(skb)) 5040 features &= ~NETIF_F_GSO_MASK; 5041 5042 /* If we are going to be using hardware TSO, always insert 5043 * VLAN tag to SKB payload for TSO frames. 5044 * 5045 * Never insert VLAN tag by HW, since segments split by 5046 * TSO engine will be un-tagged by mistake. 5047 */ 5048 if (features & NETIF_F_GSO_MASK) 5049 features &= ~(NETIF_F_HW_VLAN_STAG_TX | 5050 NETIF_F_HW_VLAN_CTAG_TX); 5051 } 5052 5053 return vlan_features_check(skb, features); 5054 } 5055 5056 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 5057 { 5058 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 5059 __be16 vlan_proto = veth->h_vlan_proto; 5060 u16 vlanid; 5061 5062 if ((vlan_proto == htons(ETH_P_8021Q) && 5063 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 5064 (vlan_proto == htons(ETH_P_8021AD) && 5065 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 5066 /* pop the vlan tag */ 5067 vlanid = ntohs(veth->h_vlan_TCI); 5068 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 5069 skb_pull(skb, VLAN_HLEN); 5070 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 5071 } 5072 } 5073 5074 /** 5075 * stmmac_rx_refill - refill used skb preallocated buffers 5076 * @priv: driver private structure 5077 * @queue: RX queue index 5078 * Description : this is to reallocate the skb for the reception process 5079 * that is based on zero-copy. 5080 */ 5081 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 5082 { 5083 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5084 int dirty = stmmac_rx_dirty(priv, queue); 5085 unsigned int entry = rx_q->dirty_rx; 5086 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 5087 5088 if (priv->dma_cap.host_dma_width <= 32) 5089 gfp |= GFP_DMA32; 5090 5091 while (dirty-- > 0) { 5092 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5093 struct dma_desc *p; 5094 bool use_rx_wd; 5095 5096 p = stmmac_get_rx_desc(priv, rx_q, entry); 5097 5098 if (!buf->page) { 5099 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5100 if (!buf->page) 5101 break; 5102 } 5103 5104 if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) { 5105 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5106 if (!buf->sec_page) 5107 break; 5108 5109 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 5110 } 5111 5112 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 5113 5114 stmmac_set_desc_addr(priv, p, buf->addr); 5115 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, 5116 stmmac_rx_check_buf2_cap(priv)); 5117 stmmac_refill_desc3(priv, rx_q, p); 5118 5119 rx_q->rx_count_frames++; 5120 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5121 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5122 rx_q->rx_count_frames = 0; 5123 5124 use_rx_wd = !priv->rx_coal_frames[queue]; 5125 use_rx_wd |= rx_q->rx_count_frames > 0; 5126 if (!priv->use_riwt) 5127 use_rx_wd = false; 5128 5129 dma_wmb(); 5130 stmmac_set_rx_owner(priv, p, use_rx_wd); 5131 5132 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5133 } 5134 rx_q->dirty_rx = entry; 5135 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5136 /* Wake up Rx DMA from the suspend state if required */ 5137 stmmac_enable_dma_reception(priv, priv->ioaddr, queue); 5138 } 5139 5140 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 5141 struct dma_desc *p, 5142 int status, unsigned int len) 5143 { 5144 unsigned int plen = 0, hlen = 0; 5145 int coe = priv->hw->rx_csum; 5146 5147 /* Not first descriptor, SPH enabled: buffer1 only carries the 5148 * split header of the first descriptor, so it is zero here. 5149 */ 5150 if (priv->sph_active && len) 5151 return 0; 5152 5153 /* First descriptor, get split header length */ 5154 stmmac_get_rx_header_len(priv, p, &hlen); 5155 if (priv->sph_active && hlen) { 5156 priv->xstats.rx_split_hdr_pkt_n++; 5157 return hlen; 5158 } 5159 5160 /* Not last descriptor and not split header: buffer1 is fully filled */ 5161 if (status & rx_not_ls) 5162 return priv->dma_conf.dma_buf_sz; 5163 5164 plen = stmmac_get_rx_frame_len(priv, p, coe); 5165 5166 /* Last descriptor and not split header: buffer1 holds the remaining 5167 * bytes of the frame, up to dma_buf_sz 5168 */ 5169 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen - len); 5170 } 5171 5172 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 5173 struct dma_desc *p, 5174 int status, unsigned int len) 5175 { 5176 int coe = priv->hw->rx_csum; 5177 unsigned int plen = 0; 5178 5179 if (!stmmac_rx_check_buf2_cap(priv)) 5180 return 0; 5181 5182 /* For GMAC4, when split header is enabled, in some rare cases, the 5183 * hardware does not fill buf2 of the first descriptor with payload. 5184 * Thus we cannot assume buf2 is always fully filled if it is not 5185 * the last descriptor. Otherwise, the length of buf2 of the second 5186 * descriptor will be calculated wrong and cause an oops. 5187 * 5188 * If this is the last descriptor, 'plen' is the length of the 5189 * received packet that was transferred to system memory. 5190 * Otherwise, it is the accumulated number of bytes that have been 5191 * transferred for the current packet. 5192 * 5193 * Thus 'plen - len' always gives the correct length of buf2. 5194 */ 5195 5196 /* Not GMAC4, or non-SPH and not last descriptor */ 5197 if ((priv->plat->core_type != DWMAC_CORE_GMAC4 || !priv->sph_active) && 5198 (status & rx_not_ls)) 5199 return priv->dma_conf.dma_buf_sz; 5200 5201 /* GMAC4 or last descriptor */ 5202 plen = stmmac_get_rx_frame_len(priv, p, coe); 5203 5204 return plen > len ? plen - len : 0; 5205 } 5206 5207 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 5208 struct xdp_frame *xdpf, bool dma_map) 5209 { 5210 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 5211 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 5212 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 5213 unsigned int entry = tx_q->cur_tx; 5214 enum stmmac_txbuf_type buf_type; 5215 struct dma_desc *tx_desc; 5216 dma_addr_t dma_addr; 5217 bool set_ic; 5218 5219 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 5220 return STMMAC_XDP_CONSUMED; 5221 5222 if (priv->est && priv->est->enable && 5223 priv->est->max_sdu[queue] && 5224 xdpf->len > priv->est->max_sdu[queue]) { 5225 priv->xstats.max_sdu_txq_drop[queue]++; 5226 return STMMAC_XDP_CONSUMED; 5227 } 5228 5229 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 5230 if (dma_map) { 5231 dma_addr = dma_map_single(priv->device, xdpf->data, 5232 xdpf->len, DMA_TO_DEVICE); 5233 if (dma_mapping_error(priv->device, dma_addr)) 5234 return STMMAC_XDP_CONSUMED; 5235 5236 buf_type = STMMAC_TXBUF_T_XDP_NDO; 5237 } else { 5238 struct page *page = virt_to_page(xdpf->data); 5239 5240 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 5241 xdpf->headroom; 5242 dma_sync_single_for_device(priv->device, dma_addr, 5243 xdpf->len, DMA_BIDIRECTIONAL); 5244 5245 buf_type = STMMAC_TXBUF_T_XDP_TX; 5246 } 5247 5248 stmmac_set_tx_dma_entry(tx_q, entry, buf_type, dma_addr, xdpf->len, 5249 false); 5250 stmmac_set_tx_dma_last_segment(tx_q, entry); 5251 5252 tx_q->xdpf[entry] = xdpf; 5253 5254 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 5255 5256 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 5257 csum, priv->descriptor_mode, true, true, 5258 xdpf->len); 5259 5260 tx_q->tx_count_frames++; 5261 5262 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 5263 set_ic = true; 5264 else 5265 set_ic = false; 5266 5267 if (set_ic) { 5268 tx_q->tx_count_frames = 0; 5269 stmmac_set_tx_ic(priv, tx_desc); 5270 u64_stats_update_begin(&txq_stats->q_syncp); 5271 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 5272 u64_stats_update_end(&txq_stats->q_syncp); 5273 } 5274 5275 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5276 5277 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 5278 tx_q->cur_tx = entry; 5279 5280 return STMMAC_XDP_TX; 5281 } 5282 5283 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 5284 int cpu) 5285 { 5286 int index = cpu; 5287 5288 if (unlikely(index < 0)) 5289 index = 0; 5290 5291 while (index >= priv->plat->tx_queues_to_use) 5292 index -= priv->plat->tx_queues_to_use; 5293 5294 return index; 5295 } 5296 5297 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 5298 struct xdp_buff *xdp) 5299 { 5300 bool zc = !!(xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL); 5301 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 5302 int cpu = smp_processor_id(); 5303 struct netdev_queue *nq; 5304 int queue; 5305 int res; 5306 5307 if (unlikely(!xdpf)) 5308 return STMMAC_XDP_CONSUMED; 5309 5310 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5311 nq = netdev_get_tx_queue(priv->dev, queue); 5312 5313 __netif_tx_lock(nq, cpu); 5314 /* Avoids TX time-out as we are sharing with slow path */ 5315 txq_trans_cond_update(nq); 5316 5317 /* For zero copy XDP_TX action, dma_map is true */ 5318 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, zc); 5319 if (res == STMMAC_XDP_TX) { 5320 stmmac_flush_tx_descriptors(priv, queue); 5321 } else if (res == STMMAC_XDP_CONSUMED && zc) { 5322 /* xdp has been freed by xdp_convert_buff_to_frame(), 5323 * no need to call xsk_buff_free() again, so return 5324 * STMMAC_XSK_CONSUMED. 5325 */ 5326 res = STMMAC_XSK_CONSUMED; 5327 xdp_return_frame(xdpf); 5328 } 5329 5330 __netif_tx_unlock(nq); 5331 5332 return res; 5333 } 5334 5335 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 5336 struct bpf_prog *prog, 5337 struct xdp_buff *xdp) 5338 { 5339 u32 act; 5340 int res; 5341 5342 act = bpf_prog_run_xdp(prog, xdp); 5343 switch (act) { 5344 case XDP_PASS: 5345 res = STMMAC_XDP_PASS; 5346 break; 5347 case XDP_TX: 5348 res = stmmac_xdp_xmit_back(priv, xdp); 5349 break; 5350 case XDP_REDIRECT: 5351 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 5352 res = STMMAC_XDP_CONSUMED; 5353 else 5354 res = STMMAC_XDP_REDIRECT; 5355 break; 5356 default: 5357 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5358 fallthrough; 5359 case XDP_ABORTED: 5360 trace_xdp_exception(priv->dev, prog, act); 5361 fallthrough; 5362 case XDP_DROP: 5363 res = STMMAC_XDP_CONSUMED; 5364 break; 5365 } 5366 5367 return res; 5368 } 5369 5370 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5371 struct xdp_buff *xdp) 5372 { 5373 struct bpf_prog *prog; 5374 int res; 5375 5376 prog = READ_ONCE(priv->xdp_prog); 5377 if (!prog) { 5378 res = STMMAC_XDP_PASS; 5379 goto out; 5380 } 5381 5382 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5383 out: 5384 return ERR_PTR(-res); 5385 } 5386 5387 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5388 int xdp_status) 5389 { 5390 int cpu = smp_processor_id(); 5391 int queue; 5392 5393 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5394 5395 if (xdp_status & STMMAC_XDP_TX) 5396 stmmac_tx_timer_arm(priv, queue); 5397 5398 if (xdp_status & STMMAC_XDP_REDIRECT) 5399 xdp_do_flush(); 5400 } 5401 5402 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5403 struct xdp_buff *xdp) 5404 { 5405 unsigned int metasize = xdp->data - xdp->data_meta; 5406 unsigned int datasize = xdp->data_end - xdp->data; 5407 struct sk_buff *skb; 5408 5409 skb = napi_alloc_skb(&ch->rxtx_napi, 5410 xdp->data_end - xdp->data_hard_start); 5411 if (unlikely(!skb)) 5412 return NULL; 5413 5414 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5415 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5416 if (metasize) 5417 skb_metadata_set(skb, metasize); 5418 5419 return skb; 5420 } 5421 5422 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5423 struct dma_desc *p, struct dma_desc *np, 5424 struct xdp_buff *xdp) 5425 { 5426 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5427 struct stmmac_channel *ch = &priv->channel[queue]; 5428 unsigned int len = xdp->data_end - xdp->data; 5429 enum pkt_hash_types hash_type; 5430 int coe = priv->hw->rx_csum; 5431 struct sk_buff *skb; 5432 u32 hash; 5433 5434 skb = stmmac_construct_skb_zc(ch, xdp); 5435 if (!skb) { 5436 priv->xstats.rx_dropped++; 5437 return; 5438 } 5439 5440 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5441 if (priv->hw->hw_vlan_en) 5442 /* MAC level stripping. */ 5443 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5444 else 5445 /* Driver level stripping. */ 5446 stmmac_rx_vlan(priv->dev, skb); 5447 skb->protocol = eth_type_trans(skb, priv->dev); 5448 5449 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5450 skb_checksum_none_assert(skb); 5451 else 5452 skb->ip_summed = CHECKSUM_UNNECESSARY; 5453 5454 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5455 skb_set_hash(skb, hash, hash_type); 5456 5457 skb_record_rx_queue(skb, queue); 5458 napi_gro_receive(&ch->rxtx_napi, skb); 5459 5460 u64_stats_update_begin(&rxq_stats->napi_syncp); 5461 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5462 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5463 u64_stats_update_end(&rxq_stats->napi_syncp); 5464 } 5465 5466 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5467 { 5468 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5469 unsigned int entry = rx_q->dirty_rx; 5470 struct dma_desc *rx_desc = NULL; 5471 bool ret = true; 5472 5473 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5474 5475 while (budget-- > 0 && entry != rx_q->cur_rx) { 5476 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5477 dma_addr_t dma_addr; 5478 bool use_rx_wd; 5479 5480 if (!buf->xdp) { 5481 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5482 if (!buf->xdp) { 5483 ret = false; 5484 break; 5485 } 5486 } 5487 5488 rx_desc = stmmac_get_rx_desc(priv, rx_q, entry); 5489 5490 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5491 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5492 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5493 stmmac_refill_desc3(priv, rx_q, rx_desc); 5494 5495 rx_q->rx_count_frames++; 5496 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5497 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5498 rx_q->rx_count_frames = 0; 5499 5500 use_rx_wd = !priv->rx_coal_frames[queue]; 5501 use_rx_wd |= rx_q->rx_count_frames > 0; 5502 if (!priv->use_riwt) 5503 use_rx_wd = false; 5504 5505 dma_wmb(); 5506 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5507 5508 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5509 } 5510 5511 if (rx_desc) { 5512 rx_q->dirty_rx = entry; 5513 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5514 } 5515 5516 return ret; 5517 } 5518 5519 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5520 { 5521 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5522 * to represent incoming packet, whereas cb field in the same structure 5523 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5524 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5525 */ 5526 return (struct stmmac_xdp_buff *)xdp; 5527 } 5528 5529 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5530 { 5531 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5532 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5533 unsigned int count = 0, error = 0, len = 0; 5534 int dirty = stmmac_rx_dirty(priv, queue); 5535 unsigned int next_entry = rx_q->cur_rx; 5536 u32 rx_errors = 0, rx_dropped = 0; 5537 unsigned int desc_size; 5538 struct bpf_prog *prog; 5539 bool failure = false; 5540 int xdp_status = 0; 5541 int status = 0; 5542 5543 if (netif_msg_rx_status(priv)) { 5544 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5545 5546 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5547 desc_size = stmmac_get_rx_desc_size(priv); 5548 5549 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5550 rx_q->dma_rx_phy, desc_size); 5551 } 5552 while (count < limit) { 5553 struct stmmac_rx_buffer *buf; 5554 struct stmmac_xdp_buff *ctx; 5555 unsigned int buf1_len = 0; 5556 struct dma_desc *np, *p; 5557 int entry; 5558 int res; 5559 5560 if (!count && rx_q->state_saved) { 5561 error = rx_q->state.error; 5562 len = rx_q->state.len; 5563 } else { 5564 rx_q->state_saved = false; 5565 error = 0; 5566 len = 0; 5567 } 5568 5569 read_again: 5570 if (count >= limit) 5571 break; 5572 5573 buf1_len = 0; 5574 entry = next_entry; 5575 buf = &rx_q->buf_pool[entry]; 5576 5577 if (dirty >= STMMAC_RX_FILL_BATCH) { 5578 failure = failure || 5579 !stmmac_rx_refill_zc(priv, queue, dirty); 5580 dirty = 0; 5581 } 5582 5583 p = stmmac_get_rx_desc(priv, rx_q, entry); 5584 5585 /* read the status of the incoming frame */ 5586 status = stmmac_rx_status(priv, &priv->xstats, p); 5587 /* check if managed by the DMA otherwise go ahead */ 5588 if (unlikely(status & dma_own)) 5589 break; 5590 5591 /* Prefetch the next RX descriptor */ 5592 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5593 priv->dma_conf.dma_rx_size); 5594 if (unlikely(next_entry == rx_q->dirty_rx)) 5595 break; 5596 5597 rx_q->cur_rx = next_entry; 5598 5599 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5600 5601 prefetch(np); 5602 5603 /* Ensure a valid XSK buffer before proceed */ 5604 if (!buf->xdp) 5605 break; 5606 5607 if (priv->extend_desc) 5608 stmmac_rx_extended_status(priv, &priv->xstats, 5609 rx_q->dma_erx + entry); 5610 if (unlikely(status == discard_frame)) { 5611 xsk_buff_free(buf->xdp); 5612 buf->xdp = NULL; 5613 dirty++; 5614 error = 1; 5615 if (!priv->hwts_rx_en) 5616 rx_errors++; 5617 } 5618 5619 if (unlikely(error && (status & rx_not_ls))) 5620 goto read_again; 5621 if (unlikely(error)) { 5622 count++; 5623 continue; 5624 } 5625 5626 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5627 if (likely(status & rx_not_ls)) { 5628 xsk_buff_free(buf->xdp); 5629 buf->xdp = NULL; 5630 dirty++; 5631 count++; 5632 goto read_again; 5633 } 5634 5635 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5636 ctx->priv = priv; 5637 ctx->desc = p; 5638 ctx->ndesc = np; 5639 5640 /* XDP ZC Frame only support primary buffers for now */ 5641 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5642 len += buf1_len; 5643 5644 /* ACS is disabled; strip manually. */ 5645 if (likely(!(status & rx_not_ls))) { 5646 buf1_len -= ETH_FCS_LEN; 5647 len -= ETH_FCS_LEN; 5648 } 5649 5650 /* RX buffer is good and fit into a XSK pool buffer */ 5651 buf->xdp->data_end = buf->xdp->data + buf1_len; 5652 xsk_buff_dma_sync_for_cpu(buf->xdp); 5653 5654 prog = READ_ONCE(priv->xdp_prog); 5655 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5656 5657 switch (res) { 5658 case STMMAC_XDP_PASS: 5659 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5660 xsk_buff_free(buf->xdp); 5661 break; 5662 case STMMAC_XDP_CONSUMED: 5663 xsk_buff_free(buf->xdp); 5664 fallthrough; 5665 case STMMAC_XSK_CONSUMED: 5666 rx_dropped++; 5667 break; 5668 case STMMAC_XDP_TX: 5669 case STMMAC_XDP_REDIRECT: 5670 xdp_status |= res; 5671 break; 5672 } 5673 5674 buf->xdp = NULL; 5675 dirty++; 5676 count++; 5677 } 5678 5679 if (status & rx_not_ls) { 5680 rx_q->state_saved = true; 5681 rx_q->state.error = error; 5682 rx_q->state.len = len; 5683 } 5684 5685 stmmac_finalize_xdp_rx(priv, xdp_status); 5686 5687 u64_stats_update_begin(&rxq_stats->napi_syncp); 5688 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5689 u64_stats_update_end(&rxq_stats->napi_syncp); 5690 5691 priv->xstats.rx_dropped += rx_dropped; 5692 priv->xstats.rx_errors += rx_errors; 5693 5694 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5695 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5696 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5697 else 5698 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5699 5700 return (int)count; 5701 } 5702 5703 return failure ? limit : (int)count; 5704 } 5705 5706 /** 5707 * stmmac_rx - manage the receive process 5708 * @priv: driver private structure 5709 * @limit: napi bugget 5710 * @queue: RX queue index. 5711 * Description : this the function called by the napi poll method. 5712 * It gets all the frames inside the ring. 5713 */ 5714 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5715 { 5716 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5717 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5718 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5719 struct stmmac_channel *ch = &priv->channel[queue]; 5720 unsigned int count = 0, error = 0, len = 0; 5721 int status = 0, coe = priv->hw->rx_csum; 5722 unsigned int next_entry = rx_q->cur_rx; 5723 enum dma_data_direction dma_dir; 5724 unsigned int desc_size; 5725 struct sk_buff *skb = NULL; 5726 struct stmmac_xdp_buff ctx; 5727 bool fcs_stripped = false; 5728 int xdp_status = 0; 5729 int bufsz; 5730 5731 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5732 bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5733 5734 if (netif_msg_rx_status(priv)) { 5735 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5736 5737 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5738 desc_size = stmmac_get_rx_desc_size(priv); 5739 5740 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5741 rx_q->dma_rx_phy, desc_size); 5742 } 5743 while (count < limit) { 5744 unsigned int buf1_len = 0, buf2_len = 0; 5745 enum pkt_hash_types hash_type; 5746 struct stmmac_rx_buffer *buf; 5747 struct dma_desc *np, *p; 5748 int entry; 5749 u32 hash; 5750 5751 if (!count && rx_q->state_saved) { 5752 skb = rx_q->state.skb; 5753 error = rx_q->state.error; 5754 len = rx_q->state.len; 5755 } else { 5756 rx_q->state_saved = false; 5757 skb = NULL; 5758 error = 0; 5759 len = 0; 5760 } 5761 5762 read_again: 5763 if (count >= limit) 5764 break; 5765 5766 buf1_len = 0; 5767 buf2_len = 0; 5768 entry = next_entry; 5769 buf = &rx_q->buf_pool[entry]; 5770 5771 p = stmmac_get_rx_desc(priv, rx_q, entry); 5772 5773 /* read the status of the incoming frame */ 5774 status = stmmac_rx_status(priv, &priv->xstats, p); 5775 /* check if managed by the DMA otherwise go ahead */ 5776 if (unlikely(status & dma_own)) 5777 break; 5778 5779 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5780 priv->dma_conf.dma_rx_size); 5781 if (unlikely(next_entry == rx_q->dirty_rx)) 5782 break; 5783 5784 rx_q->cur_rx = next_entry; 5785 5786 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5787 5788 prefetch(np); 5789 5790 if (priv->extend_desc) 5791 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5792 if (unlikely(status == discard_frame)) { 5793 page_pool_put_page(rx_q->page_pool, buf->page, 0, true); 5794 buf->page = NULL; 5795 error = 1; 5796 if (!priv->hwts_rx_en) 5797 rx_errors++; 5798 } 5799 5800 if (unlikely(error && (status & rx_not_ls))) 5801 goto read_again; 5802 if (unlikely(error)) { 5803 dev_kfree_skb(skb); 5804 skb = NULL; 5805 count++; 5806 continue; 5807 } 5808 5809 /* Buffer is good. Go on. */ 5810 5811 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5812 len += buf1_len; 5813 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5814 len += buf2_len; 5815 5816 /* ACS is disabled; strip manually. */ 5817 if (likely(!(status & rx_not_ls))) 5818 len -= ETH_FCS_LEN; 5819 5820 if (!skb) { 5821 unsigned int pre_len, sync_len; 5822 5823 /* Each frame starts here: reset the FCS handling */ 5824 fcs_stripped = false; 5825 5826 dma_sync_single_for_cpu(priv->device, buf->addr, 5827 buf1_len, dma_dir); 5828 net_prefetch(page_address(buf->page) + 5829 buf->page_offset); 5830 5831 if (stmmac_xdp_is_enabled(priv) && !buf2_len) { 5832 buf1_len -= ETH_FCS_LEN; 5833 fcs_stripped = true; 5834 } 5835 5836 xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq); 5837 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5838 buf->page_offset, buf1_len, true); 5839 5840 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5841 buf->page_offset; 5842 5843 ctx.priv = priv; 5844 ctx.desc = p; 5845 ctx.ndesc = np; 5846 5847 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5848 /* Due xdp_adjust_tail: DMA sync for_device 5849 * cover max len CPU touch 5850 */ 5851 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5852 buf->page_offset; 5853 sync_len = max(sync_len, pre_len); 5854 5855 /* For Not XDP_PASS verdict */ 5856 if (IS_ERR(skb)) { 5857 unsigned int xdp_res = -PTR_ERR(skb); 5858 5859 if (xdp_res & STMMAC_XDP_CONSUMED) { 5860 page_pool_put_page(rx_q->page_pool, 5861 virt_to_head_page(ctx.xdp.data), 5862 sync_len, true); 5863 buf->page = NULL; 5864 rx_dropped++; 5865 5866 /* Clear skb as it was set as 5867 * status by XDP program. 5868 */ 5869 skb = NULL; 5870 5871 if (unlikely((status & rx_not_ls))) 5872 goto read_again; 5873 5874 count++; 5875 continue; 5876 } else if (xdp_res & (STMMAC_XDP_TX | 5877 STMMAC_XDP_REDIRECT)) { 5878 xdp_status |= xdp_res; 5879 buf->page = NULL; 5880 skb = NULL; 5881 count++; 5882 continue; 5883 } 5884 } 5885 } 5886 5887 if (!skb) { 5888 unsigned int head_pad_len; 5889 5890 /* XDP program may expand or reduce tail */ 5891 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5892 5893 skb = napi_build_skb(page_address(buf->page), 5894 rx_q->napi_skb_frag_size); 5895 if (!skb) { 5896 page_pool_recycle_direct(rx_q->page_pool, 5897 buf->page); 5898 buf->page = NULL; 5899 rx_dropped++; 5900 count++; 5901 goto drain_data; 5902 } 5903 5904 /* XDP program may adjust header */ 5905 head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; 5906 skb_reserve(skb, head_pad_len); 5907 skb_put(skb, buf1_len); 5908 skb_mark_for_recycle(skb); 5909 buf->page = NULL; 5910 } else if (buf1_len) { 5911 dma_sync_single_for_cpu(priv->device, buf->addr, 5912 buf1_len, dma_dir); 5913 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5914 buf->page, buf->page_offset, buf1_len, 5915 priv->dma_conf.dma_buf_sz); 5916 buf->page = NULL; 5917 } 5918 5919 if (buf2_len) { 5920 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5921 buf2_len, dma_dir); 5922 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5923 buf->sec_page, 0, buf2_len, 5924 priv->dma_conf.dma_buf_sz); 5925 buf->sec_page = NULL; 5926 } 5927 5928 drain_data: 5929 if (likely(status & rx_not_ls)) 5930 goto read_again; 5931 if (!skb) 5932 continue; 5933 5934 /* Got entire packet into SKB. Finish it. */ 5935 5936 /* Remove FCS if needed */ 5937 if (!fcs_stripped && pskb_trim(skb, len)) { 5938 dev_kfree_skb_any(skb); 5939 skb = NULL; 5940 rx_dropped++; 5941 count++; 5942 continue; 5943 } 5944 5945 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5946 5947 if (priv->hw->hw_vlan_en) 5948 /* MAC level stripping. */ 5949 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5950 else 5951 /* Driver level stripping. */ 5952 stmmac_rx_vlan(priv->dev, skb); 5953 5954 skb->protocol = eth_type_trans(skb, priv->dev); 5955 5956 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) || 5957 (status & csum_none)) 5958 skb_checksum_none_assert(skb); 5959 else 5960 skb->ip_summed = CHECKSUM_UNNECESSARY; 5961 5962 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5963 skb_set_hash(skb, hash, hash_type); 5964 5965 skb_record_rx_queue(skb, queue); 5966 napi_gro_receive(&ch->rx_napi, skb); 5967 skb = NULL; 5968 5969 rx_packets++; 5970 rx_bytes += len; 5971 count++; 5972 } 5973 5974 if (status & rx_not_ls || skb) { 5975 rx_q->state_saved = true; 5976 rx_q->state.skb = skb; 5977 rx_q->state.error = error; 5978 rx_q->state.len = len; 5979 } 5980 5981 stmmac_finalize_xdp_rx(priv, xdp_status); 5982 5983 stmmac_rx_refill(priv, queue); 5984 5985 u64_stats_update_begin(&rxq_stats->napi_syncp); 5986 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5987 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5988 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5989 u64_stats_update_end(&rxq_stats->napi_syncp); 5990 5991 priv->xstats.rx_dropped += rx_dropped; 5992 priv->xstats.rx_errors += rx_errors; 5993 5994 return count; 5995 } 5996 5997 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5998 { 5999 struct stmmac_channel *ch = 6000 container_of(napi, struct stmmac_channel, rx_napi); 6001 struct stmmac_priv *priv = ch->priv_data; 6002 struct stmmac_rxq_stats *rxq_stats; 6003 u32 chan = ch->index; 6004 int work_done; 6005 6006 rxq_stats = &priv->xstats.rxq_stats[chan]; 6007 u64_stats_update_begin(&rxq_stats->napi_syncp); 6008 u64_stats_inc(&rxq_stats->napi.poll); 6009 u64_stats_update_end(&rxq_stats->napi_syncp); 6010 6011 work_done = stmmac_rx(priv, budget, chan); 6012 if (work_done < budget && napi_complete_done(napi, work_done)) { 6013 unsigned long flags; 6014 6015 spin_lock_irqsave(&ch->lock, flags); 6016 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 6017 spin_unlock_irqrestore(&ch->lock, flags); 6018 } 6019 6020 return work_done; 6021 } 6022 6023 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 6024 { 6025 struct stmmac_channel *ch = 6026 container_of(napi, struct stmmac_channel, tx_napi); 6027 struct stmmac_priv *priv = ch->priv_data; 6028 struct stmmac_txq_stats *txq_stats; 6029 bool pending_packets = false; 6030 u32 chan = ch->index; 6031 int work_done; 6032 6033 txq_stats = &priv->xstats.txq_stats[chan]; 6034 u64_stats_update_begin(&txq_stats->napi_syncp); 6035 u64_stats_inc(&txq_stats->napi.poll); 6036 u64_stats_update_end(&txq_stats->napi_syncp); 6037 6038 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 6039 work_done = min(work_done, budget); 6040 6041 if (work_done < budget && napi_complete_done(napi, work_done)) { 6042 unsigned long flags; 6043 6044 spin_lock_irqsave(&ch->lock, flags); 6045 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 6046 spin_unlock_irqrestore(&ch->lock, flags); 6047 } 6048 6049 /* TX still have packet to handle, check if we need to arm tx timer */ 6050 if (pending_packets) 6051 stmmac_tx_timer_arm(priv, chan); 6052 6053 return work_done; 6054 } 6055 6056 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 6057 { 6058 struct stmmac_channel *ch = 6059 container_of(napi, struct stmmac_channel, rxtx_napi); 6060 struct stmmac_priv *priv = ch->priv_data; 6061 bool tx_pending_packets = false; 6062 int rx_done, tx_done, rxtx_done; 6063 struct stmmac_rxq_stats *rxq_stats; 6064 struct stmmac_txq_stats *txq_stats; 6065 u32 chan = ch->index; 6066 6067 rxq_stats = &priv->xstats.rxq_stats[chan]; 6068 u64_stats_update_begin(&rxq_stats->napi_syncp); 6069 u64_stats_inc(&rxq_stats->napi.poll); 6070 u64_stats_update_end(&rxq_stats->napi_syncp); 6071 6072 txq_stats = &priv->xstats.txq_stats[chan]; 6073 u64_stats_update_begin(&txq_stats->napi_syncp); 6074 u64_stats_inc(&txq_stats->napi.poll); 6075 u64_stats_update_end(&txq_stats->napi_syncp); 6076 6077 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 6078 tx_done = min(tx_done, budget); 6079 6080 rx_done = stmmac_rx_zc(priv, budget, chan); 6081 6082 rxtx_done = max(tx_done, rx_done); 6083 6084 /* If either TX or RX work is not complete, return budget 6085 * and keep pooling 6086 */ 6087 if (rxtx_done >= budget) 6088 return budget; 6089 6090 /* all work done, exit the polling mode */ 6091 if (napi_complete_done(napi, rxtx_done)) { 6092 unsigned long flags; 6093 6094 spin_lock_irqsave(&ch->lock, flags); 6095 /* Both RX and TX work done are complete, 6096 * so enable both RX & TX IRQs. 6097 */ 6098 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6099 spin_unlock_irqrestore(&ch->lock, flags); 6100 } 6101 6102 /* TX still have packet to handle, check if we need to arm tx timer */ 6103 if (tx_pending_packets) 6104 stmmac_tx_timer_arm(priv, chan); 6105 6106 return min(rxtx_done, budget - 1); 6107 } 6108 6109 /** 6110 * stmmac_tx_timeout 6111 * @dev : Pointer to net device structure 6112 * @txqueue: the index of the hanging transmit queue 6113 * Description: this function is called when a packet transmission fails to 6114 * complete within a reasonable time. The driver will mark the error in the 6115 * netdev structure and arrange for the device to be reset to a sane state 6116 * in order to transmit a new packet. 6117 */ 6118 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 6119 { 6120 struct stmmac_priv *priv = netdev_priv(dev); 6121 6122 stmmac_global_err(priv); 6123 } 6124 6125 /** 6126 * stmmac_set_rx_mode - entry point for multicast addressing 6127 * @dev : pointer to the device structure 6128 * Description: 6129 * This function is a driver entry point which gets called by the kernel 6130 * whenever multicast addresses must be enabled/disabled. 6131 * Return value: 6132 * void. 6133 * 6134 * FIXME: This may need RXC to be running, but it may be called with BH 6135 * disabled, which means we can't call phylink_rx_clk_stop*(). 6136 */ 6137 static void stmmac_set_rx_mode(struct net_device *dev) 6138 { 6139 struct stmmac_priv *priv = netdev_priv(dev); 6140 6141 stmmac_set_filter(priv, priv->hw, dev); 6142 } 6143 6144 /** 6145 * stmmac_change_mtu - entry point to change MTU size for the device. 6146 * @dev : device pointer. 6147 * @new_mtu : the new MTU size for the device. 6148 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 6149 * to drive packet transmission. Ethernet has an MTU of 1500 octets 6150 * (ETH_DATA_LEN). This value can be changed with ifconfig. 6151 * Return value: 6152 * 0 on success and an appropriate (-)ve integer as defined in errno.h 6153 * file on failure. 6154 */ 6155 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 6156 { 6157 struct stmmac_priv *priv = netdev_priv(dev); 6158 int txfifosz = priv->plat->tx_fifo_size; 6159 struct stmmac_dma_conf *dma_conf; 6160 const int mtu = new_mtu; 6161 int ret; 6162 6163 if (txfifosz == 0) 6164 txfifosz = priv->dma_cap.tx_fifo_size; 6165 6166 txfifosz /= priv->plat->tx_queues_to_use; 6167 6168 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 6169 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 6170 return -EINVAL; 6171 } 6172 6173 new_mtu = STMMAC_ALIGN(new_mtu); 6174 6175 /* If condition true, FIFO is too small or MTU too large */ 6176 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 6177 return -EINVAL; 6178 6179 if (netif_running(dev)) { 6180 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 6181 /* Try to allocate the new DMA conf with the new mtu */ 6182 dma_conf = stmmac_setup_dma_desc(priv, mtu); 6183 if (IS_ERR(dma_conf)) { 6184 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 6185 mtu); 6186 return PTR_ERR(dma_conf); 6187 } 6188 6189 __stmmac_release(dev); 6190 6191 ret = __stmmac_open(dev, dma_conf); 6192 if (ret) { 6193 free_dma_desc_resources(priv, dma_conf); 6194 kfree(dma_conf); 6195 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 6196 return ret; 6197 } 6198 6199 kfree(dma_conf); 6200 6201 stmmac_set_rx_mode(dev); 6202 } 6203 6204 WRITE_ONCE(dev->mtu, mtu); 6205 netdev_update_features(dev); 6206 6207 return 0; 6208 } 6209 6210 static netdev_features_t stmmac_fix_features(struct net_device *dev, 6211 netdev_features_t features) 6212 { 6213 struct stmmac_priv *priv = netdev_priv(dev); 6214 6215 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 6216 features &= ~NETIF_F_RXCSUM; 6217 6218 if (!priv->plat->tx_coe) 6219 features &= ~NETIF_F_CSUM_MASK; 6220 6221 /* Some GMAC devices have a bugged Jumbo frame support that 6222 * needs to have the Tx COE disabled for oversized frames 6223 * (due to limited buffer sizes). In this case we disable 6224 * the TX csum insertion in the TDES and not use SF. 6225 */ 6226 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 6227 features &= ~NETIF_F_CSUM_MASK; 6228 6229 return features; 6230 } 6231 6232 static int stmmac_set_features(struct net_device *netdev, 6233 netdev_features_t features) 6234 { 6235 struct stmmac_priv *priv = netdev_priv(netdev); 6236 6237 /* Keep the COE Type in case of csum is supporting */ 6238 if (features & NETIF_F_RXCSUM) 6239 priv->hw->rx_csum = priv->plat->rx_coe; 6240 else 6241 priv->hw->rx_csum = 0; 6242 /* No check needed because rx_coe has been set before and it will be 6243 * fixed in case of issue. 6244 */ 6245 stmmac_rx_ipc(priv, priv->hw); 6246 6247 if (priv->sph_capable) { 6248 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 6249 u8 chan; 6250 6251 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 6252 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6253 } 6254 6255 if (features & NETIF_F_HW_VLAN_CTAG_RX) 6256 priv->hw->hw_vlan_en = true; 6257 else 6258 priv->hw->hw_vlan_en = false; 6259 6260 phylink_rx_clk_stop_block(priv->phylink); 6261 stmmac_set_hw_vlan_mode(priv, priv->hw); 6262 phylink_rx_clk_stop_unblock(priv->phylink); 6263 6264 return 0; 6265 } 6266 6267 static void stmmac_common_interrupt(struct stmmac_priv *priv) 6268 { 6269 u8 rx_cnt = priv->plat->rx_queues_to_use; 6270 u8 tx_cnt = priv->plat->tx_queues_to_use; 6271 u8 queues_count; 6272 bool xmac; 6273 u8 queue; 6274 6275 xmac = dwmac_is_xmac(priv->plat->core_type); 6276 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 6277 6278 if (priv->irq_wake) 6279 pm_wakeup_event(priv->device, 0); 6280 6281 if (priv->dma_cap.estsel) 6282 stmmac_est_irq_status(priv, priv, priv->dev, 6283 &priv->xstats, tx_cnt); 6284 6285 if (stmmac_fpe_supported(priv)) 6286 stmmac_fpe_irq_status(priv); 6287 6288 /* To handle GMAC own interrupts */ 6289 if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) { 6290 int status = stmmac_host_irq_status(priv, &priv->xstats); 6291 6292 if (unlikely(status)) { 6293 /* For LPI we need to save the tx status */ 6294 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 6295 priv->tx_path_in_lpi_mode = true; 6296 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 6297 priv->tx_path_in_lpi_mode = false; 6298 } 6299 6300 for (queue = 0; queue < queues_count; queue++) 6301 stmmac_host_mtl_irq_status(priv, priv->hw, queue); 6302 6303 stmmac_timestamp_interrupt(priv, priv); 6304 } 6305 } 6306 6307 /** 6308 * stmmac_interrupt - main ISR 6309 * @irq: interrupt number. 6310 * @dev_id: to pass the net device pointer. 6311 * Description: this is the main driver interrupt service routine. 6312 * It can call: 6313 * o DMA service routine (to manage incoming frame reception and transmission 6314 * status) 6315 * o Core interrupts to manage: remote wake-up, management counter, LPI 6316 * interrupts. 6317 */ 6318 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6319 { 6320 struct net_device *dev = (struct net_device *)dev_id; 6321 struct stmmac_priv *priv = netdev_priv(dev); 6322 6323 /* Check if adapter is up */ 6324 if (test_bit(STMMAC_DOWN, &priv->state)) 6325 return IRQ_HANDLED; 6326 6327 /* Check ASP error if it isn't delivered via an individual IRQ */ 6328 if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) 6329 return IRQ_HANDLED; 6330 6331 /* To handle Common interrupts */ 6332 stmmac_common_interrupt(priv); 6333 6334 /* To handle DMA interrupts */ 6335 stmmac_dma_interrupt(priv); 6336 6337 return IRQ_HANDLED; 6338 } 6339 6340 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6341 { 6342 struct net_device *dev = (struct net_device *)dev_id; 6343 struct stmmac_priv *priv = netdev_priv(dev); 6344 6345 /* Check if adapter is up */ 6346 if (test_bit(STMMAC_DOWN, &priv->state)) 6347 return IRQ_HANDLED; 6348 6349 /* To handle Common interrupts */ 6350 stmmac_common_interrupt(priv); 6351 6352 return IRQ_HANDLED; 6353 } 6354 6355 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6356 { 6357 struct net_device *dev = (struct net_device *)dev_id; 6358 struct stmmac_priv *priv = netdev_priv(dev); 6359 6360 /* Check if adapter is up */ 6361 if (test_bit(STMMAC_DOWN, &priv->state)) 6362 return IRQ_HANDLED; 6363 6364 /* Check if a fatal error happened */ 6365 stmmac_safety_feat_interrupt(priv); 6366 6367 return IRQ_HANDLED; 6368 } 6369 6370 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6371 { 6372 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6373 struct stmmac_dma_conf *dma_conf; 6374 int chan = tx_q->queue_index; 6375 struct stmmac_priv *priv; 6376 int status; 6377 6378 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6379 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6380 6381 /* Check if adapter is up */ 6382 if (test_bit(STMMAC_DOWN, &priv->state)) 6383 return IRQ_HANDLED; 6384 6385 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6386 6387 if (unlikely(status & tx_hard_error_bump_tc)) { 6388 /* Try to bump up the dma threshold on this failure */ 6389 stmmac_bump_dma_threshold(priv, chan); 6390 } else if (unlikely(status == tx_hard_error)) { 6391 stmmac_tx_err(priv, chan); 6392 } 6393 6394 return IRQ_HANDLED; 6395 } 6396 6397 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6398 { 6399 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6400 struct stmmac_dma_conf *dma_conf; 6401 int chan = rx_q->queue_index; 6402 struct stmmac_priv *priv; 6403 6404 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6405 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6406 6407 /* Check if adapter is up */ 6408 if (test_bit(STMMAC_DOWN, &priv->state)) 6409 return IRQ_HANDLED; 6410 6411 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6412 6413 return IRQ_HANDLED; 6414 } 6415 6416 /** 6417 * stmmac_ioctl - Entry point for the Ioctl 6418 * @dev: Device pointer. 6419 * @rq: An IOCTL specific structure, that can contain a pointer to 6420 * a proprietary structure used to pass information to the driver. 6421 * @cmd: IOCTL command 6422 * Description: Forward the PHY ioctls to phylink 6423 * Return: Zero on success or negative error code. 6424 */ 6425 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6426 { 6427 struct stmmac_priv *priv = netdev_priv (dev); 6428 6429 if (!netif_running(dev)) 6430 return -EINVAL; 6431 6432 return phylink_mii_ioctl(priv->phylink, rq, cmd); 6433 } 6434 6435 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6436 void *cb_priv) 6437 { 6438 struct stmmac_priv *priv = cb_priv; 6439 int ret = -EOPNOTSUPP; 6440 6441 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6442 return ret; 6443 6444 __stmmac_disable_all_queues(priv); 6445 6446 switch (type) { 6447 case TC_SETUP_CLSU32: 6448 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6449 break; 6450 case TC_SETUP_CLSFLOWER: 6451 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6452 break; 6453 default: 6454 break; 6455 } 6456 6457 stmmac_enable_all_queues(priv); 6458 return ret; 6459 } 6460 6461 static LIST_HEAD(stmmac_block_cb_list); 6462 6463 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6464 void *type_data) 6465 { 6466 struct stmmac_priv *priv = netdev_priv(ndev); 6467 6468 switch (type) { 6469 case TC_QUERY_CAPS: 6470 return stmmac_tc_query_caps(priv, priv, type_data); 6471 case TC_SETUP_QDISC_MQPRIO: 6472 return stmmac_tc_setup_mqprio(priv, priv, type_data); 6473 case TC_SETUP_BLOCK: 6474 return flow_block_cb_setup_simple(type_data, 6475 &stmmac_block_cb_list, 6476 stmmac_setup_tc_block_cb, 6477 priv, priv, true); 6478 case TC_SETUP_QDISC_CBS: 6479 return stmmac_tc_setup_cbs(priv, priv, type_data); 6480 case TC_SETUP_QDISC_TAPRIO: 6481 return stmmac_tc_setup_taprio(priv, priv, type_data); 6482 case TC_SETUP_QDISC_ETF: 6483 return stmmac_tc_setup_etf(priv, priv, type_data); 6484 default: 6485 return -EOPNOTSUPP; 6486 } 6487 } 6488 6489 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6490 struct net_device *sb_dev) 6491 { 6492 int gso = skb_shinfo(skb)->gso_type; 6493 6494 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6495 /* 6496 * There is no way to determine the number of TSO/USO 6497 * capable Queues. Let's use always the Queue 0 6498 * because if TSO/USO is supported then at least this 6499 * one will be capable. 6500 */ 6501 return 0; 6502 } 6503 6504 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6505 } 6506 6507 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6508 { 6509 struct stmmac_priv *priv = netdev_priv(ndev); 6510 int ret = 0; 6511 6512 ret = pm_runtime_resume_and_get(priv->device); 6513 if (ret < 0) 6514 return ret; 6515 6516 ret = eth_mac_addr(ndev, addr); 6517 if (ret) 6518 goto set_mac_error; 6519 6520 phylink_rx_clk_stop_block(priv->phylink); 6521 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6522 phylink_rx_clk_stop_unblock(priv->phylink); 6523 6524 set_mac_error: 6525 pm_runtime_put(priv->device); 6526 6527 return ret; 6528 } 6529 6530 #ifdef CONFIG_DEBUG_FS 6531 static struct dentry *stmmac_fs_dir; 6532 6533 static void sysfs_display_ring(void *head, int size, int extend_desc, 6534 struct seq_file *seq, dma_addr_t dma_phy_addr) 6535 { 6536 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6537 struct dma_desc *p = (struct dma_desc *)head; 6538 unsigned int desc_size; 6539 dma_addr_t dma_addr; 6540 int i; 6541 6542 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6543 for (i = 0; i < size; i++) { 6544 dma_addr = dma_phy_addr + i * desc_size; 6545 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6546 i, &dma_addr, 6547 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6548 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6549 if (extend_desc) 6550 p = &(++ep)->basic; 6551 else 6552 p++; 6553 } 6554 } 6555 6556 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6557 { 6558 struct net_device *dev = seq->private; 6559 struct stmmac_priv *priv = netdev_priv(dev); 6560 u8 rx_count = priv->plat->rx_queues_to_use; 6561 u8 tx_count = priv->plat->tx_queues_to_use; 6562 u8 queue; 6563 6564 if ((dev->flags & IFF_UP) == 0) 6565 return 0; 6566 6567 for (queue = 0; queue < rx_count; queue++) { 6568 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6569 6570 seq_printf(seq, "RX Queue %d:\n", queue); 6571 6572 if (priv->extend_desc) { 6573 seq_printf(seq, "Extended descriptor ring:\n"); 6574 sysfs_display_ring((void *)rx_q->dma_erx, 6575 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6576 } else { 6577 seq_printf(seq, "Descriptor ring:\n"); 6578 sysfs_display_ring((void *)rx_q->dma_rx, 6579 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6580 } 6581 } 6582 6583 for (queue = 0; queue < tx_count; queue++) { 6584 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6585 6586 seq_printf(seq, "TX Queue %d:\n", queue); 6587 6588 if (priv->extend_desc) { 6589 seq_printf(seq, "Extended descriptor ring:\n"); 6590 sysfs_display_ring((void *)tx_q->dma_etx, 6591 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6592 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6593 seq_printf(seq, "Descriptor ring:\n"); 6594 sysfs_display_ring((void *)tx_q->dma_tx, 6595 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6596 } 6597 } 6598 6599 return 0; 6600 } 6601 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6602 6603 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6604 { 6605 static const char * const dwxgmac_timestamp_source[] = { 6606 "None", 6607 "Internal", 6608 "External", 6609 "Both", 6610 }; 6611 static const char * const dwxgmac_safety_feature_desc[] = { 6612 "No", 6613 "All Safety Features with ECC and Parity", 6614 "All Safety Features without ECC or Parity", 6615 "All Safety Features with Parity Only", 6616 "ECC Only", 6617 "UNDEFINED", 6618 "UNDEFINED", 6619 "UNDEFINED", 6620 }; 6621 struct net_device *dev = seq->private; 6622 struct stmmac_priv *priv = netdev_priv(dev); 6623 6624 if (!priv->hw_cap_support) { 6625 seq_printf(seq, "DMA HW features not supported\n"); 6626 return 0; 6627 } 6628 6629 seq_printf(seq, "==============================\n"); 6630 seq_printf(seq, "\tDMA HW features\n"); 6631 seq_printf(seq, "==============================\n"); 6632 6633 seq_printf(seq, "\t10/100 Mbps: %s\n", 6634 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6635 seq_printf(seq, "\t1000 Mbps: %s\n", 6636 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6637 seq_printf(seq, "\tHalf duplex: %s\n", 6638 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6639 if (priv->plat->core_type == DWMAC_CORE_XGMAC) { 6640 seq_printf(seq, 6641 "\tNumber of Additional MAC address registers: %d\n", 6642 priv->dma_cap.multi_addr); 6643 } else { 6644 seq_printf(seq, "\tHash Filter: %s\n", 6645 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6646 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6647 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6648 } 6649 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6650 (priv->dma_cap.pcs) ? "Y" : "N"); 6651 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6652 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6653 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6654 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6655 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6656 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6657 seq_printf(seq, "\tRMON module: %s\n", 6658 (priv->dma_cap.rmon) ? "Y" : "N"); 6659 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6660 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6661 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6662 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6663 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 6664 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6665 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6666 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6667 (priv->dma_cap.eee) ? "Y" : "N"); 6668 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6669 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6670 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6671 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6672 priv->plat->core_type == DWMAC_CORE_XGMAC) { 6673 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6674 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6675 } else { 6676 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6677 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6678 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6679 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6680 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6681 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6682 } 6683 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6684 priv->dma_cap.number_rx_channel); 6685 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6686 priv->dma_cap.number_tx_channel); 6687 seq_printf(seq, "\tNumber of Additional RX queues: %u\n", 6688 priv->dma_cap.number_rx_queues); 6689 seq_printf(seq, "\tNumber of Additional TX queues: %u\n", 6690 priv->dma_cap.number_tx_queues); 6691 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6692 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6693 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6694 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6695 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6696 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6697 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6698 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6699 priv->dma_cap.pps_out_num); 6700 seq_printf(seq, "\tSafety Features: %s\n", 6701 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6702 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6703 priv->dma_cap.frpsel ? "Y" : "N"); 6704 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6705 priv->dma_cap.host_dma_width); 6706 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6707 priv->dma_cap.rssen ? "Y" : "N"); 6708 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6709 priv->dma_cap.vlhash ? "Y" : "N"); 6710 seq_printf(seq, "\tSplit Header: %s\n", 6711 priv->dma_cap.sphen ? "Y" : "N"); 6712 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6713 priv->dma_cap.vlins ? "Y" : "N"); 6714 seq_printf(seq, "\tDouble VLAN: %s\n", 6715 priv->dma_cap.dvlan ? "Y" : "N"); 6716 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6717 priv->dma_cap.l3l4fnum); 6718 seq_printf(seq, "\tARP Offloading: %s\n", 6719 priv->dma_cap.arpoffsel ? "Y" : "N"); 6720 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6721 priv->dma_cap.estsel ? "Y" : "N"); 6722 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6723 priv->dma_cap.fpesel ? "Y" : "N"); 6724 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6725 priv->dma_cap.tbssel ? "Y" : "N"); 6726 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6727 priv->dma_cap.tbs_ch_num); 6728 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6729 priv->dma_cap.sgfsel ? "Y" : "N"); 6730 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6731 BIT(priv->dma_cap.ttsfd) >> 1); 6732 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6733 priv->dma_cap.numtc); 6734 seq_printf(seq, "\tDCB Feature: %s\n", 6735 priv->dma_cap.dcben ? "Y" : "N"); 6736 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6737 priv->dma_cap.advthword ? "Y" : "N"); 6738 seq_printf(seq, "\tPTP Offload: %s\n", 6739 priv->dma_cap.ptoen ? "Y" : "N"); 6740 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6741 priv->dma_cap.osten ? "Y" : "N"); 6742 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6743 priv->dma_cap.pfcen ? "Y" : "N"); 6744 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6745 BIT(priv->dma_cap.frpes) << 6); 6746 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6747 BIT(priv->dma_cap.frpbs) << 6); 6748 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6749 priv->dma_cap.frppipe_num); 6750 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6751 priv->dma_cap.nrvf_num ? 6752 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6753 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6754 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6755 seq_printf(seq, "\tDepth of GCL: %lu\n", 6756 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6757 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6758 priv->dma_cap.cbtisel ? "Y" : "N"); 6759 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6760 priv->dma_cap.aux_snapshot_n); 6761 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6762 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6763 seq_printf(seq, "\tEnhanced DMA: %s\n", 6764 priv->dma_cap.edma ? "Y" : "N"); 6765 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6766 priv->dma_cap.ediffc ? "Y" : "N"); 6767 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6768 priv->dma_cap.vxn ? "Y" : "N"); 6769 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6770 priv->dma_cap.dbgmem ? "Y" : "N"); 6771 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6772 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6773 return 0; 6774 } 6775 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6776 6777 /* Use network device events to rename debugfs file entries. 6778 */ 6779 static int stmmac_device_event(struct notifier_block *unused, 6780 unsigned long event, void *ptr) 6781 { 6782 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6783 struct stmmac_priv *priv = netdev_priv(dev); 6784 6785 if (dev->netdev_ops != &stmmac_netdev_ops) 6786 goto done; 6787 6788 switch (event) { 6789 case NETDEV_CHANGENAME: 6790 debugfs_change_name(priv->dbgfs_dir, "%s", dev->name); 6791 break; 6792 } 6793 done: 6794 return NOTIFY_DONE; 6795 } 6796 6797 static struct notifier_block stmmac_notifier = { 6798 .notifier_call = stmmac_device_event, 6799 }; 6800 6801 static void stmmac_init_fs(struct net_device *dev) 6802 { 6803 struct stmmac_priv *priv = netdev_priv(dev); 6804 6805 rtnl_lock(); 6806 6807 /* Create per netdev entries */ 6808 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6809 6810 /* Entry to report DMA RX/TX rings */ 6811 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6812 &stmmac_rings_status_fops); 6813 6814 /* Entry to report the DMA HW features */ 6815 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6816 &stmmac_dma_cap_fops); 6817 6818 rtnl_unlock(); 6819 } 6820 6821 static void stmmac_exit_fs(struct net_device *dev) 6822 { 6823 struct stmmac_priv *priv = netdev_priv(dev); 6824 6825 debugfs_remove_recursive(priv->dbgfs_dir); 6826 } 6827 #endif /* CONFIG_DEBUG_FS */ 6828 6829 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6830 { 6831 unsigned char *data = (unsigned char *)&vid_le; 6832 unsigned char data_byte = 0; 6833 u32 crc = ~0x0; 6834 u32 temp = 0; 6835 int i, bits; 6836 6837 bits = get_bitmask_order(VLAN_VID_MASK); 6838 for (i = 0; i < bits; i++) { 6839 if ((i % 8) == 0) 6840 data_byte = data[i / 8]; 6841 6842 temp = ((crc & 1) ^ data_byte) & 1; 6843 crc >>= 1; 6844 data_byte >>= 1; 6845 6846 if (temp) 6847 crc ^= 0xedb88320; 6848 } 6849 6850 return crc; 6851 } 6852 6853 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6854 { 6855 u32 crc, hash = 0; 6856 u16 pmatch = 0; 6857 int count = 0; 6858 u16 vid = 0; 6859 6860 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6861 __le16 vid_le = cpu_to_le16(vid); 6862 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6863 hash |= (1 << crc); 6864 count++; 6865 } 6866 6867 if (!priv->dma_cap.vlhash) { 6868 if (count > 2) /* VID = 0 always passes filter */ 6869 return -EOPNOTSUPP; 6870 6871 pmatch = vid; 6872 hash = 0; 6873 } 6874 6875 if (!netif_running(priv->dev)) 6876 return 0; 6877 6878 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6879 } 6880 6881 /* FIXME: This may need RXC to be running, but it may be called with BH 6882 * disabled, which means we can't call phylink_rx_clk_stop*(). 6883 */ 6884 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6885 { 6886 struct stmmac_priv *priv = netdev_priv(ndev); 6887 unsigned int num_double_vlans; 6888 bool is_double = false; 6889 int ret; 6890 6891 ret = pm_runtime_resume_and_get(priv->device); 6892 if (ret < 0) 6893 return ret; 6894 6895 if (be16_to_cpu(proto) == ETH_P_8021AD) 6896 is_double = true; 6897 6898 set_bit(vid, priv->active_vlans); 6899 num_double_vlans = priv->num_double_vlans + is_double; 6900 ret = stmmac_vlan_update(priv, num_double_vlans); 6901 if (ret) { 6902 clear_bit(vid, priv->active_vlans); 6903 goto err_pm_put; 6904 } 6905 6906 if (priv->hw->num_vlan) { 6907 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6908 if (ret) { 6909 clear_bit(vid, priv->active_vlans); 6910 stmmac_vlan_update(priv, priv->num_double_vlans); 6911 goto err_pm_put; 6912 } 6913 } 6914 6915 priv->num_double_vlans = num_double_vlans; 6916 6917 err_pm_put: 6918 pm_runtime_put(priv->device); 6919 6920 return ret; 6921 } 6922 6923 /* FIXME: This may need RXC to be running, but it may be called with BH 6924 * disabled, which means we can't call phylink_rx_clk_stop*(). 6925 */ 6926 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6927 { 6928 struct stmmac_priv *priv = netdev_priv(ndev); 6929 unsigned int num_double_vlans; 6930 bool is_double = false; 6931 int ret; 6932 6933 ret = pm_runtime_resume_and_get(priv->device); 6934 if (ret < 0) 6935 return ret; 6936 6937 if (be16_to_cpu(proto) == ETH_P_8021AD) 6938 is_double = true; 6939 6940 clear_bit(vid, priv->active_vlans); 6941 num_double_vlans = priv->num_double_vlans - is_double; 6942 ret = stmmac_vlan_update(priv, num_double_vlans); 6943 if (ret) { 6944 set_bit(vid, priv->active_vlans); 6945 goto del_vlan_error; 6946 } 6947 6948 if (priv->hw->num_vlan) { 6949 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6950 if (ret) { 6951 set_bit(vid, priv->active_vlans); 6952 stmmac_vlan_update(priv, priv->num_double_vlans); 6953 goto del_vlan_error; 6954 } 6955 } 6956 6957 priv->num_double_vlans = num_double_vlans; 6958 6959 del_vlan_error: 6960 pm_runtime_put(priv->device); 6961 6962 return ret; 6963 } 6964 6965 static void stmmac_vlan_restore(struct stmmac_priv *priv) 6966 { 6967 if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) 6968 return; 6969 6970 if (priv->hw->num_vlan) 6971 stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); 6972 6973 stmmac_vlan_update(priv, priv->num_double_vlans); 6974 } 6975 6976 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6977 { 6978 struct stmmac_priv *priv = netdev_priv(dev); 6979 6980 switch (bpf->command) { 6981 case XDP_SETUP_PROG: 6982 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6983 case XDP_SETUP_XSK_POOL: 6984 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6985 bpf->xsk.queue_id); 6986 default: 6987 return -EOPNOTSUPP; 6988 } 6989 } 6990 6991 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6992 struct xdp_frame **frames, u32 flags) 6993 { 6994 struct stmmac_priv *priv = netdev_priv(dev); 6995 int cpu = smp_processor_id(); 6996 struct netdev_queue *nq; 6997 int i, nxmit = 0; 6998 int queue; 6999 7000 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 7001 return -ENETDOWN; 7002 7003 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 7004 return -EINVAL; 7005 7006 queue = stmmac_xdp_get_tx_queue(priv, cpu); 7007 nq = netdev_get_tx_queue(priv->dev, queue); 7008 7009 __netif_tx_lock(nq, cpu); 7010 /* Avoids TX time-out as we are sharing with slow path */ 7011 txq_trans_cond_update(nq); 7012 7013 for (i = 0; i < num_frames; i++) { 7014 int res; 7015 7016 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 7017 if (res == STMMAC_XDP_CONSUMED) 7018 break; 7019 7020 nxmit++; 7021 } 7022 7023 if (flags & XDP_XMIT_FLUSH) { 7024 stmmac_flush_tx_descriptors(priv, queue); 7025 stmmac_tx_timer_arm(priv, queue); 7026 } 7027 7028 __netif_tx_unlock(nq); 7029 7030 return nxmit; 7031 } 7032 7033 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 7034 { 7035 struct stmmac_channel *ch = &priv->channel[queue]; 7036 unsigned long flags; 7037 7038 spin_lock_irqsave(&ch->lock, flags); 7039 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7040 spin_unlock_irqrestore(&ch->lock, flags); 7041 7042 stmmac_stop_rx_dma(priv, queue); 7043 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7044 } 7045 7046 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 7047 { 7048 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7049 struct stmmac_channel *ch = &priv->channel[queue]; 7050 unsigned long flags; 7051 int ret; 7052 7053 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7054 if (ret) { 7055 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 7056 return; 7057 } 7058 7059 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 7060 if (ret) { 7061 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7062 netdev_err(priv->dev, "Failed to init RX desc.\n"); 7063 return; 7064 } 7065 7066 stmmac_reset_rx_queue(priv, queue); 7067 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 7068 7069 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7070 rx_q->dma_rx_phy, queue); 7071 7072 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->buf_alloc_num); 7073 7074 stmmac_set_queue_rx_buf_size(priv, rx_q, queue); 7075 7076 stmmac_start_rx_dma(priv, queue); 7077 7078 spin_lock_irqsave(&ch->lock, flags); 7079 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7080 spin_unlock_irqrestore(&ch->lock, flags); 7081 } 7082 7083 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 7084 { 7085 struct stmmac_channel *ch = &priv->channel[queue]; 7086 unsigned long flags; 7087 7088 spin_lock_irqsave(&ch->lock, flags); 7089 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7090 spin_unlock_irqrestore(&ch->lock, flags); 7091 7092 stmmac_stop_tx_dma(priv, queue); 7093 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7094 } 7095 7096 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 7097 { 7098 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7099 struct stmmac_channel *ch = &priv->channel[queue]; 7100 unsigned long flags; 7101 int ret; 7102 7103 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7104 if (ret) { 7105 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 7106 return; 7107 } 7108 7109 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 7110 if (ret) { 7111 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7112 netdev_err(priv->dev, "Failed to init TX desc.\n"); 7113 return; 7114 } 7115 7116 stmmac_reset_tx_queue(priv, queue); 7117 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 7118 7119 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7120 tx_q->dma_tx_phy, queue); 7121 7122 if (tx_q->tbs & STMMAC_TBS_AVAIL) 7123 stmmac_enable_tbs(priv, priv->ioaddr, 1, queue); 7124 7125 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, 0); 7126 7127 stmmac_start_tx_dma(priv, queue); 7128 7129 spin_lock_irqsave(&ch->lock, flags); 7130 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7131 spin_unlock_irqrestore(&ch->lock, flags); 7132 } 7133 7134 void stmmac_xdp_release(struct net_device *dev) 7135 { 7136 struct stmmac_priv *priv = netdev_priv(dev); 7137 u8 chan; 7138 7139 /* Ensure tx function is not running */ 7140 netif_tx_disable(dev); 7141 7142 /* Disable NAPI process */ 7143 stmmac_disable_all_queues(priv); 7144 7145 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7146 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7147 7148 /* Free the IRQ lines */ 7149 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 7150 7151 /* Stop TX/RX DMA channels */ 7152 stmmac_stop_all_dma(priv); 7153 7154 /* Release and free the Rx/Tx resources */ 7155 free_dma_desc_resources(priv, &priv->dma_conf); 7156 7157 /* Disable the MAC Rx/Tx */ 7158 stmmac_mac_set(priv, priv->ioaddr, false); 7159 7160 /* set trans_start so we don't get spurious 7161 * watchdogs during reset 7162 */ 7163 netif_trans_update(dev); 7164 netif_carrier_off(dev); 7165 } 7166 7167 int stmmac_xdp_open(struct net_device *dev) 7168 { 7169 struct stmmac_priv *priv = netdev_priv(dev); 7170 u8 rx_cnt = priv->plat->rx_queues_to_use; 7171 u8 tx_cnt = priv->plat->tx_queues_to_use; 7172 u8 dma_csr_ch = max(rx_cnt, tx_cnt); 7173 struct stmmac_rx_queue *rx_q; 7174 struct stmmac_tx_queue *tx_q; 7175 bool sph_en; 7176 u8 chan; 7177 int ret; 7178 7179 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 7180 if (ret < 0) { 7181 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 7182 __func__); 7183 goto dma_desc_error; 7184 } 7185 7186 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 7187 if (ret < 0) { 7188 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 7189 __func__); 7190 goto init_error; 7191 } 7192 7193 stmmac_reset_queues_param(priv); 7194 7195 /* DMA CSR Channel configuration */ 7196 for (chan = 0; chan < dma_csr_ch; chan++) { 7197 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 7198 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 7199 } 7200 7201 /* Adjust Split header */ 7202 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 7203 7204 /* DMA RX Channel Configuration */ 7205 for (chan = 0; chan < rx_cnt; chan++) { 7206 rx_q = &priv->dma_conf.rx_queue[chan]; 7207 7208 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7209 rx_q->dma_rx_phy, chan); 7210 7211 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 7212 rx_q->buf_alloc_num); 7213 7214 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 7215 7216 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 7217 } 7218 7219 /* DMA TX Channel Configuration */ 7220 for (chan = 0; chan < tx_cnt; chan++) { 7221 tx_q = &priv->dma_conf.tx_queue[chan]; 7222 7223 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7224 tx_q->dma_tx_phy, chan); 7225 7226 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 7227 7228 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 7229 } 7230 7231 /* Enable the MAC Rx/Tx */ 7232 stmmac_mac_set(priv, priv->ioaddr, true); 7233 7234 /* Start Rx & Tx DMA Channels */ 7235 stmmac_start_all_dma(priv); 7236 7237 ret = stmmac_request_irq(dev); 7238 if (ret) 7239 goto irq_error; 7240 7241 /* Enable NAPI process*/ 7242 stmmac_enable_all_queues(priv); 7243 netif_carrier_on(dev); 7244 netif_tx_start_all_queues(dev); 7245 stmmac_enable_all_dma_irq(priv); 7246 7247 return 0; 7248 7249 irq_error: 7250 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7251 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7252 7253 init_error: 7254 free_dma_desc_resources(priv, &priv->dma_conf); 7255 dma_desc_error: 7256 return ret; 7257 } 7258 7259 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 7260 { 7261 struct stmmac_priv *priv = netdev_priv(dev); 7262 struct stmmac_rx_queue *rx_q; 7263 struct stmmac_tx_queue *tx_q; 7264 struct stmmac_channel *ch; 7265 7266 if (test_bit(STMMAC_DOWN, &priv->state) || 7267 !netif_carrier_ok(priv->dev)) 7268 return -ENETDOWN; 7269 7270 if (!stmmac_xdp_is_enabled(priv)) 7271 return -EINVAL; 7272 7273 if (queue >= priv->plat->rx_queues_to_use || 7274 queue >= priv->plat->tx_queues_to_use) 7275 return -EINVAL; 7276 7277 rx_q = &priv->dma_conf.rx_queue[queue]; 7278 tx_q = &priv->dma_conf.tx_queue[queue]; 7279 ch = &priv->channel[queue]; 7280 7281 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 7282 return -EINVAL; 7283 7284 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 7285 /* EQoS does not have per-DMA channel SW interrupt, 7286 * so we schedule RX Napi straight-away. 7287 */ 7288 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 7289 __napi_schedule(&ch->rxtx_napi); 7290 } 7291 7292 return 0; 7293 } 7294 7295 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7296 { 7297 struct stmmac_priv *priv = netdev_priv(dev); 7298 u8 tx_cnt = priv->plat->tx_queues_to_use; 7299 u8 rx_cnt = priv->plat->rx_queues_to_use; 7300 unsigned int start; 7301 u8 q; 7302 7303 for (q = 0; q < tx_cnt; q++) { 7304 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7305 u64 tx_packets; 7306 u64 tx_bytes; 7307 7308 do { 7309 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 7310 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 7311 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 7312 do { 7313 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 7314 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 7315 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 7316 7317 stats->tx_packets += tx_packets; 7318 stats->tx_bytes += tx_bytes; 7319 } 7320 7321 for (q = 0; q < rx_cnt; q++) { 7322 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7323 u64 rx_packets; 7324 u64 rx_bytes; 7325 7326 do { 7327 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 7328 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 7329 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 7330 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 7331 7332 stats->rx_packets += rx_packets; 7333 stats->rx_bytes += rx_bytes; 7334 } 7335 7336 stats->rx_dropped = priv->xstats.rx_dropped; 7337 stats->rx_errors = priv->xstats.rx_errors; 7338 stats->tx_dropped = priv->xstats.tx_dropped; 7339 stats->tx_errors = priv->xstats.tx_errors; 7340 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7341 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7342 stats->rx_length_errors = priv->xstats.rx_length; 7343 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7344 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7345 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7346 } 7347 7348 static const struct net_device_ops stmmac_netdev_ops = { 7349 .ndo_open = stmmac_open, 7350 .ndo_start_xmit = stmmac_xmit, 7351 .ndo_features_check = stmmac_features_check, 7352 .ndo_stop = stmmac_release, 7353 .ndo_change_mtu = stmmac_change_mtu, 7354 .ndo_fix_features = stmmac_fix_features, 7355 .ndo_set_features = stmmac_set_features, 7356 .ndo_set_rx_mode = stmmac_set_rx_mode, 7357 .ndo_tx_timeout = stmmac_tx_timeout, 7358 .ndo_eth_ioctl = stmmac_ioctl, 7359 .ndo_get_stats64 = stmmac_get_stats64, 7360 .ndo_setup_tc = stmmac_setup_tc, 7361 .ndo_select_queue = stmmac_select_queue, 7362 .ndo_set_mac_address = stmmac_set_mac_address, 7363 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7364 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7365 .ndo_bpf = stmmac_bpf, 7366 .ndo_xdp_xmit = stmmac_xdp_xmit, 7367 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7368 .ndo_hwtstamp_get = stmmac_hwtstamp_get, 7369 .ndo_hwtstamp_set = stmmac_hwtstamp_set, 7370 }; 7371 7372 static void stmmac_reset_subtask(struct stmmac_priv *priv) 7373 { 7374 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7375 return; 7376 if (test_bit(STMMAC_DOWN, &priv->state)) 7377 return; 7378 7379 netdev_err(priv->dev, "Reset adapter.\n"); 7380 7381 rtnl_lock(); 7382 netif_trans_update(priv->dev); 7383 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7384 usleep_range(1000, 2000); 7385 7386 set_bit(STMMAC_DOWN, &priv->state); 7387 dev_close(priv->dev); 7388 dev_open(priv->dev, NULL); 7389 clear_bit(STMMAC_DOWN, &priv->state); 7390 clear_bit(STMMAC_RESETING, &priv->state); 7391 rtnl_unlock(); 7392 } 7393 7394 static void stmmac_service_task(struct work_struct *work) 7395 { 7396 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7397 service_task); 7398 7399 stmmac_reset_subtask(priv); 7400 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7401 } 7402 7403 static void stmmac_print_actphyif(struct stmmac_priv *priv) 7404 { 7405 const char **phyif_table; 7406 const char *actphyif_str; 7407 size_t phyif_table_size; 7408 7409 switch (priv->plat->core_type) { 7410 case DWMAC_CORE_MAC100: 7411 return; 7412 7413 case DWMAC_CORE_GMAC: 7414 case DWMAC_CORE_GMAC4: 7415 phyif_table = stmmac_dwmac_actphyif; 7416 phyif_table_size = ARRAY_SIZE(stmmac_dwmac_actphyif); 7417 break; 7418 7419 case DWMAC_CORE_XGMAC: 7420 phyif_table = stmmac_dwxgmac_phyif; 7421 phyif_table_size = ARRAY_SIZE(stmmac_dwxgmac_phyif); 7422 break; 7423 } 7424 7425 if (priv->dma_cap.actphyif < phyif_table_size) 7426 actphyif_str = phyif_table[priv->dma_cap.actphyif]; 7427 else 7428 actphyif_str = NULL; 7429 7430 if (!actphyif_str) 7431 actphyif_str = "unknown"; 7432 7433 dev_info(priv->device, "Active PHY interface: %s (%u)\n", 7434 actphyif_str, priv->dma_cap.actphyif); 7435 } 7436 7437 /** 7438 * stmmac_hw_init - Init the MAC device 7439 * @priv: driver private structure 7440 * Description: this function is to configure the MAC device according to 7441 * some platform parameters or the HW capability register. It prepares the 7442 * driver to use either ring or chain modes and to setup either enhanced or 7443 * normal descriptors. 7444 */ 7445 static int stmmac_hw_init(struct stmmac_priv *priv) 7446 { 7447 int ret; 7448 7449 /* dwmac-sun8i only work in chain mode */ 7450 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7451 chain_mode = 1; 7452 priv->chain_mode = !!chain_mode; 7453 7454 /* Initialize HW Interface */ 7455 ret = stmmac_hwif_init(priv); 7456 if (ret) 7457 return ret; 7458 7459 /* Get the HW capability (new GMAC newer than 3.50a) */ 7460 priv->hw_cap_support = stmmac_get_hw_features(priv); 7461 if (priv->hw_cap_support) { 7462 dev_info(priv->device, "DMA HW capability register supported\n"); 7463 7464 /* We can override some gmac/dma configuration fields: e.g. 7465 * enh_desc, tx_coe (e.g. that are passed through the 7466 * platform) with the values from the HW capability 7467 * register (if supported). 7468 */ 7469 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7470 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7471 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7472 if (priv->dma_cap.hash_tb_sz) { 7473 priv->hw->multicast_filter_bins = 7474 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7475 priv->hw->mcast_bits_log2 = 7476 ilog2(priv->hw->multicast_filter_bins); 7477 } 7478 7479 /* TXCOE doesn't work in thresh DMA mode */ 7480 if (priv->plat->force_thresh_dma_mode) 7481 priv->plat->tx_coe = false; 7482 else 7483 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7484 7485 /* In case of GMAC4 rx_coe is from HW cap register. */ 7486 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7487 7488 if (priv->dma_cap.rx_coe_type2) 7489 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7490 else if (priv->dma_cap.rx_coe_type1) 7491 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7492 7493 stmmac_print_actphyif(priv); 7494 } else { 7495 dev_info(priv->device, "No HW DMA feature register supported\n"); 7496 } 7497 7498 if (priv->plat->rx_coe) { 7499 priv->hw->rx_csum = priv->plat->rx_coe; 7500 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7501 if (priv->synopsys_id < DWMAC_CORE_4_00) 7502 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7503 } 7504 if (priv->plat->tx_coe) 7505 dev_info(priv->device, "TX Checksum insertion supported\n"); 7506 7507 if (priv->plat->pmt) { 7508 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7509 device_set_wakeup_capable(priv->device, 1); 7510 devm_pm_set_wake_irq(priv->device, priv->wol_irq); 7511 } 7512 7513 if (priv->dma_cap.number_rx_queues && 7514 priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) { 7515 dev_warn(priv->device, 7516 "Number of Rx queues (%u) exceeds dma capability\n", 7517 priv->plat->rx_queues_to_use); 7518 priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues; 7519 } 7520 if (priv->dma_cap.number_tx_queues && 7521 priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) { 7522 dev_warn(priv->device, 7523 "Number of Tx queues (%u) exceeds dma capability\n", 7524 priv->plat->tx_queues_to_use); 7525 priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues; 7526 } 7527 7528 if (priv->dma_cap.rx_fifo_size && 7529 priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) { 7530 dev_warn(priv->device, 7531 "Rx FIFO size (%u) exceeds dma capability\n", 7532 priv->plat->rx_fifo_size); 7533 priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size; 7534 } 7535 if (priv->dma_cap.tx_fifo_size && 7536 priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) { 7537 dev_warn(priv->device, 7538 "Tx FIFO size (%u) exceeds dma capability\n", 7539 priv->plat->tx_fifo_size); 7540 priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size; 7541 } 7542 7543 priv->hw->vlan_fail_q_en = 7544 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7545 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7546 7547 /* Run HW quirks, if any */ 7548 if (priv->hwif_quirks) { 7549 ret = priv->hwif_quirks(priv); 7550 if (ret) 7551 return ret; 7552 } 7553 7554 /* Set alternate descriptor size (which tells the hardware that 7555 * descriptors are 8 32-bit words) when using extended descriptors 7556 * with ring mode. Only applicable for pre-v4.0 cores. Platform glue 7557 * is not expected to change this. 7558 */ 7559 priv->plat->dma_cfg->atds = priv->extend_desc && 7560 priv->descriptor_mode == STMMAC_RING_MODE; 7561 7562 /* Rx Watchdog is available in the COREs newer than the 3.40. 7563 * In some case, for example on bugged HW this feature 7564 * has to be disable and this can be done by passing the 7565 * riwt_off field from the platform. 7566 */ 7567 if ((priv->synopsys_id >= DWMAC_CORE_3_50 || 7568 priv->plat->core_type == DWMAC_CORE_XGMAC) && 7569 !priv->plat->riwt_off) { 7570 priv->use_riwt = 1; 7571 dev_info(priv->device, 7572 "Enable RX Mitigation via HW Watchdog Timer\n"); 7573 } 7574 7575 /* Unimplemented PCS init (as indicated by stmmac_do_callback() 7576 * perversely returning -EINVAL) is non-fatal. 7577 */ 7578 ret = stmmac_mac_pcs_init(priv); 7579 if (ret != -EINVAL) 7580 return ret; 7581 7582 return 0; 7583 } 7584 7585 static void stmmac_napi_add(struct net_device *dev) 7586 { 7587 struct stmmac_priv *priv = netdev_priv(dev); 7588 u8 queue, maxq; 7589 7590 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7591 7592 for (queue = 0; queue < maxq; queue++) { 7593 struct stmmac_channel *ch = &priv->channel[queue]; 7594 7595 ch->priv_data = priv; 7596 ch->index = queue; 7597 spin_lock_init(&ch->lock); 7598 7599 if (queue < priv->plat->rx_queues_to_use) { 7600 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7601 } 7602 if (queue < priv->plat->tx_queues_to_use) { 7603 netif_napi_add_tx(dev, &ch->tx_napi, 7604 stmmac_napi_poll_tx); 7605 } 7606 if (queue < priv->plat->rx_queues_to_use && 7607 queue < priv->plat->tx_queues_to_use) { 7608 netif_napi_add(dev, &ch->rxtx_napi, 7609 stmmac_napi_poll_rxtx); 7610 } 7611 } 7612 } 7613 7614 static void stmmac_napi_del(struct net_device *dev) 7615 { 7616 struct stmmac_priv *priv = netdev_priv(dev); 7617 u8 queue, maxq; 7618 7619 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7620 7621 for (queue = 0; queue < maxq; queue++) { 7622 struct stmmac_channel *ch = &priv->channel[queue]; 7623 7624 if (queue < priv->plat->rx_queues_to_use) 7625 netif_napi_del(&ch->rx_napi); 7626 if (queue < priv->plat->tx_queues_to_use) 7627 netif_napi_del(&ch->tx_napi); 7628 if (queue < priv->plat->rx_queues_to_use && 7629 queue < priv->plat->tx_queues_to_use) { 7630 netif_napi_del(&ch->rxtx_napi); 7631 } 7632 } 7633 } 7634 7635 int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt) 7636 { 7637 struct stmmac_priv *priv = netdev_priv(dev); 7638 int ret = 0, i; 7639 7640 if (netif_running(dev)) 7641 stmmac_release(dev); 7642 7643 stmmac_napi_del(dev); 7644 7645 priv->plat->rx_queues_to_use = rx_cnt; 7646 priv->plat->tx_queues_to_use = tx_cnt; 7647 if (!netif_is_rxfh_configured(dev)) 7648 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7649 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7650 rx_cnt); 7651 7652 stmmac_napi_add(dev); 7653 7654 if (netif_running(dev)) 7655 ret = stmmac_open(dev); 7656 7657 return ret; 7658 } 7659 7660 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7661 { 7662 struct stmmac_priv *priv = netdev_priv(dev); 7663 int ret = 0; 7664 7665 if (netif_running(dev)) 7666 stmmac_release(dev); 7667 7668 priv->dma_conf.dma_rx_size = rx_size; 7669 priv->dma_conf.dma_tx_size = tx_size; 7670 7671 if (netif_running(dev)) 7672 ret = stmmac_open(dev); 7673 7674 return ret; 7675 } 7676 7677 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7678 { 7679 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7680 struct dma_desc *desc_contains_ts = ctx->desc; 7681 struct stmmac_priv *priv = ctx->priv; 7682 struct dma_desc *ndesc = ctx->ndesc; 7683 struct dma_desc *desc = ctx->desc; 7684 u64 ns = 0; 7685 7686 if (!priv->hwts_rx_en) 7687 return -ENODATA; 7688 7689 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7690 if (dwmac_is_xmac(priv->plat->core_type)) 7691 desc_contains_ts = ndesc; 7692 7693 /* Check if timestamp is available */ 7694 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7695 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7696 ns -= priv->plat->cdc_error_adj; 7697 *timestamp = ns_to_ktime(ns); 7698 return 0; 7699 } 7700 7701 return -ENODATA; 7702 } 7703 7704 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7705 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7706 }; 7707 7708 static int stmmac_dl_ts_coarse_set(struct devlink *dl, u32 id, 7709 struct devlink_param_gset_ctx *ctx, 7710 struct netlink_ext_ack *extack) 7711 { 7712 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7713 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7714 7715 priv->tsfupdt_coarse = ctx->val.vbool; 7716 7717 if (priv->tsfupdt_coarse) 7718 priv->systime_flags &= ~PTP_TCR_TSCFUPDT; 7719 else 7720 priv->systime_flags |= PTP_TCR_TSCFUPDT; 7721 7722 /* In Coarse mode, we can use a smaller subsecond increment, let's 7723 * reconfigure the systime, subsecond increment and addend. 7724 */ 7725 stmmac_update_subsecond_increment(priv); 7726 7727 return 0; 7728 } 7729 7730 static int stmmac_dl_ts_coarse_get(struct devlink *dl, u32 id, 7731 struct devlink_param_gset_ctx *ctx, 7732 struct netlink_ext_ack *extack) 7733 { 7734 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7735 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7736 7737 ctx->val.vbool = priv->tsfupdt_coarse; 7738 7739 return 0; 7740 } 7741 7742 static const struct devlink_param stmmac_devlink_params[] = { 7743 DEVLINK_PARAM_DRIVER(STMMAC_DEVLINK_PARAM_ID_TS_COARSE, "phc_coarse_adj", 7744 DEVLINK_PARAM_TYPE_BOOL, 7745 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 7746 stmmac_dl_ts_coarse_get, 7747 stmmac_dl_ts_coarse_set, NULL), 7748 }; 7749 7750 /* None of the generic devlink parameters are implemented */ 7751 static const struct devlink_ops stmmac_devlink_ops = {}; 7752 7753 static int stmmac_register_devlink(struct stmmac_priv *priv) 7754 { 7755 struct stmmac_devlink_priv *dl_priv; 7756 int ret; 7757 7758 /* For now, what is exposed over devlink is only relevant when 7759 * timestamping is available and we have a valid ptp clock rate 7760 */ 7761 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp) || 7762 !priv->plat->clk_ptp_rate) 7763 return 0; 7764 7765 priv->devlink = devlink_alloc(&stmmac_devlink_ops, sizeof(*dl_priv), 7766 priv->device); 7767 if (!priv->devlink) 7768 return -ENOMEM; 7769 7770 dl_priv = devlink_priv(priv->devlink); 7771 dl_priv->stmmac_priv = priv; 7772 7773 ret = devlink_params_register(priv->devlink, stmmac_devlink_params, 7774 ARRAY_SIZE(stmmac_devlink_params)); 7775 if (ret) 7776 goto dl_free; 7777 7778 devlink_register(priv->devlink); 7779 return 0; 7780 7781 dl_free: 7782 devlink_free(priv->devlink); 7783 7784 return ret; 7785 } 7786 7787 static void stmmac_unregister_devlink(struct stmmac_priv *priv) 7788 { 7789 if (!priv->devlink) 7790 return; 7791 7792 devlink_unregister(priv->devlink); 7793 devlink_params_unregister(priv->devlink, stmmac_devlink_params, 7794 ARRAY_SIZE(stmmac_devlink_params)); 7795 devlink_free(priv->devlink); 7796 } 7797 7798 struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev) 7799 { 7800 struct plat_stmmacenet_data *plat_dat; 7801 int i; 7802 7803 plat_dat = devm_kzalloc(dev, sizeof(*plat_dat), GFP_KERNEL); 7804 if (!plat_dat) 7805 return NULL; 7806 7807 plat_dat->dma_cfg = &plat_dat->__dma_cfg; 7808 7809 /* Set the defaults: 7810 * - phy autodetection 7811 * - determine GMII_Address CR field from CSR clock 7812 * - allow MTU up to JUMBO_LEN 7813 * - hash table size 7814 * - one unicast filter entry 7815 */ 7816 plat_dat->phy_addr = -1; 7817 plat_dat->clk_csr = -1; 7818 plat_dat->maxmtu = JUMBO_LEN; 7819 plat_dat->multicast_filter_bins = HASH_TABLE_SIZE; 7820 plat_dat->unicast_filter_entries = 1; 7821 7822 /* Set the mtl defaults */ 7823 plat_dat->tx_queues_to_use = 1; 7824 plat_dat->rx_queues_to_use = 1; 7825 7826 /* Setup the default RX queue channel map */ 7827 for (i = 0; i < ARRAY_SIZE(plat_dat->rx_queues_cfg); i++) 7828 plat_dat->rx_queues_cfg[i].chan = i; 7829 7830 return plat_dat; 7831 } 7832 EXPORT_SYMBOL_GPL(stmmac_plat_dat_alloc); 7833 7834 static int __stmmac_dvr_probe(struct device *device, 7835 struct plat_stmmacenet_data *plat_dat, 7836 struct stmmac_resources *res) 7837 { 7838 struct net_device *ndev = NULL; 7839 struct stmmac_priv *priv; 7840 int i, ret = 0; 7841 u8 rxq; 7842 7843 if (!plat_dat->dma_cfg || !plat_dat->dma_cfg->pbl) { 7844 dev_err(device, "invalid DMA configuration\n"); 7845 return -EINVAL; 7846 } 7847 7848 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7849 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7850 if (!ndev) 7851 return -ENOMEM; 7852 7853 SET_NETDEV_DEV(ndev, device); 7854 7855 priv = netdev_priv(ndev); 7856 priv->device = device; 7857 priv->dev = ndev; 7858 7859 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7860 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7861 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7862 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7863 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7864 } 7865 7866 priv->xstats.pcpu_stats = 7867 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7868 if (!priv->xstats.pcpu_stats) 7869 return -ENOMEM; 7870 7871 stmmac_set_ethtool_ops(ndev); 7872 priv->pause_time = pause; 7873 priv->plat = plat_dat; 7874 priv->ioaddr = res->addr; 7875 priv->dev->base_addr = (unsigned long)res->addr; 7876 priv->plat->dma_cfg->multi_msi_en = 7877 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7878 7879 priv->dev->irq = res->irq; 7880 priv->wol_irq = res->wol_irq; 7881 priv->sfty_irq = res->sfty_irq; 7882 7883 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) { 7884 ret = stmmac_msi_init(priv, res); 7885 if (ret) 7886 return ret; 7887 } 7888 7889 if (!is_zero_ether_addr(res->mac)) 7890 eth_hw_addr_set(priv->dev, res->mac); 7891 7892 dev_set_drvdata(device, priv->dev); 7893 7894 /* Verify driver arguments */ 7895 stmmac_verify_args(); 7896 7897 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7898 if (!priv->af_xdp_zc_qps) 7899 return -ENOMEM; 7900 7901 /* Allocate workqueue */ 7902 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7903 if (!priv->wq) { 7904 dev_err(priv->device, "failed to create workqueue\n"); 7905 ret = -ENOMEM; 7906 goto error_wq_init; 7907 } 7908 7909 INIT_WORK(&priv->service_task, stmmac_service_task); 7910 7911 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 7912 7913 /* Override with kernel parameters if supplied XXX CRS XXX 7914 * this needs to have multiple instances 7915 */ 7916 if ((phyaddr >= 0) && (phyaddr <= 31)) 7917 priv->plat->phy_addr = phyaddr; 7918 7919 if (priv->plat->stmmac_rst) { 7920 ret = reset_control_assert(priv->plat->stmmac_rst); 7921 reset_control_deassert(priv->plat->stmmac_rst); 7922 /* Some reset controllers have only reset callback instead of 7923 * assert + deassert callbacks pair. 7924 */ 7925 if (ret == -ENOTSUPP) 7926 reset_control_reset(priv->plat->stmmac_rst); 7927 } 7928 7929 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7930 if (ret == -ENOTSUPP) 7931 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7932 ERR_PTR(ret)); 7933 7934 /* Wait a bit for the reset to take effect */ 7935 udelay(10); 7936 7937 /* Init MAC and get the capabilities */ 7938 ret = stmmac_hw_init(priv); 7939 if (ret) 7940 goto error_hw_init; 7941 7942 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7943 */ 7944 if (priv->synopsys_id < DWMAC_CORE_5_20) 7945 priv->plat->dma_cfg->dche = false; 7946 7947 stmmac_check_ether_addr(priv); 7948 7949 ndev->netdev_ops = &stmmac_netdev_ops; 7950 7951 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7952 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7953 7954 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7955 NETIF_F_RXCSUM; 7956 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7957 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7958 7959 ret = stmmac_tc_init(priv, priv); 7960 if (!ret) { 7961 ndev->hw_features |= NETIF_F_HW_TC; 7962 } 7963 7964 stmmac_set_gso_features(ndev); 7965 7966 if (priv->dma_cap.sphen && 7967 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7968 ndev->hw_features |= NETIF_F_GRO; 7969 priv->sph_capable = true; 7970 priv->sph_active = priv->sph_capable; 7971 dev_info(priv->device, "SPH feature enabled\n"); 7972 } 7973 7974 /* Ideally our host DMA address width is the same as for the 7975 * device. However, it may differ and then we have to use our 7976 * host DMA width for allocation and the device DMA width for 7977 * register handling. 7978 */ 7979 if (priv->plat->host_dma_width) 7980 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7981 else 7982 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7983 7984 if (priv->dma_cap.host_dma_width) { 7985 ret = dma_set_mask_and_coherent(device, 7986 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7987 if (!ret) { 7988 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7989 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7990 7991 /* 7992 * If more than 32 bits can be addressed, make sure to 7993 * enable enhanced addressing mode. 7994 */ 7995 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7996 priv->plat->dma_cfg->eame = true; 7997 } else { 7998 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7999 if (ret) { 8000 dev_err(priv->device, "Failed to set DMA Mask\n"); 8001 goto error_hw_init; 8002 } 8003 8004 priv->dma_cap.host_dma_width = 32; 8005 } 8006 } 8007 8008 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 8009 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 8010 #ifdef STMMAC_VLAN_TAG_USED 8011 /* Both mac100 and gmac support receive VLAN tag detection */ 8012 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 8013 if (dwmac_is_xmac(priv->plat->core_type)) { 8014 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 8015 priv->hw->hw_vlan_en = true; 8016 } 8017 if (priv->dma_cap.vlhash) { 8018 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 8019 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 8020 } 8021 if (priv->dma_cap.vlins) 8022 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 8023 #endif 8024 priv->msg_enable = netif_msg_init(debug, default_msg_level); 8025 8026 priv->xstats.threshold = tc; 8027 8028 /* Initialize RSS */ 8029 rxq = priv->plat->rx_queues_to_use; 8030 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 8031 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 8032 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 8033 8034 if (priv->dma_cap.rssen && priv->plat->rss_en) 8035 ndev->features |= NETIF_F_RXHASH; 8036 8037 ndev->vlan_features |= ndev->features; 8038 8039 /* MTU range: 46 - hw-specific max */ 8040 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 8041 8042 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 8043 ndev->max_mtu = XGMAC_JUMBO_LEN; 8044 else if (priv->plat->enh_desc || priv->synopsys_id >= DWMAC_CORE_4_00) 8045 ndev->max_mtu = JUMBO_LEN; 8046 else 8047 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 8048 8049 /* Warn if the platform's maxmtu is smaller than the minimum MTU, 8050 * otherwise clamp the maximum MTU above to the platform's maxmtu. 8051 */ 8052 if (priv->plat->maxmtu < ndev->min_mtu) 8053 dev_warn(priv->device, 8054 "%s: warning: maxmtu having invalid value (%d)\n", 8055 __func__, priv->plat->maxmtu); 8056 else if (priv->plat->maxmtu < ndev->max_mtu) 8057 ndev->max_mtu = priv->plat->maxmtu; 8058 8059 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 8060 8061 /* Setup channels NAPI */ 8062 stmmac_napi_add(ndev); 8063 8064 mutex_init(&priv->lock); 8065 rwlock_init(&priv->ptp_lock); 8066 8067 stmmac_fpe_init(priv); 8068 8069 stmmac_check_pcs_mode(priv); 8070 8071 pm_runtime_get_noresume(device); 8072 pm_runtime_set_active(device); 8073 if (!pm_runtime_enabled(device)) 8074 pm_runtime_enable(device); 8075 8076 ret = stmmac_mdio_register(ndev); 8077 if (ret < 0) { 8078 dev_err_probe(priv->device, ret, 8079 "MDIO bus (id: %d) registration failed\n", 8080 priv->plat->bus_id); 8081 goto error_mdio_register; 8082 } 8083 8084 ret = stmmac_pcs_setup(ndev); 8085 if (ret) 8086 goto error_pcs_setup; 8087 8088 ret = stmmac_phylink_setup(priv); 8089 if (ret) { 8090 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 8091 goto error_phy_setup; 8092 } 8093 8094 ret = stmmac_register_devlink(priv); 8095 if (ret) 8096 goto error_devlink_setup; 8097 8098 ret = register_netdev(ndev); 8099 if (ret) { 8100 dev_err(priv->device, "%s: ERROR %i registering the device\n", 8101 __func__, ret); 8102 goto error_netdev_register; 8103 } 8104 8105 #ifdef CONFIG_DEBUG_FS 8106 stmmac_init_fs(ndev); 8107 #endif 8108 8109 if (priv->plat->dump_debug_regs) 8110 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 8111 8112 /* Let pm_runtime_put() disable the clocks. 8113 * If CONFIG_PM is not enabled, the clocks will stay powered. 8114 */ 8115 pm_runtime_put(device); 8116 8117 return ret; 8118 8119 error_netdev_register: 8120 stmmac_unregister_devlink(priv); 8121 error_devlink_setup: 8122 phylink_destroy(priv->phylink); 8123 error_phy_setup: 8124 stmmac_pcs_clean(ndev); 8125 error_pcs_setup: 8126 stmmac_mdio_unregister(ndev); 8127 error_mdio_register: 8128 stmmac_napi_del(ndev); 8129 error_hw_init: 8130 destroy_workqueue(priv->wq); 8131 error_wq_init: 8132 bitmap_free(priv->af_xdp_zc_qps); 8133 8134 return ret; 8135 } 8136 8137 /** 8138 * stmmac_dvr_probe 8139 * @dev: device pointer 8140 * @plat_dat: platform data pointer 8141 * @res: stmmac resource pointer 8142 * Description: this is the main probe function used to 8143 * call the alloc_etherdev, allocate the priv structure. 8144 * Return: 8145 * returns 0 on success, otherwise errno. 8146 */ 8147 int stmmac_dvr_probe(struct device *dev, struct plat_stmmacenet_data *plat_dat, 8148 struct stmmac_resources *res) 8149 { 8150 int ret; 8151 8152 if (plat_dat->init) { 8153 ret = plat_dat->init(dev, plat_dat->bsp_priv); 8154 if (ret) 8155 return ret; 8156 } 8157 8158 ret = __stmmac_dvr_probe(dev, plat_dat, res); 8159 if (ret && plat_dat->exit) 8160 plat_dat->exit(dev, plat_dat->bsp_priv); 8161 8162 return ret; 8163 } 8164 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 8165 8166 /** 8167 * stmmac_dvr_remove 8168 * @dev: device pointer 8169 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 8170 * changes the link status, releases the DMA descriptor rings. 8171 */ 8172 void stmmac_dvr_remove(struct device *dev) 8173 { 8174 struct net_device *ndev = dev_get_drvdata(dev); 8175 struct stmmac_priv *priv = netdev_priv(ndev); 8176 8177 netdev_info(priv->dev, "%s: removing driver", __func__); 8178 8179 pm_runtime_get_sync(dev); 8180 8181 unregister_netdev(ndev); 8182 8183 #ifdef CONFIG_DEBUG_FS 8184 stmmac_exit_fs(ndev); 8185 #endif 8186 stmmac_unregister_devlink(priv); 8187 8188 phylink_destroy(priv->phylink); 8189 if (priv->plat->stmmac_rst) 8190 reset_control_assert(priv->plat->stmmac_rst); 8191 reset_control_assert(priv->plat->stmmac_ahb_rst); 8192 8193 stmmac_pcs_clean(ndev); 8194 stmmac_mdio_unregister(ndev); 8195 8196 destroy_workqueue(priv->wq); 8197 mutex_destroy(&priv->lock); 8198 bitmap_free(priv->af_xdp_zc_qps); 8199 8200 pm_runtime_disable(dev); 8201 pm_runtime_put_noidle(dev); 8202 8203 if (priv->plat->exit) 8204 priv->plat->exit(dev, priv->plat->bsp_priv); 8205 } 8206 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 8207 8208 /** 8209 * stmmac_suspend - suspend callback 8210 * @dev: device pointer 8211 * Description: this is the function to suspend the device and it is called 8212 * by the platform driver to stop the network queue, release the resources, 8213 * program the PMT register (for WoL), clean and release driver resources. 8214 */ 8215 int stmmac_suspend(struct device *dev) 8216 { 8217 struct net_device *ndev = dev_get_drvdata(dev); 8218 struct stmmac_priv *priv = netdev_priv(ndev); 8219 u8 chan; 8220 8221 if (!ndev || !netif_running(ndev)) 8222 goto suspend_bsp; 8223 8224 mutex_lock(&priv->lock); 8225 8226 netif_device_detach(ndev); 8227 8228 stmmac_disable_all_queues(priv); 8229 8230 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 8231 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 8232 8233 if (priv->eee_sw_timer_en) { 8234 priv->tx_path_in_lpi_mode = false; 8235 timer_delete_sync(&priv->eee_ctrl_timer); 8236 } 8237 8238 /* Stop TX/RX DMA */ 8239 stmmac_stop_all_dma(priv); 8240 8241 stmmac_legacy_serdes_power_down(priv); 8242 8243 /* Enable Power down mode by programming the PMT regs */ 8244 if (priv->wolopts) { 8245 stmmac_pmt(priv, priv->hw, priv->wolopts); 8246 priv->irq_wake = 1; 8247 } else { 8248 stmmac_mac_set(priv, priv->ioaddr, false); 8249 pinctrl_pm_select_sleep_state(priv->device); 8250 } 8251 8252 mutex_unlock(&priv->lock); 8253 8254 rtnl_lock(); 8255 phylink_suspend(priv->phylink, !!priv->wolopts); 8256 rtnl_unlock(); 8257 8258 if (stmmac_fpe_supported(priv)) 8259 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 8260 8261 suspend_bsp: 8262 if (priv->plat->suspend) 8263 return priv->plat->suspend(dev, priv->plat->bsp_priv); 8264 8265 return 0; 8266 } 8267 EXPORT_SYMBOL_GPL(stmmac_suspend); 8268 8269 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 8270 { 8271 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 8272 8273 rx_q->cur_rx = 0; 8274 rx_q->dirty_rx = 0; 8275 } 8276 8277 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 8278 { 8279 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 8280 8281 tx_q->cur_tx = 0; 8282 tx_q->dirty_tx = 0; 8283 tx_q->mss = 0; 8284 8285 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 8286 } 8287 8288 /** 8289 * stmmac_reset_queues_param - reset queue parameters 8290 * @priv: device pointer 8291 */ 8292 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 8293 { 8294 u8 rx_cnt = priv->plat->rx_queues_to_use; 8295 u8 tx_cnt = priv->plat->tx_queues_to_use; 8296 u8 queue; 8297 8298 for (queue = 0; queue < rx_cnt; queue++) 8299 stmmac_reset_rx_queue(priv, queue); 8300 8301 for (queue = 0; queue < tx_cnt; queue++) 8302 stmmac_reset_tx_queue(priv, queue); 8303 } 8304 8305 /** 8306 * stmmac_resume - resume callback 8307 * @dev: device pointer 8308 * Description: when resume this function is invoked to setup the DMA and CORE 8309 * in a usable state. 8310 */ 8311 int stmmac_resume(struct device *dev) 8312 { 8313 struct net_device *ndev = dev_get_drvdata(dev); 8314 struct stmmac_priv *priv = netdev_priv(ndev); 8315 int ret; 8316 8317 if (priv->plat->resume) { 8318 ret = priv->plat->resume(dev, priv->plat->bsp_priv); 8319 if (ret) 8320 return ret; 8321 } 8322 8323 if (!netif_running(ndev)) 8324 return 0; 8325 8326 /* Power Down bit, into the PM register, is cleared 8327 * automatically as soon as a magic packet or a Wake-up frame 8328 * is received. Anyway, it's better to manually clear 8329 * this bit because it can generate problems while resuming 8330 * from another devices (e.g. serial console). 8331 */ 8332 if (priv->wolopts) { 8333 mutex_lock(&priv->lock); 8334 stmmac_pmt(priv, priv->hw, 0); 8335 mutex_unlock(&priv->lock); 8336 priv->irq_wake = 0; 8337 } else { 8338 pinctrl_pm_select_default_state(priv->device); 8339 /* reset the phy so that it's ready */ 8340 if (priv->mii) 8341 stmmac_mdio_reset(priv->mii); 8342 } 8343 8344 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 8345 ret = stmmac_legacy_serdes_power_up(priv); 8346 if (ret < 0) 8347 return ret; 8348 } 8349 8350 rtnl_lock(); 8351 8352 /* Prepare the PHY to resume, ensuring that its clocks which are 8353 * necessary for the MAC DMA reset to complete are running 8354 */ 8355 phylink_prepare_resume(priv->phylink); 8356 8357 mutex_lock(&priv->lock); 8358 8359 stmmac_reset_queues_param(priv); 8360 8361 stmmac_free_tx_skbufs(priv); 8362 stmmac_clear_descriptors(priv, &priv->dma_conf); 8363 8364 ret = stmmac_hw_setup(ndev); 8365 if (ret < 0) { 8366 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 8367 stmmac_legacy_serdes_power_down(priv); 8368 mutex_unlock(&priv->lock); 8369 rtnl_unlock(); 8370 return ret; 8371 } 8372 8373 stmmac_init_timestamping(priv); 8374 8375 stmmac_init_coalesce(priv); 8376 phylink_rx_clk_stop_block(priv->phylink); 8377 stmmac_set_rx_mode(ndev); 8378 phylink_rx_clk_stop_unblock(priv->phylink); 8379 8380 stmmac_vlan_restore(priv); 8381 8382 stmmac_enable_all_queues(priv); 8383 stmmac_enable_all_dma_irq(priv); 8384 8385 mutex_unlock(&priv->lock); 8386 8387 /* phylink_resume() must be called after the hardware has been 8388 * initialised because it may bring the link up immediately in a 8389 * workqueue thread, which will race with initialisation. 8390 */ 8391 phylink_resume(priv->phylink); 8392 rtnl_unlock(); 8393 8394 netif_device_attach(ndev); 8395 8396 return 0; 8397 } 8398 EXPORT_SYMBOL_GPL(stmmac_resume); 8399 8400 /* This is not the same as EXPORT_GPL_SIMPLE_DEV_PM_OPS() when CONFIG_PM=n */ 8401 DEFINE_SIMPLE_DEV_PM_OPS(stmmac_simple_pm_ops, stmmac_suspend, stmmac_resume); 8402 EXPORT_SYMBOL_GPL(stmmac_simple_pm_ops); 8403 8404 #ifndef MODULE 8405 static int __init stmmac_cmdline_opt(char *str) 8406 { 8407 char *opt; 8408 8409 if (!str || !*str) 8410 return 1; 8411 while ((opt = strsep(&str, ",")) != NULL) { 8412 if (!strncmp(opt, "debug:", 6)) { 8413 if (kstrtoint(opt + 6, 0, &debug)) 8414 goto err; 8415 } else if (!strncmp(opt, "phyaddr:", 8)) { 8416 if (kstrtoint(opt + 8, 0, &phyaddr)) 8417 goto err; 8418 } else if (!strncmp(opt, "tc:", 3)) { 8419 if (kstrtoint(opt + 3, 0, &tc)) 8420 goto err; 8421 } else if (!strncmp(opt, "watchdog:", 9)) { 8422 if (kstrtoint(opt + 9, 0, &watchdog)) 8423 goto err; 8424 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 8425 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 8426 goto err; 8427 } else if (!strncmp(opt, "pause:", 6)) { 8428 if (kstrtoint(opt + 6, 0, &pause)) 8429 goto err; 8430 } else if (!strncmp(opt, "eee_timer:", 10)) { 8431 if (kstrtoint(opt + 10, 0, &eee_timer)) 8432 goto err; 8433 } else if (!strncmp(opt, "chain_mode:", 11)) { 8434 if (kstrtoint(opt + 11, 0, &chain_mode)) 8435 goto err; 8436 } 8437 } 8438 return 1; 8439 8440 err: 8441 pr_err("%s: ERROR broken module parameter conversion", __func__); 8442 return 1; 8443 } 8444 8445 __setup("stmmaceth=", stmmac_cmdline_opt); 8446 #endif /* MODULE */ 8447 8448 static int __init stmmac_init(void) 8449 { 8450 #ifdef CONFIG_DEBUG_FS 8451 /* Create debugfs main directory if it doesn't exist yet */ 8452 if (!stmmac_fs_dir) 8453 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 8454 register_netdevice_notifier(&stmmac_notifier); 8455 #endif 8456 8457 return 0; 8458 } 8459 8460 static void __exit stmmac_exit(void) 8461 { 8462 #ifdef CONFIG_DEBUG_FS 8463 unregister_netdevice_notifier(&stmmac_notifier); 8464 debugfs_remove_recursive(stmmac_fs_dir); 8465 #endif 8466 } 8467 8468 module_init(stmmac_init) 8469 module_exit(stmmac_exit) 8470 8471 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 8472 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 8473 MODULE_LICENSE("GPL"); 8474