1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/circ_buf.h> 18 #include <linux/clk.h> 19 #include <linux/kernel.h> 20 #include <linux/interrupt.h> 21 #include <linux/ip.h> 22 #include <linux/tcp.h> 23 #include <linux/skbuff.h> 24 #include <linux/ethtool.h> 25 #include <linux/if_ether.h> 26 #include <linux/crc32.h> 27 #include <linux/mii.h> 28 #include <linux/if.h> 29 #include <linux/if_vlan.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/slab.h> 32 #include <linux/pm_runtime.h> 33 #include <linux/pm_wakeirq.h> 34 #include <linux/prefetch.h> 35 #include <linux/pinctrl/consumer.h> 36 #ifdef CONFIG_DEBUG_FS 37 #include <linux/debugfs.h> 38 #include <linux/seq_file.h> 39 #endif /* CONFIG_DEBUG_FS */ 40 #include <linux/net_tstamp.h> 41 #include <linux/phylink.h> 42 #include <linux/udp.h> 43 #include <linux/bpf_trace.h> 44 #include <net/devlink.h> 45 #include <net/page_pool/helpers.h> 46 #include <net/pkt_cls.h> 47 #include <net/xdp_sock_drv.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac_fpe.h" 50 #include "stmmac.h" 51 #include "stmmac_pcs.h" 52 #include "stmmac_xdp.h" 53 #include <linux/reset.h> 54 #include <linux/of_mdio.h> 55 #include "dwmac1000.h" 56 #include "dwxgmac2.h" 57 #include "hwif.h" 58 59 /* As long as the interface is active, we keep the timestamping counter enabled 60 * with fine resolution and binary rollover. This avoid non-monotonic behavior 61 * (clock jumps) when changing timestamping settings at runtime. 62 */ 63 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCTRLSSR) 64 65 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 66 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 67 68 /* Module parameters */ 69 #define TX_TIMEO 5000 70 static int watchdog = TX_TIMEO; 71 module_param(watchdog, int, 0644); 72 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 73 74 static int debug = -1; 75 module_param(debug, int, 0644); 76 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 77 78 static int phyaddr = -1; 79 module_param(phyaddr, int, 0444); 80 MODULE_PARM_DESC(phyaddr, "Physical device address"); 81 82 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 83 84 /* Limit to make sure XDP TX and slow path can coexist */ 85 #define STMMAC_XSK_TX_BUDGET_MAX 256 86 #define STMMAC_TX_XSK_AVAIL 16 87 #define STMMAC_RX_FILL_BATCH 16 88 89 #define STMMAC_XDP_PASS 0 90 #define STMMAC_XDP_CONSUMED BIT(0) 91 #define STMMAC_XDP_TX BIT(1) 92 #define STMMAC_XDP_REDIRECT BIT(2) 93 #define STMMAC_XSK_CONSUMED BIT(3) 94 95 static int flow_ctrl = 0xdead; 96 module_param(flow_ctrl, int, 0644); 97 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)"); 98 99 static int pause = PAUSE_TIME; 100 module_param(pause, int, 0644); 101 MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)"); 102 103 #define TC_DEFAULT 64 104 static int tc = TC_DEFAULT; 105 module_param(tc, int, 0644); 106 MODULE_PARM_DESC(tc, "DMA threshold control value"); 107 108 /* This is unused */ 109 #define DEFAULT_BUFSIZE 1536 110 static int buf_sz = DEFAULT_BUFSIZE; 111 module_param(buf_sz, int, 0644); 112 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 113 114 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 115 NETIF_MSG_LINK | NETIF_MSG_IFUP | 116 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 117 118 #define STMMAC_DEFAULT_LPI_TIMER 1000 119 static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 120 module_param(eee_timer, uint, 0644); 121 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 122 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 123 124 /* By default the driver will use the ring mode to manage tx and rx descriptors, 125 * but allow user to force to use the chain instead of the ring 126 */ 127 static unsigned int chain_mode; 128 module_param(chain_mode, int, 0444); 129 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 130 131 static const char *stmmac_dwmac_actphyif[8] = { 132 [PHY_INTF_SEL_GMII_MII] = "GMII/MII", 133 [PHY_INTF_SEL_RGMII] = "RGMII", 134 [PHY_INTF_SEL_SGMII] = "SGMII", 135 [PHY_INTF_SEL_TBI] = "TBI", 136 [PHY_INTF_SEL_RMII] = "RMII", 137 [PHY_INTF_SEL_RTBI] = "RTBI", 138 [PHY_INTF_SEL_SMII] = "SMII", 139 [PHY_INTF_SEL_REVMII] = "REVMII", 140 }; 141 142 static const char *stmmac_dwxgmac_phyif[4] = { 143 [PHY_INTF_GMII] = "GMII", 144 [PHY_INTF_RGMII] = "RGMII", 145 }; 146 147 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 148 /* For MSI interrupts handling */ 149 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 150 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 151 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 152 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 153 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 154 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 155 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 156 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 157 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 158 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 159 u32 rxmode, u32 chan); 160 static void stmmac_vlan_restore(struct stmmac_priv *priv); 161 162 #ifdef CONFIG_DEBUG_FS 163 static const struct net_device_ops stmmac_netdev_ops; 164 static void stmmac_init_fs(struct net_device *dev); 165 static void stmmac_exit_fs(struct net_device *dev); 166 #endif 167 168 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 169 170 struct stmmac_devlink_priv { 171 struct stmmac_priv *stmmac_priv; 172 }; 173 174 enum stmmac_dl_param_id { 175 STMMAC_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, 176 STMMAC_DEVLINK_PARAM_ID_TS_COARSE, 177 }; 178 179 /** 180 * stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock 181 * @bsp_priv: BSP private data structure (unused) 182 * @clk_tx_i: the transmit clock 183 * @interface: the selected interface mode 184 * @speed: the speed that the MAC will be operating at 185 * 186 * Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps, 187 * 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least 188 * MII, GMII, RGMII and RMII interface modes. Platforms can hook this into 189 * the plat_data->set_clk_tx_rate method directly, call it via their own 190 * implementation, or implement their own method should they have more 191 * complex requirements. It is intended to only be used in this method. 192 * 193 * plat_data->clk_tx_i must be filled in. 194 */ 195 int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, 196 phy_interface_t interface, int speed) 197 { 198 long rate = rgmii_clock(speed); 199 200 /* Silently ignore unsupported speeds as rgmii_clock() only 201 * supports 10, 100 and 1000Mbps. We do not want to spit 202 * errors for 2500 and higher speeds here. 203 */ 204 if (rate < 0) 205 return 0; 206 207 return clk_set_rate(clk_tx_i, rate); 208 } 209 EXPORT_SYMBOL_GPL(stmmac_set_clk_tx_rate); 210 211 /** 212 * stmmac_axi_blen_to_mask() - convert a burst length array to reg value 213 * @regval: pointer to a u32 for the resulting register value 214 * @blen: pointer to an array of u32 containing the burst length values in bytes 215 * @len: the number of entries in the @blen array 216 */ 217 void stmmac_axi_blen_to_mask(u32 *regval, const u32 *blen, size_t len) 218 { 219 size_t i; 220 u32 val; 221 222 for (val = i = 0; i < len; i++) { 223 u32 burst = blen[i]; 224 225 /* Burst values of zero must be skipped. */ 226 if (!burst) 227 continue; 228 229 /* The valid range for the burst length is 4 to 256 inclusive, 230 * and it must be a power of two. 231 */ 232 if (burst < 4 || burst > 256 || !is_power_of_2(burst)) { 233 pr_err("stmmac: invalid burst length %u at index %zu\n", 234 burst, i); 235 continue; 236 } 237 238 /* Since burst is a power of two, and the register field starts 239 * with burst = 4, shift right by two bits so bit 0 of the field 240 * corresponds with the minimum value. 241 */ 242 val |= burst >> 2; 243 } 244 245 *regval = FIELD_PREP(DMA_AXI_BLEN_MASK, val); 246 } 247 EXPORT_SYMBOL_GPL(stmmac_axi_blen_to_mask); 248 249 /** 250 * stmmac_verify_args - verify the driver parameters. 251 * Description: it checks the driver parameters and set a default in case of 252 * errors. 253 */ 254 static void stmmac_verify_args(void) 255 { 256 if (unlikely(watchdog < 0)) 257 watchdog = TX_TIMEO; 258 if (unlikely((pause < 0) || (pause > 0xffff))) 259 pause = PAUSE_TIME; 260 261 if (flow_ctrl != 0xdead) 262 pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n"); 263 } 264 265 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 266 { 267 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 268 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 269 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 270 u8 queue; 271 272 for (queue = 0; queue < maxq; queue++) { 273 struct stmmac_channel *ch = &priv->channel[queue]; 274 275 if (stmmac_xdp_is_enabled(priv) && 276 test_bit(queue, priv->af_xdp_zc_qps)) { 277 napi_disable(&ch->rxtx_napi); 278 continue; 279 } 280 281 if (queue < rx_queues_cnt) 282 napi_disable(&ch->rx_napi); 283 if (queue < tx_queues_cnt) 284 napi_disable(&ch->tx_napi); 285 } 286 } 287 288 /** 289 * stmmac_disable_all_queues - Disable all queues 290 * @priv: driver private structure 291 */ 292 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 293 { 294 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 295 struct stmmac_rx_queue *rx_q; 296 u8 queue; 297 298 /* synchronize_rcu() needed for pending XDP buffers to drain */ 299 for (queue = 0; queue < rx_queues_cnt; queue++) { 300 rx_q = &priv->dma_conf.rx_queue[queue]; 301 if (rx_q->xsk_pool) { 302 synchronize_rcu(); 303 break; 304 } 305 } 306 307 __stmmac_disable_all_queues(priv); 308 } 309 310 /** 311 * stmmac_enable_all_queues - Enable all queues 312 * @priv: driver private structure 313 */ 314 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 315 { 316 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 317 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 318 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 319 u8 queue; 320 321 for (queue = 0; queue < maxq; queue++) { 322 struct stmmac_channel *ch = &priv->channel[queue]; 323 324 if (stmmac_xdp_is_enabled(priv) && 325 test_bit(queue, priv->af_xdp_zc_qps)) { 326 napi_enable(&ch->rxtx_napi); 327 continue; 328 } 329 330 if (queue < rx_queues_cnt) 331 napi_enable(&ch->rx_napi); 332 if (queue < tx_queues_cnt) 333 napi_enable(&ch->tx_napi); 334 } 335 } 336 337 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 338 { 339 if (!test_bit(STMMAC_DOWN, &priv->state) && 340 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 341 queue_work(priv->wq, &priv->service_task); 342 } 343 344 static void stmmac_global_err(struct stmmac_priv *priv) 345 { 346 netif_carrier_off(priv->dev); 347 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 348 stmmac_service_event_schedule(priv); 349 } 350 351 static void print_pkt(unsigned char *buf, int len) 352 { 353 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 354 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 355 } 356 357 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 358 { 359 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 360 361 return CIRC_SPACE(tx_q->cur_tx, tx_q->dirty_tx, 362 priv->dma_conf.dma_tx_size); 363 } 364 365 static size_t stmmac_get_tx_desc_size(struct stmmac_priv *priv, 366 struct stmmac_tx_queue *tx_q) 367 { 368 if (priv->extend_desc) 369 return sizeof(struct dma_extended_desc); 370 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 371 return sizeof(struct dma_edesc); 372 else 373 return sizeof(struct dma_desc); 374 } 375 376 static struct dma_desc *stmmac_get_tx_desc(struct stmmac_priv *priv, 377 struct stmmac_tx_queue *tx_q, 378 unsigned int index) 379 { 380 if (priv->extend_desc) 381 return &tx_q->dma_etx[index].basic; 382 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 383 return &tx_q->dma_entx[index].basic; 384 else 385 return &tx_q->dma_tx[index]; 386 } 387 388 static void stmmac_set_queue_tx_tail_ptr(struct stmmac_priv *priv, 389 struct stmmac_tx_queue *tx_q, 390 unsigned int chan, unsigned int index) 391 { 392 size_t desc_size; 393 u32 tx_tail_addr; 394 395 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 396 397 tx_tail_addr = tx_q->dma_tx_phy + index * desc_size; 398 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_tail_addr, chan); 399 } 400 401 static size_t stmmac_get_rx_desc_size(struct stmmac_priv *priv) 402 { 403 if (priv->extend_desc) 404 return sizeof(struct dma_extended_desc); 405 else 406 return sizeof(struct dma_desc); 407 } 408 409 static struct dma_desc *stmmac_get_rx_desc(struct stmmac_priv *priv, 410 struct stmmac_rx_queue *rx_q, 411 unsigned int index) 412 { 413 if (priv->extend_desc) 414 return &rx_q->dma_erx[index].basic; 415 else 416 return &rx_q->dma_rx[index]; 417 } 418 419 static void stmmac_set_queue_rx_tail_ptr(struct stmmac_priv *priv, 420 struct stmmac_rx_queue *rx_q, 421 unsigned int chan, unsigned int index) 422 { 423 /* This only needs to deal with normal descriptors as enhanced 424 * descriptiors are only supported with dwmac1000 (<v4.0) which 425 * does not implement .set_rx_tail_ptr 426 */ 427 u32 rx_tail_addr = rx_q->dma_rx_phy + index * sizeof(struct dma_desc); 428 429 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_tail_addr, chan); 430 } 431 432 static void stmmac_set_queue_rx_buf_size(struct stmmac_priv *priv, 433 struct stmmac_rx_queue *rx_q, 434 unsigned int chan) 435 { 436 u32 buf_size; 437 438 if (rx_q->xsk_pool && rx_q->buf_alloc_num) 439 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 440 else 441 buf_size = priv->dma_conf.dma_buf_sz; 442 443 stmmac_set_dma_bfsize(priv, priv->ioaddr, buf_size, chan); 444 } 445 446 /** 447 * stmmac_rx_dirty - Get RX queue dirty 448 * @priv: driver private structure 449 * @queue: RX queue index 450 */ 451 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 452 { 453 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 454 455 return CIRC_CNT(rx_q->cur_rx, rx_q->dirty_rx, 456 priv->dma_conf.dma_rx_size); 457 } 458 459 static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) 460 { 461 u8 tx_cnt = priv->plat->tx_queues_to_use; 462 u8 queue; 463 464 /* check if all TX queues have the work finished */ 465 for (queue = 0; queue < tx_cnt; queue++) { 466 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 467 468 if (tx_q->dirty_tx != tx_q->cur_tx) 469 return true; /* still unfinished work */ 470 } 471 472 return false; 473 } 474 475 static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv) 476 { 477 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 478 } 479 480 /** 481 * stmmac_try_to_start_sw_lpi - check and enter in LPI mode 482 * @priv: driver private structure 483 * Description: this function is to verify and enter in LPI mode in case of 484 * EEE. 485 */ 486 static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv) 487 { 488 if (stmmac_eee_tx_busy(priv)) { 489 stmmac_restart_sw_lpi_timer(priv); 490 return; 491 } 492 493 /* Check and enter in LPI mode */ 494 if (!priv->tx_path_in_lpi_mode) 495 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED, 496 priv->tx_lpi_clk_stop, 0); 497 } 498 499 /** 500 * stmmac_stop_sw_lpi - stop transmitting LPI 501 * @priv: driver private structure 502 * Description: When using software-controlled LPI, stop transmitting LPI state. 503 */ 504 static void stmmac_stop_sw_lpi(struct stmmac_priv *priv) 505 { 506 timer_delete_sync(&priv->eee_ctrl_timer); 507 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 508 priv->tx_path_in_lpi_mode = false; 509 } 510 511 /** 512 * stmmac_eee_ctrl_timer - EEE TX SW timer. 513 * @t: timer_list struct containing private info 514 * Description: 515 * if there is no data transfer and if we are not in LPI state, 516 * then MAC Transmitter can be moved to LPI state. 517 */ 518 static void stmmac_eee_ctrl_timer(struct timer_list *t) 519 { 520 struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer); 521 522 stmmac_try_to_start_sw_lpi(priv); 523 } 524 525 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 526 * @priv: driver private structure 527 * @p : descriptor pointer 528 * @skb : the socket buffer 529 * Description : 530 * This function will read timestamp from the descriptor & pass it to stack. 531 * and also perform some sanity checks. 532 */ 533 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 534 struct dma_desc *p, struct sk_buff *skb) 535 { 536 struct skb_shared_hwtstamps shhwtstamp; 537 bool found = false; 538 u64 ns = 0; 539 540 if (!priv->hwts_tx_en) 541 return; 542 543 /* exit if skb doesn't support hw tstamp */ 544 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 545 return; 546 547 /* check tx tstamp status */ 548 if (stmmac_get_tx_timestamp_status(priv, p)) { 549 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 550 found = true; 551 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 552 found = true; 553 } 554 555 if (found) { 556 ns -= priv->plat->cdc_error_adj; 557 558 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 559 shhwtstamp.hwtstamp = ns_to_ktime(ns); 560 561 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 562 /* pass tstamp to stack */ 563 skb_tstamp_tx(skb, &shhwtstamp); 564 } 565 } 566 567 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 568 * @priv: driver private structure 569 * @p : descriptor pointer 570 * @np : next descriptor pointer 571 * @skb : the socket buffer 572 * Description : 573 * This function will read received packet's timestamp from the descriptor 574 * and pass it to stack. It also perform some sanity checks. 575 */ 576 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 577 struct dma_desc *np, struct sk_buff *skb) 578 { 579 struct skb_shared_hwtstamps *shhwtstamp = NULL; 580 struct dma_desc *desc = p; 581 u64 ns = 0; 582 583 if (!priv->hwts_rx_en) 584 return; 585 /* For GMAC4, the valid timestamp is from CTX next desc. */ 586 if (dwmac_is_xmac(priv->plat->core_type)) 587 desc = np; 588 589 /* Check if timestamp is available */ 590 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 591 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 592 593 ns -= priv->plat->cdc_error_adj; 594 595 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 596 shhwtstamp = skb_hwtstamps(skb); 597 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 598 shhwtstamp->hwtstamp = ns_to_ktime(ns); 599 } else { 600 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 601 } 602 } 603 604 static void stmmac_update_subsecond_increment(struct stmmac_priv *priv) 605 { 606 bool xmac = dwmac_is_xmac(priv->plat->core_type); 607 u32 sec_inc = 0; 608 u64 temp = 0; 609 610 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 611 612 /* program Sub Second Increment reg */ 613 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 614 priv->plat->clk_ptp_rate, 615 xmac, &sec_inc); 616 temp = div_u64(1000000000ULL, sec_inc); 617 618 /* Store sub second increment for later use */ 619 priv->sub_second_inc = sec_inc; 620 621 /* calculate default added value: 622 * formula is : 623 * addend = (2^32)/freq_div_ratio; 624 * where, freq_div_ratio = 1e9ns/sec_inc 625 */ 626 temp = (u64)(temp << 32); 627 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 628 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 629 } 630 631 /** 632 * stmmac_hwtstamp_set - control hardware timestamping. 633 * @dev: device pointer. 634 * @config: the timestamping configuration. 635 * @extack: netlink extended ack structure for error reporting. 636 * Description: 637 * This function configures the MAC to enable/disable both outgoing(TX) 638 * and incoming(RX) packets time stamping based on user input. 639 * Return Value: 640 * 0 on success and an appropriate -ve integer on failure. 641 */ 642 static int stmmac_hwtstamp_set(struct net_device *dev, 643 struct kernel_hwtstamp_config *config, 644 struct netlink_ext_ack *extack) 645 { 646 struct stmmac_priv *priv = netdev_priv(dev); 647 u32 ptp_v2 = 0; 648 u32 tstamp_all = 0; 649 u32 ptp_over_ipv4_udp = 0; 650 u32 ptp_over_ipv6_udp = 0; 651 u32 ptp_over_ethernet = 0; 652 u32 snap_type_sel = 0; 653 u32 ts_master_en = 0; 654 u32 ts_event_en = 0; 655 656 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 657 NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping"); 658 priv->hwts_tx_en = 0; 659 priv->hwts_rx_en = 0; 660 661 return -EOPNOTSUPP; 662 } 663 664 if (!netif_running(dev)) { 665 NL_SET_ERR_MSG_MOD(extack, 666 "Cannot change timestamping configuration while down"); 667 return -ENODEV; 668 } 669 670 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 671 __func__, config->flags, config->tx_type, config->rx_filter); 672 673 if (config->tx_type != HWTSTAMP_TX_OFF && 674 config->tx_type != HWTSTAMP_TX_ON) 675 return -ERANGE; 676 677 if (priv->adv_ts) { 678 switch (config->rx_filter) { 679 case HWTSTAMP_FILTER_NONE: 680 /* time stamp no incoming packet at all */ 681 config->rx_filter = HWTSTAMP_FILTER_NONE; 682 break; 683 684 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 685 /* PTP v1, UDP, any kind of event packet */ 686 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 687 /* 'xmac' hardware can support Sync, Pdelay_Req and 688 * Pdelay_resp by setting bit14 and bits17/16 to 01 689 * This leaves Delay_Req timestamps out. 690 * Enable all events *and* general purpose message 691 * timestamping 692 */ 693 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 694 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 695 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 696 break; 697 698 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 699 /* PTP v1, UDP, Sync packet */ 700 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 701 /* take time stamp for SYNC messages only */ 702 ts_event_en = PTP_TCR_TSEVNTENA; 703 704 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 705 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 706 break; 707 708 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 709 /* PTP v1, UDP, Delay_req packet */ 710 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 711 /* take time stamp for Delay_Req messages only */ 712 ts_master_en = PTP_TCR_TSMSTRENA; 713 ts_event_en = PTP_TCR_TSEVNTENA; 714 715 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 716 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 717 break; 718 719 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 720 /* PTP v2, UDP, any kind of event packet */ 721 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 722 ptp_v2 = PTP_TCR_TSVER2ENA; 723 /* take time stamp for all event messages */ 724 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 725 726 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 727 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 728 break; 729 730 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 731 /* PTP v2, UDP, Sync packet */ 732 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 733 ptp_v2 = PTP_TCR_TSVER2ENA; 734 /* take time stamp for SYNC messages only */ 735 ts_event_en = PTP_TCR_TSEVNTENA; 736 737 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 738 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 739 break; 740 741 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 742 /* PTP v2, UDP, Delay_req packet */ 743 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 744 ptp_v2 = PTP_TCR_TSVER2ENA; 745 /* take time stamp for Delay_Req messages only */ 746 ts_master_en = PTP_TCR_TSMSTRENA; 747 ts_event_en = PTP_TCR_TSEVNTENA; 748 749 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 750 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 751 break; 752 753 case HWTSTAMP_FILTER_PTP_V2_EVENT: 754 /* PTP v2/802.AS1 any layer, any kind of event packet */ 755 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 756 ptp_v2 = PTP_TCR_TSVER2ENA; 757 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 758 if (priv->synopsys_id < DWMAC_CORE_3_70 && 759 priv->plat->core_type != DWMAC_CORE_XGMAC) 760 ts_event_en = PTP_TCR_TSEVNTENA; 761 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 762 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 763 ptp_over_ethernet = PTP_TCR_TSIPENA; 764 break; 765 766 case HWTSTAMP_FILTER_PTP_V2_SYNC: 767 /* PTP v2/802.AS1, any layer, Sync packet */ 768 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 769 ptp_v2 = PTP_TCR_TSVER2ENA; 770 /* take time stamp for SYNC messages only */ 771 ts_event_en = PTP_TCR_TSEVNTENA; 772 773 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 774 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 775 ptp_over_ethernet = PTP_TCR_TSIPENA; 776 break; 777 778 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 779 /* PTP v2/802.AS1, any layer, Delay_req packet */ 780 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 781 ptp_v2 = PTP_TCR_TSVER2ENA; 782 /* take time stamp for Delay_Req messages only */ 783 ts_master_en = PTP_TCR_TSMSTRENA; 784 ts_event_en = PTP_TCR_TSEVNTENA; 785 786 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 787 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 788 ptp_over_ethernet = PTP_TCR_TSIPENA; 789 break; 790 791 case HWTSTAMP_FILTER_NTP_ALL: 792 case HWTSTAMP_FILTER_ALL: 793 /* time stamp any incoming packet */ 794 config->rx_filter = HWTSTAMP_FILTER_ALL; 795 tstamp_all = PTP_TCR_TSENALL; 796 break; 797 798 default: 799 return -ERANGE; 800 } 801 } else { 802 switch (config->rx_filter) { 803 case HWTSTAMP_FILTER_NONE: 804 config->rx_filter = HWTSTAMP_FILTER_NONE; 805 break; 806 default: 807 /* PTP v1, UDP, any kind of event packet */ 808 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 809 break; 810 } 811 } 812 priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE; 813 priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON; 814 815 priv->systime_flags = STMMAC_HWTS_ACTIVE; 816 if (!priv->tsfupdt_coarse) 817 priv->systime_flags |= PTP_TCR_TSCFUPDT; 818 819 if (priv->hwts_tx_en || priv->hwts_rx_en) { 820 priv->systime_flags |= tstamp_all | ptp_v2 | 821 ptp_over_ethernet | ptp_over_ipv6_udp | 822 ptp_over_ipv4_udp | ts_event_en | 823 ts_master_en | snap_type_sel; 824 } 825 826 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 827 828 priv->tstamp_config = *config; 829 830 return 0; 831 } 832 833 /** 834 * stmmac_hwtstamp_get - read hardware timestamping. 835 * @dev: device pointer. 836 * @config: the timestamping configuration. 837 * Description: 838 * This function obtain the current hardware timestamping settings 839 * as requested. 840 */ 841 static int stmmac_hwtstamp_get(struct net_device *dev, 842 struct kernel_hwtstamp_config *config) 843 { 844 struct stmmac_priv *priv = netdev_priv(dev); 845 846 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 847 return -EOPNOTSUPP; 848 849 *config = priv->tstamp_config; 850 851 return 0; 852 } 853 854 /** 855 * stmmac_init_tstamp_counter - init hardware timestamping counter 856 * @priv: driver private structure 857 * @systime_flags: timestamping flags 858 * Description: 859 * Initialize hardware counter for packet timestamping. 860 * This is valid as long as the interface is open and not suspended. 861 * Will be rerun after resuming from suspend, case in which the timestamping 862 * flags updated by stmmac_hwtstamp_set() also need to be restored. 863 */ 864 static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, 865 u32 systime_flags) 866 { 867 struct timespec64 now; 868 869 if (!priv->plat->clk_ptp_rate) { 870 netdev_err(priv->dev, "Invalid PTP clock rate"); 871 return -EINVAL; 872 } 873 874 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 875 priv->systime_flags = systime_flags; 876 877 stmmac_update_subsecond_increment(priv); 878 879 /* initialize system time */ 880 ktime_get_real_ts64(&now); 881 882 /* lower 32 bits of tv_sec are safe until y2106 */ 883 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 884 885 return 0; 886 } 887 888 /** 889 * stmmac_init_timestamping - initialise timestamping 890 * @priv: driver private structure 891 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 892 * This is done by looking at the HW cap. register. 893 * This function also registers the ptp driver. 894 */ 895 static int stmmac_init_timestamping(struct stmmac_priv *priv) 896 { 897 bool xmac = dwmac_is_xmac(priv->plat->core_type); 898 int ret; 899 900 if (priv->plat->ptp_clk_freq_config) 901 priv->plat->ptp_clk_freq_config(priv); 902 903 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) { 904 netdev_info(priv->dev, "PTP not supported by HW\n"); 905 return -EOPNOTSUPP; 906 } 907 908 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE | 909 PTP_TCR_TSCFUPDT); 910 if (ret) { 911 netdev_warn(priv->dev, "PTP init failed\n"); 912 return ret; 913 } 914 915 priv->adv_ts = 0; 916 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 917 if (xmac && priv->dma_cap.atime_stamp) 918 priv->adv_ts = 1; 919 /* Dwmac 3.x core with extend_desc can support adv_ts */ 920 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 921 priv->adv_ts = 1; 922 923 if (priv->dma_cap.time_stamp) 924 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 925 926 if (priv->adv_ts) 927 netdev_info(priv->dev, 928 "IEEE 1588-2008 Advanced Timestamp supported\n"); 929 930 memset(&priv->tstamp_config, 0, sizeof(priv->tstamp_config)); 931 priv->hwts_tx_en = 0; 932 priv->hwts_rx_en = 0; 933 934 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 935 stmmac_hwtstamp_correct_latency(priv, priv); 936 937 return 0; 938 } 939 940 static void stmmac_setup_ptp(struct stmmac_priv *priv) 941 { 942 int ret; 943 944 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 945 if (ret < 0) 946 netdev_warn(priv->dev, 947 "failed to enable PTP reference clock: %pe\n", 948 ERR_PTR(ret)); 949 950 if (stmmac_init_timestamping(priv) == 0) 951 stmmac_ptp_register(priv); 952 } 953 954 static void stmmac_release_ptp(struct stmmac_priv *priv) 955 { 956 stmmac_ptp_unregister(priv); 957 clk_disable_unprepare(priv->plat->clk_ptp_ref); 958 } 959 960 static void stmmac_legacy_serdes_power_down(struct stmmac_priv *priv) 961 { 962 if (priv->plat->serdes_powerdown && priv->legacy_serdes_is_powered) 963 priv->plat->serdes_powerdown(priv->dev, priv->plat->bsp_priv); 964 965 priv->legacy_serdes_is_powered = false; 966 } 967 968 static int stmmac_legacy_serdes_power_up(struct stmmac_priv *priv) 969 { 970 int ret; 971 972 if (!priv->plat->serdes_powerup) 973 return 0; 974 975 ret = priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 976 if (ret < 0) 977 netdev_err(priv->dev, "SerDes powerup failed\n"); 978 else 979 priv->legacy_serdes_is_powered = true; 980 981 return ret; 982 } 983 984 /** 985 * stmmac_mac_flow_ctrl - Configure flow control in all queues 986 * @priv: driver private structure 987 * @duplex: duplex passed to the next function 988 * @flow_ctrl: desired flow control modes 989 * Description: It is used for configuring the flow control in all queues 990 */ 991 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex, 992 unsigned int flow_ctrl) 993 { 994 u8 tx_cnt = priv->plat->tx_queues_to_use; 995 996 stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time, 997 tx_cnt); 998 } 999 1000 static unsigned long stmmac_mac_get_caps(struct phylink_config *config, 1001 phy_interface_t interface) 1002 { 1003 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1004 1005 /* Refresh the MAC-specific capabilities */ 1006 stmmac_mac_update_caps(priv); 1007 1008 if (priv->hw_cap_support && !priv->dma_cap.half_duplex) 1009 priv->hw->link.caps &= ~(MAC_1000HD | MAC_100HD | MAC_10HD); 1010 1011 config->mac_capabilities = priv->hw->link.caps; 1012 1013 if (priv->plat->max_speed) 1014 phylink_limit_mac_speed(config, priv->plat->max_speed); 1015 1016 return config->mac_capabilities; 1017 } 1018 1019 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 1020 phy_interface_t interface) 1021 { 1022 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1023 struct phylink_pcs *pcs; 1024 1025 if (priv->plat->select_pcs) { 1026 pcs = priv->plat->select_pcs(priv, interface); 1027 if (!IS_ERR(pcs)) 1028 return pcs; 1029 } 1030 1031 if (priv->integrated_pcs && 1032 test_bit(interface, priv->integrated_pcs->pcs.supported_interfaces)) 1033 return &priv->integrated_pcs->pcs; 1034 1035 return NULL; 1036 } 1037 1038 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 1039 const struct phylink_link_state *state) 1040 { 1041 /* Nothing to do, xpcs_config() handles everything */ 1042 } 1043 1044 static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode, 1045 phy_interface_t interface) 1046 { 1047 struct net_device *ndev = to_net_dev(config->dev); 1048 struct stmmac_priv *priv = netdev_priv(ndev); 1049 1050 if (priv->plat->mac_finish) 1051 priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, 1052 interface); 1053 1054 return 0; 1055 } 1056 1057 static void stmmac_mac_link_down(struct phylink_config *config, 1058 unsigned int mode, phy_interface_t interface) 1059 { 1060 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1061 1062 stmmac_mac_set(priv, priv->ioaddr, false); 1063 if (priv->dma_cap.eee) 1064 stmmac_set_eee_pls(priv, priv->hw, false); 1065 1066 if (stmmac_fpe_supported(priv)) 1067 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, false); 1068 } 1069 1070 static void stmmac_mac_link_up(struct phylink_config *config, 1071 struct phy_device *phy, 1072 unsigned int mode, phy_interface_t interface, 1073 int speed, int duplex, 1074 bool tx_pause, bool rx_pause) 1075 { 1076 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1077 unsigned int flow_ctrl; 1078 u32 old_ctrl, ctrl; 1079 int ret; 1080 1081 if (priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) 1082 stmmac_legacy_serdes_power_up(priv); 1083 1084 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1085 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1086 1087 switch (speed) { 1088 case SPEED_100000: 1089 ctrl |= priv->hw->link.xlgmii.speed100000; 1090 break; 1091 case SPEED_50000: 1092 ctrl |= priv->hw->link.xlgmii.speed50000; 1093 break; 1094 case SPEED_40000: 1095 ctrl |= priv->hw->link.xlgmii.speed40000; 1096 break; 1097 case SPEED_25000: 1098 ctrl |= priv->hw->link.xlgmii.speed25000; 1099 break; 1100 case SPEED_10000: 1101 ctrl |= priv->hw->link.xgmii.speed10000; 1102 break; 1103 case SPEED_5000: 1104 ctrl |= priv->hw->link.xgmii.speed5000; 1105 break; 1106 case SPEED_2500: 1107 if (interface == PHY_INTERFACE_MODE_USXGMII) 1108 ctrl |= priv->hw->link.xgmii.speed2500; 1109 else 1110 ctrl |= priv->hw->link.speed2500; 1111 break; 1112 case SPEED_1000: 1113 ctrl |= priv->hw->link.speed1000; 1114 break; 1115 case SPEED_100: 1116 ctrl |= priv->hw->link.speed100; 1117 break; 1118 case SPEED_10: 1119 ctrl |= priv->hw->link.speed10; 1120 break; 1121 default: 1122 netdev_err(priv->dev, 1123 "unsupported speed %s on %s, leaving the MAC disabled\n", 1124 phy_speed_to_str(speed), phy_modes(interface)); 1125 return; 1126 } 1127 1128 if (priv->plat->fix_mac_speed) 1129 priv->plat->fix_mac_speed(priv->plat->bsp_priv, interface, 1130 speed, mode); 1131 1132 if (!duplex) 1133 ctrl &= ~priv->hw->link.duplex; 1134 else 1135 ctrl |= priv->hw->link.duplex; 1136 1137 /* Flow Control operation */ 1138 if (rx_pause && tx_pause) 1139 flow_ctrl = FLOW_AUTO; 1140 else if (rx_pause && !tx_pause) 1141 flow_ctrl = FLOW_RX; 1142 else if (!rx_pause && tx_pause) 1143 flow_ctrl = FLOW_TX; 1144 else 1145 flow_ctrl = FLOW_OFF; 1146 1147 stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl); 1148 1149 if (ctrl != old_ctrl) 1150 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1151 1152 if (priv->plat->set_clk_tx_rate) { 1153 ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv, 1154 priv->plat->clk_tx_i, 1155 interface, speed); 1156 if (ret < 0) 1157 netdev_err(priv->dev, 1158 "failed to configure %s transmit clock for %dMbps: %pe\n", 1159 phy_modes(interface), speed, ERR_PTR(ret)); 1160 } 1161 1162 stmmac_mac_set(priv, priv->ioaddr, true); 1163 if (priv->dma_cap.eee) 1164 stmmac_set_eee_pls(priv, priv->hw, true); 1165 1166 if (stmmac_fpe_supported(priv)) 1167 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, true); 1168 1169 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1170 stmmac_hwtstamp_correct_latency(priv, priv); 1171 } 1172 1173 static void stmmac_mac_disable_tx_lpi(struct phylink_config *config) 1174 { 1175 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1176 1177 priv->eee_active = false; 1178 1179 mutex_lock(&priv->lock); 1180 1181 priv->eee_enabled = false; 1182 1183 netdev_dbg(priv->dev, "disable EEE\n"); 1184 priv->eee_sw_timer_en = false; 1185 timer_delete_sync(&priv->eee_ctrl_timer); 1186 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 1187 priv->tx_path_in_lpi_mode = false; 1188 1189 stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS); 1190 mutex_unlock(&priv->lock); 1191 } 1192 1193 static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 1194 bool tx_clk_stop) 1195 { 1196 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1197 int ret; 1198 1199 priv->tx_lpi_timer = timer; 1200 priv->eee_active = true; 1201 1202 mutex_lock(&priv->lock); 1203 1204 priv->eee_enabled = true; 1205 1206 /* Update the transmit clock stop according to PHY capability if 1207 * the platform allows 1208 */ 1209 if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP) 1210 priv->tx_lpi_clk_stop = tx_clk_stop; 1211 1212 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 1213 STMMAC_DEFAULT_TWT_LS); 1214 1215 /* Try to configure the hardware timer. */ 1216 ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER, 1217 priv->tx_lpi_clk_stop, priv->tx_lpi_timer); 1218 1219 if (ret) { 1220 /* Hardware timer mode not supported, or value out of range. 1221 * Fall back to using software LPI mode 1222 */ 1223 priv->eee_sw_timer_en = true; 1224 stmmac_restart_sw_lpi_timer(priv); 1225 } 1226 1227 mutex_unlock(&priv->lock); 1228 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 1229 1230 return 0; 1231 } 1232 1233 static int stmmac_mac_wol_set(struct phylink_config *config, u32 wolopts, 1234 const u8 *sopass) 1235 { 1236 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1237 1238 device_set_wakeup_enable(priv->device, !!wolopts); 1239 1240 mutex_lock(&priv->lock); 1241 priv->wolopts = wolopts; 1242 mutex_unlock(&priv->lock); 1243 1244 return 0; 1245 } 1246 1247 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1248 .mac_get_caps = stmmac_mac_get_caps, 1249 .mac_select_pcs = stmmac_mac_select_pcs, 1250 .mac_config = stmmac_mac_config, 1251 .mac_finish = stmmac_mac_finish, 1252 .mac_link_down = stmmac_mac_link_down, 1253 .mac_link_up = stmmac_mac_link_up, 1254 .mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi, 1255 .mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi, 1256 .mac_wol_set = stmmac_mac_wol_set, 1257 }; 1258 1259 /** 1260 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1261 * @priv: driver private structure 1262 * Description: this is to verify if the HW supports the PCS. 1263 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1264 * configured for the TBI, RTBI, or SGMII PHY interface. 1265 */ 1266 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1267 { 1268 int interface = priv->plat->phy_interface; 1269 int speed = priv->plat->mac_port_sel_speed; 1270 1271 if (priv->dma_cap.pcs && interface == PHY_INTERFACE_MODE_SGMII) { 1272 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1273 1274 switch (speed) { 1275 case SPEED_10: 1276 case SPEED_100: 1277 case SPEED_1000: 1278 priv->hw->reverse_sgmii_enable = true; 1279 break; 1280 1281 default: 1282 dev_warn(priv->device, "invalid port speed\n"); 1283 fallthrough; 1284 case 0: 1285 priv->hw->reverse_sgmii_enable = false; 1286 break; 1287 } 1288 } 1289 } 1290 1291 /** 1292 * stmmac_init_phy - PHY initialization 1293 * @dev: net device structure 1294 * Description: it initializes the driver's PHY state, and attaches the PHY 1295 * to the mac driver. 1296 * Return value: 1297 * 0 on success 1298 */ 1299 static int stmmac_init_phy(struct net_device *dev) 1300 { 1301 struct stmmac_priv *priv = netdev_priv(dev); 1302 int mode = priv->plat->phy_interface; 1303 struct fwnode_handle *phy_fwnode; 1304 struct fwnode_handle *fwnode; 1305 struct ethtool_keee eee; 1306 u32 dev_flags = 0; 1307 int ret; 1308 1309 if (!phylink_expects_phy(priv->phylink)) 1310 return 0; 1311 1312 if (priv->hw->xpcs && 1313 xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73) 1314 return 0; 1315 1316 fwnode = dev_fwnode(priv->device); 1317 if (fwnode) 1318 phy_fwnode = fwnode_get_phy_node(fwnode); 1319 else 1320 phy_fwnode = NULL; 1321 1322 if (priv->plat->flags & STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD) 1323 dev_flags |= PHY_F_KEEP_PREAMBLE_BEFORE_SFD; 1324 1325 /* Some DT bindings do not set-up the PHY handle. Let's try to 1326 * manually parse it 1327 */ 1328 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1329 int addr = priv->plat->phy_addr; 1330 struct phy_device *phydev; 1331 1332 if (addr < 0) { 1333 /* If a custom PCS is in use, no PHY is needed */ 1334 if (priv->hw->phylink_pcs) 1335 return 0; 1336 1337 netdev_err(priv->dev, "no phy found\n"); 1338 return -ENODEV; 1339 } 1340 1341 phydev = mdiobus_get_phy(priv->mii, addr); 1342 if (!phydev) { 1343 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1344 return -ENODEV; 1345 } 1346 1347 phydev->dev_flags |= dev_flags; 1348 1349 ret = phylink_connect_phy(priv->phylink, phydev); 1350 } else { 1351 fwnode_handle_put(phy_fwnode); 1352 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, dev_flags); 1353 } 1354 1355 if (ret) { 1356 netdev_err(priv->dev, "cannot attach to PHY (error: %pe)\n", 1357 ERR_PTR(ret)); 1358 return ret; 1359 } 1360 1361 /* Configure phylib's copy of the LPI timer. Normally, 1362 * phylink_config.lpi_timer_default would do this, but there is a 1363 * chance that userspace could change the eee_timer setting via sysfs 1364 * before the first open. Thus, preserve existing behaviour. 1365 */ 1366 if (!phylink_ethtool_get_eee(priv->phylink, &eee)) { 1367 eee.tx_lpi_timer = priv->tx_lpi_timer; 1368 phylink_ethtool_set_eee(priv->phylink, &eee); 1369 } 1370 1371 return 0; 1372 } 1373 1374 static int stmmac_phylink_setup(struct stmmac_priv *priv) 1375 { 1376 struct phylink_config *config; 1377 struct phylink_pcs *pcs; 1378 struct phylink *phylink; 1379 1380 config = &priv->phylink_config; 1381 1382 config->dev = &priv->dev->dev; 1383 config->type = PHYLINK_NETDEV; 1384 config->mac_managed_pm = true; 1385 1386 /* Stmmac always requires an RX clock for hardware initialization */ 1387 config->mac_requires_rxc = true; 1388 1389 /* Disable EEE RX clock stop to ensure VLAN register access works 1390 * correctly. 1391 */ 1392 if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI) && 1393 !(priv->dev->features & NETIF_F_VLAN_FEATURES)) 1394 config->eee_rx_clk_stop_enable = true; 1395 1396 /* Set the default transmit clock stop bit based on the platform glue */ 1397 priv->tx_lpi_clk_stop = priv->plat->flags & 1398 STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; 1399 1400 /* Get the PHY interface modes (at the PHY end of the link) that 1401 * are supported by the platform. 1402 */ 1403 if (priv->plat->get_interfaces) 1404 priv->plat->get_interfaces(priv, priv->plat->bsp_priv, 1405 config->supported_interfaces); 1406 1407 config->default_an_inband = priv->plat->default_an_inband; 1408 1409 /* Set the platform/firmware specified interface mode if the 1410 * supported interfaces have not already been provided using 1411 * phy_interface as a last resort. 1412 */ 1413 if (phy_interface_empty(config->supported_interfaces)) 1414 __set_bit(priv->plat->phy_interface, 1415 config->supported_interfaces); 1416 1417 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1418 if (priv->hw->xpcs) 1419 pcs = xpcs_to_phylink_pcs(priv->hw->xpcs); 1420 else 1421 pcs = priv->hw->phylink_pcs; 1422 1423 if (pcs) 1424 phy_interface_or(config->supported_interfaces, 1425 config->supported_interfaces, 1426 pcs->supported_interfaces); 1427 1428 /* Some platforms, e.g. iMX8MP, wire lpi_intr_o to the same interrupt 1429 * used for stmmac's main interrupts, which leads to interrupt storms. 1430 * STMMAC_FLAG_EEE_DISABLE allows EEE to be disabled on such platforms. 1431 */ 1432 if (priv->dma_cap.eee && 1433 !(priv->plat->flags & STMMAC_FLAG_EEE_DISABLE)) { 1434 /* The GMAC 3.74a databook states that EEE is only supported 1435 * in MII, GMII, and RGMII interfaces. 1436 */ 1437 __set_bit(PHY_INTERFACE_MODE_MII, config->lpi_interfaces); 1438 __set_bit(PHY_INTERFACE_MODE_GMII, config->lpi_interfaces); 1439 phy_interface_set_rgmii(config->lpi_interfaces); 1440 1441 /* If we have a non-integrated PCS, assume that it is connected 1442 * to the GMAC using GMII or another EEE compatible interface, 1443 * and thus all PCS-supported interfaces support LPI. 1444 */ 1445 if (pcs) 1446 phy_interface_or(config->lpi_interfaces, 1447 config->lpi_interfaces, 1448 pcs->supported_interfaces); 1449 1450 /* All full duplex speeds above 100Mbps are supported */ 1451 config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD; 1452 config->lpi_timer_default = eee_timer * 1000; 1453 config->eee_enabled_default = true; 1454 } 1455 1456 config->wol_phy_speed_ctrl = true; 1457 if (priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL) { 1458 config->wol_phy_legacy = true; 1459 } else { 1460 if (priv->dma_cap.pmt_remote_wake_up) 1461 config->wol_mac_support |= WAKE_UCAST; 1462 if (priv->dma_cap.pmt_magic_frame) 1463 config->wol_mac_support |= WAKE_MAGIC; 1464 } 1465 1466 phylink = phylink_create(config, dev_fwnode(priv->device), 1467 priv->plat->phy_interface, 1468 &stmmac_phylink_mac_ops); 1469 if (IS_ERR(phylink)) 1470 return PTR_ERR(phylink); 1471 1472 priv->phylink = phylink; 1473 return 0; 1474 } 1475 1476 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1477 struct stmmac_dma_conf *dma_conf) 1478 { 1479 u8 rx_cnt = priv->plat->rx_queues_to_use; 1480 unsigned int desc_size; 1481 void *head_rx; 1482 u8 queue; 1483 1484 /* Display RX rings */ 1485 for (queue = 0; queue < rx_cnt; queue++) { 1486 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1487 1488 pr_info("\tRX Queue %u rings\n", queue); 1489 1490 head_rx = stmmac_get_rx_desc(priv, rx_q, 0); 1491 desc_size = stmmac_get_rx_desc_size(priv); 1492 1493 /* Display RX ring */ 1494 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1495 rx_q->dma_rx_phy, desc_size); 1496 } 1497 } 1498 1499 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1500 struct stmmac_dma_conf *dma_conf) 1501 { 1502 u8 tx_cnt = priv->plat->tx_queues_to_use; 1503 unsigned int desc_size; 1504 void *head_tx; 1505 u8 queue; 1506 1507 /* Display TX rings */ 1508 for (queue = 0; queue < tx_cnt; queue++) { 1509 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1510 1511 pr_info("\tTX Queue %d rings\n", queue); 1512 1513 head_tx = stmmac_get_tx_desc(priv, tx_q, 0); 1514 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 1515 1516 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1517 tx_q->dma_tx_phy, desc_size); 1518 } 1519 } 1520 1521 static void stmmac_display_rings(struct stmmac_priv *priv, 1522 struct stmmac_dma_conf *dma_conf) 1523 { 1524 /* Display RX ring */ 1525 stmmac_display_rx_rings(priv, dma_conf); 1526 1527 /* Display TX ring */ 1528 stmmac_display_tx_rings(priv, dma_conf); 1529 } 1530 1531 static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) 1532 { 1533 if (stmmac_xdp_is_enabled(priv)) 1534 return XDP_PACKET_HEADROOM + NET_IP_ALIGN; 1535 1536 return NET_SKB_PAD + NET_IP_ALIGN; 1537 } 1538 1539 static int stmmac_set_bfsize(int len) 1540 { 1541 int ret; 1542 1543 if (len > BUF_SIZE_8KiB) 1544 ret = BUF_SIZE_16KiB; 1545 else if (len > BUF_SIZE_4KiB) 1546 ret = BUF_SIZE_8KiB; 1547 else if (len > BUF_SIZE_2KiB) 1548 ret = BUF_SIZE_4KiB; 1549 else if (len > DEFAULT_BUFSIZE) 1550 ret = BUF_SIZE_2KiB; 1551 else 1552 ret = DEFAULT_BUFSIZE; 1553 1554 return ret; 1555 } 1556 1557 /** 1558 * stmmac_clear_rx_descriptors - clear RX descriptors 1559 * @priv: driver private structure 1560 * @dma_conf: structure to take the dma data 1561 * @queue: RX queue index 1562 * Description: this function is called to clear the RX descriptors 1563 * in case of both basic and extended descriptors are used. 1564 */ 1565 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1566 struct stmmac_dma_conf *dma_conf, 1567 u32 queue) 1568 { 1569 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1570 struct dma_desc *desc; 1571 int i; 1572 1573 /* Clear the RX descriptors */ 1574 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1575 desc = stmmac_get_rx_desc(priv, rx_q, i); 1576 1577 stmmac_init_rx_desc(priv, desc, priv->use_riwt, 1578 priv->descriptor_mode, 1579 (i == dma_conf->dma_rx_size - 1), 1580 dma_conf->dma_buf_sz); 1581 } 1582 } 1583 1584 /** 1585 * stmmac_clear_tx_descriptors - clear tx descriptors 1586 * @priv: driver private structure 1587 * @dma_conf: structure to take the dma data 1588 * @queue: TX queue index. 1589 * Description: this function is called to clear the TX descriptors 1590 * in case of both basic and extended descriptors are used. 1591 */ 1592 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1593 struct stmmac_dma_conf *dma_conf, 1594 u32 queue) 1595 { 1596 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1597 int i; 1598 1599 /* Clear the TX descriptors */ 1600 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1601 int last = (i == (dma_conf->dma_tx_size - 1)); 1602 struct dma_desc *p; 1603 1604 p = stmmac_get_tx_desc(priv, tx_q, i); 1605 stmmac_init_tx_desc(priv, p, priv->descriptor_mode, last); 1606 } 1607 } 1608 1609 /** 1610 * stmmac_clear_descriptors - clear descriptors 1611 * @priv: driver private structure 1612 * @dma_conf: structure to take the dma data 1613 * Description: this function is called to clear the TX and RX descriptors 1614 * in case of both basic and extended descriptors are used. 1615 */ 1616 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1617 struct stmmac_dma_conf *dma_conf) 1618 { 1619 u8 rx_queue_cnt = priv->plat->rx_queues_to_use; 1620 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 1621 u8 queue; 1622 1623 /* Clear the RX descriptors */ 1624 for (queue = 0; queue < rx_queue_cnt; queue++) 1625 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1626 1627 /* Clear the TX descriptors */ 1628 for (queue = 0; queue < tx_queue_cnt; queue++) 1629 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1630 } 1631 1632 static bool stmmac_rx_check_buf2_cap(struct stmmac_priv *priv) 1633 { 1634 /* Only cores that can program an independent secondary RX buffer 1635 * (used for scatter-gather overflow or split-header payload) back 1636 * buffer2. Legacy cores have no set_sec_addr op, so buffer2 is 1637 * never handed to the hardware there. 1638 */ 1639 return priv->hw->desc && priv->hw->desc->set_sec_addr; 1640 } 1641 1642 /** 1643 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1644 * @priv: driver private structure 1645 * @dma_conf: structure to take the dma data 1646 * @p: descriptor pointer 1647 * @i: descriptor index 1648 * @flags: gfp flag 1649 * @queue: RX queue index 1650 * Description: this function is called to allocate a receive buffer, perform 1651 * the DMA mapping and init the descriptor. 1652 */ 1653 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1654 struct stmmac_dma_conf *dma_conf, 1655 struct dma_desc *p, 1656 int i, gfp_t flags, u32 queue) 1657 { 1658 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1659 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1660 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1661 1662 if (priv->dma_cap.host_dma_width <= 32) 1663 gfp |= GFP_DMA32; 1664 1665 if (!buf->page) { 1666 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1667 if (!buf->page) 1668 return -ENOMEM; 1669 buf->page_offset = stmmac_rx_offset(priv); 1670 } 1671 1672 if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) { 1673 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1674 if (!buf->sec_page) 1675 return -ENOMEM; 1676 1677 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1678 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1679 } 1680 1681 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1682 1683 stmmac_set_desc_addr(priv, p, buf->addr); 1684 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1685 stmmac_init_desc3(priv, p); 1686 1687 return 0; 1688 } 1689 1690 /** 1691 * stmmac_free_rx_buffer - free RX dma buffers 1692 * @priv: private structure 1693 * @rx_q: RX queue 1694 * @i: buffer index. 1695 */ 1696 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1697 struct stmmac_rx_queue *rx_q, 1698 int i) 1699 { 1700 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1701 1702 if (buf->page) 1703 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1704 buf->page = NULL; 1705 1706 if (buf->sec_page) 1707 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1708 buf->sec_page = NULL; 1709 } 1710 1711 /** 1712 * stmmac_free_tx_buffer - free RX dma buffers 1713 * @priv: private structure 1714 * @dma_conf: structure to take the dma data 1715 * @queue: RX queue index 1716 * @i: buffer index. 1717 */ 1718 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1719 struct stmmac_dma_conf *dma_conf, 1720 u32 queue, int i) 1721 { 1722 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1723 1724 if (tx_q->tx_skbuff_dma[i].buf && 1725 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1726 if (tx_q->tx_skbuff_dma[i].map_as_page) 1727 dma_unmap_page(priv->device, 1728 tx_q->tx_skbuff_dma[i].buf, 1729 tx_q->tx_skbuff_dma[i].len, 1730 DMA_TO_DEVICE); 1731 else 1732 dma_unmap_single(priv->device, 1733 tx_q->tx_skbuff_dma[i].buf, 1734 tx_q->tx_skbuff_dma[i].len, 1735 DMA_TO_DEVICE); 1736 } 1737 1738 if (tx_q->xdpf[i] && 1739 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1740 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1741 xdp_return_frame(tx_q->xdpf[i]); 1742 tx_q->xdpf[i] = NULL; 1743 } 1744 1745 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1746 tx_q->xsk_frames_done++; 1747 1748 if (tx_q->tx_skbuff[i] && 1749 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1750 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1751 tx_q->tx_skbuff[i] = NULL; 1752 } 1753 1754 tx_q->tx_skbuff_dma[i].buf = 0; 1755 tx_q->tx_skbuff_dma[i].map_as_page = false; 1756 } 1757 1758 /** 1759 * dma_free_rx_skbufs - free RX dma buffers 1760 * @priv: private structure 1761 * @dma_conf: structure to take the dma data 1762 * @queue: RX queue index 1763 */ 1764 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1765 struct stmmac_dma_conf *dma_conf, 1766 u32 queue) 1767 { 1768 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1769 int i; 1770 1771 for (i = 0; i < dma_conf->dma_rx_size; i++) 1772 stmmac_free_rx_buffer(priv, rx_q, i); 1773 } 1774 1775 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1776 struct stmmac_dma_conf *dma_conf, 1777 u32 queue, gfp_t flags) 1778 { 1779 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1780 int i; 1781 1782 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1783 struct dma_desc *p; 1784 int ret; 1785 1786 p = stmmac_get_rx_desc(priv, rx_q, i); 1787 1788 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1789 queue); 1790 if (ret) 1791 return ret; 1792 1793 rx_q->buf_alloc_num++; 1794 } 1795 1796 return 0; 1797 } 1798 1799 /** 1800 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1801 * @priv: private structure 1802 * @dma_conf: structure to take the dma data 1803 * @queue: RX queue index 1804 */ 1805 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1806 struct stmmac_dma_conf *dma_conf, 1807 u32 queue) 1808 { 1809 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1810 int i; 1811 1812 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1813 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1814 1815 if (!buf->xdp) 1816 continue; 1817 1818 xsk_buff_free(buf->xdp); 1819 buf->xdp = NULL; 1820 } 1821 } 1822 1823 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1824 struct stmmac_dma_conf *dma_conf, 1825 u32 queue) 1826 { 1827 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1828 int i; 1829 1830 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1831 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1832 * use this macro to make sure no size violations. 1833 */ 1834 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1835 1836 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1837 struct stmmac_rx_buffer *buf; 1838 dma_addr_t dma_addr; 1839 struct dma_desc *p; 1840 1841 p = stmmac_get_rx_desc(priv, rx_q, i); 1842 1843 buf = &rx_q->buf_pool[i]; 1844 1845 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1846 if (!buf->xdp) 1847 return -ENOMEM; 1848 1849 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1850 stmmac_set_desc_addr(priv, p, dma_addr); 1851 rx_q->buf_alloc_num++; 1852 } 1853 1854 return 0; 1855 } 1856 1857 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1858 { 1859 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1860 return NULL; 1861 1862 return xsk_get_pool_from_qid(priv->dev, queue); 1863 } 1864 1865 /** 1866 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1867 * @priv: driver private structure 1868 * @dma_conf: structure to take the dma data 1869 * @queue: RX queue index 1870 * @flags: gfp flag. 1871 * Description: this function initializes the DMA RX descriptors 1872 * and allocates the socket buffers. It supports the chained and ring 1873 * modes. 1874 */ 1875 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1876 struct stmmac_dma_conf *dma_conf, 1877 u32 queue, gfp_t flags) 1878 { 1879 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1880 void *des; 1881 int ret; 1882 1883 netif_dbg(priv, probe, priv->dev, 1884 "(%s) dma_rx_phy=0x%08x\n", __func__, 1885 (u32)rx_q->dma_rx_phy); 1886 1887 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1888 1889 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1890 1891 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1892 1893 if (rx_q->xsk_pool) { 1894 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1895 MEM_TYPE_XSK_BUFF_POOL, 1896 NULL)); 1897 netdev_info(priv->dev, 1898 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1899 queue); 1900 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1901 } else { 1902 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1903 MEM_TYPE_PAGE_POOL, 1904 rx_q->page_pool)); 1905 netdev_info(priv->dev, 1906 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1907 queue); 1908 } 1909 1910 if (rx_q->xsk_pool) { 1911 /* RX XDP ZC buffer pool may not be populated, e.g. 1912 * xdpsock TX-only. 1913 */ 1914 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1915 } else { 1916 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1917 if (ret < 0) 1918 return -ENOMEM; 1919 } 1920 1921 /* Setup the chained descriptor addresses */ 1922 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 1923 if (priv->extend_desc) 1924 des = rx_q->dma_erx; 1925 else 1926 des = rx_q->dma_rx; 1927 1928 stmmac_mode_init(priv, des, rx_q->dma_rx_phy, 1929 dma_conf->dma_rx_size, priv->extend_desc); 1930 } 1931 1932 return 0; 1933 } 1934 1935 static int init_dma_rx_desc_rings(struct net_device *dev, 1936 struct stmmac_dma_conf *dma_conf, 1937 gfp_t flags) 1938 { 1939 struct stmmac_priv *priv = netdev_priv(dev); 1940 u8 rx_count = priv->plat->rx_queues_to_use; 1941 int queue; 1942 int ret; 1943 1944 /* RX INITIALIZATION */ 1945 netif_dbg(priv, probe, priv->dev, 1946 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1947 1948 for (queue = 0; queue < rx_count; queue++) { 1949 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1950 if (ret) 1951 goto err_init_rx_buffers; 1952 } 1953 1954 return 0; 1955 1956 err_init_rx_buffers: 1957 while (queue >= 0) { 1958 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1959 1960 if (rx_q->xsk_pool) 1961 dma_free_rx_xskbufs(priv, dma_conf, queue); 1962 else 1963 dma_free_rx_skbufs(priv, dma_conf, queue); 1964 1965 rx_q->buf_alloc_num = 0; 1966 rx_q->xsk_pool = NULL; 1967 1968 queue--; 1969 } 1970 1971 return ret; 1972 } 1973 1974 static void stmmac_set_tx_dma_entry(struct stmmac_tx_queue *tx_q, 1975 unsigned int entry, 1976 enum stmmac_txbuf_type type, 1977 dma_addr_t addr, size_t len, 1978 bool map_as_page) 1979 { 1980 tx_q->tx_skbuff_dma[entry].buf = addr; 1981 tx_q->tx_skbuff_dma[entry].len = len; 1982 tx_q->tx_skbuff_dma[entry].buf_type = type; 1983 tx_q->tx_skbuff_dma[entry].map_as_page = map_as_page; 1984 tx_q->tx_skbuff_dma[entry].last_segment = false; 1985 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1986 } 1987 1988 static void stmmac_set_tx_skb_dma_entry(struct stmmac_tx_queue *tx_q, 1989 unsigned int entry, dma_addr_t addr, 1990 size_t len, bool map_as_page) 1991 { 1992 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_SKB, addr, len, 1993 map_as_page); 1994 } 1995 1996 static void stmmac_set_tx_dma_last_segment(struct stmmac_tx_queue *tx_q, 1997 unsigned int entry) 1998 { 1999 tx_q->tx_skbuff_dma[entry].last_segment = true; 2000 } 2001 2002 /** 2003 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 2004 * @priv: driver private structure 2005 * @dma_conf: structure to take the dma data 2006 * @queue: TX queue index 2007 * Description: this function initializes the DMA TX descriptors 2008 * and allocates the socket buffers. It supports the chained and ring 2009 * modes. 2010 */ 2011 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 2012 struct stmmac_dma_conf *dma_conf, 2013 u32 queue) 2014 { 2015 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2016 int i; 2017 2018 netif_dbg(priv, probe, priv->dev, 2019 "(%s) dma_tx_phy=0x%08x\n", __func__, 2020 (u32)tx_q->dma_tx_phy); 2021 2022 /* Setup the chained descriptor addresses */ 2023 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 2024 if (priv->extend_desc) 2025 stmmac_mode_init(priv, tx_q->dma_etx, 2026 tx_q->dma_tx_phy, 2027 dma_conf->dma_tx_size, 1); 2028 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 2029 stmmac_mode_init(priv, tx_q->dma_tx, 2030 tx_q->dma_tx_phy, 2031 dma_conf->dma_tx_size, 0); 2032 } 2033 2034 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 2035 2036 for (i = 0; i < dma_conf->dma_tx_size; i++) { 2037 struct dma_desc *p; 2038 2039 p = stmmac_get_tx_desc(priv, tx_q, i); 2040 stmmac_clear_desc(priv, p); 2041 stmmac_set_tx_skb_dma_entry(tx_q, i, 0, 0, false); 2042 2043 tx_q->tx_skbuff[i] = NULL; 2044 } 2045 2046 return 0; 2047 } 2048 2049 static int init_dma_tx_desc_rings(struct net_device *dev, 2050 struct stmmac_dma_conf *dma_conf) 2051 { 2052 struct stmmac_priv *priv = netdev_priv(dev); 2053 u8 tx_queue_cnt; 2054 u8 queue; 2055 2056 tx_queue_cnt = priv->plat->tx_queues_to_use; 2057 2058 for (queue = 0; queue < tx_queue_cnt; queue++) 2059 __init_dma_tx_desc_rings(priv, dma_conf, queue); 2060 2061 return 0; 2062 } 2063 2064 /** 2065 * init_dma_desc_rings - init the RX/TX descriptor rings 2066 * @dev: net device structure 2067 * @dma_conf: structure to take the dma data 2068 * @flags: gfp flag. 2069 * Description: this function initializes the DMA RX/TX descriptors 2070 * and allocates the socket buffers. It supports the chained and ring 2071 * modes. 2072 */ 2073 static int init_dma_desc_rings(struct net_device *dev, 2074 struct stmmac_dma_conf *dma_conf, 2075 gfp_t flags) 2076 { 2077 struct stmmac_priv *priv = netdev_priv(dev); 2078 int ret; 2079 2080 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 2081 if (ret) 2082 return ret; 2083 2084 ret = init_dma_tx_desc_rings(dev, dma_conf); 2085 2086 stmmac_clear_descriptors(priv, dma_conf); 2087 2088 if (netif_msg_hw(priv)) 2089 stmmac_display_rings(priv, dma_conf); 2090 2091 return ret; 2092 } 2093 2094 /** 2095 * dma_free_tx_skbufs - free TX dma buffers 2096 * @priv: private structure 2097 * @dma_conf: structure to take the dma data 2098 * @queue: TX queue index 2099 */ 2100 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 2101 struct stmmac_dma_conf *dma_conf, 2102 u32 queue) 2103 { 2104 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2105 int i; 2106 2107 tx_q->xsk_frames_done = 0; 2108 2109 for (i = 0; i < dma_conf->dma_tx_size; i++) 2110 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 2111 2112 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 2113 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2114 tx_q->xsk_frames_done = 0; 2115 tx_q->xsk_pool = NULL; 2116 } 2117 } 2118 2119 /** 2120 * stmmac_free_tx_skbufs - free TX skb buffers 2121 * @priv: private structure 2122 */ 2123 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 2124 { 2125 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 2126 u8 queue; 2127 2128 for (queue = 0; queue < tx_queue_cnt; queue++) 2129 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 2130 } 2131 2132 /** 2133 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 2134 * @priv: private structure 2135 * @dma_conf: structure to take the dma data 2136 * @queue: RX queue index 2137 */ 2138 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 2139 struct stmmac_dma_conf *dma_conf, 2140 u32 queue) 2141 { 2142 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2143 size_t size; 2144 void *addr; 2145 2146 /* Release the DMA RX socket buffers */ 2147 if (rx_q->xsk_pool) 2148 dma_free_rx_xskbufs(priv, dma_conf, queue); 2149 else 2150 dma_free_rx_skbufs(priv, dma_conf, queue); 2151 2152 rx_q->buf_alloc_num = 0; 2153 rx_q->xsk_pool = NULL; 2154 2155 /* Free DMA regions of consistent memory previously allocated */ 2156 if (priv->extend_desc) 2157 addr = rx_q->dma_erx; 2158 else 2159 addr = rx_q->dma_rx; 2160 2161 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2162 2163 dma_free_coherent(priv->device, size, addr, rx_q->dma_rx_phy); 2164 2165 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 2166 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 2167 2168 kfree(rx_q->buf_pool); 2169 if (rx_q->page_pool) 2170 page_pool_destroy(rx_q->page_pool); 2171 } 2172 2173 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 2174 struct stmmac_dma_conf *dma_conf) 2175 { 2176 u8 rx_count = priv->plat->rx_queues_to_use; 2177 u8 queue; 2178 2179 /* Free RX queue resources */ 2180 for (queue = 0; queue < rx_count; queue++) 2181 __free_dma_rx_desc_resources(priv, dma_conf, queue); 2182 } 2183 2184 /** 2185 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 2186 * @priv: private structure 2187 * @dma_conf: structure to take the dma data 2188 * @queue: TX queue index 2189 */ 2190 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 2191 struct stmmac_dma_conf *dma_conf, 2192 u32 queue) 2193 { 2194 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2195 size_t size; 2196 void *addr; 2197 2198 /* Release the DMA TX socket buffers */ 2199 dma_free_tx_skbufs(priv, dma_conf, queue); 2200 2201 if (priv->extend_desc) { 2202 addr = tx_q->dma_etx; 2203 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 2204 addr = tx_q->dma_entx; 2205 } else { 2206 addr = tx_q->dma_tx; 2207 } 2208 2209 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2210 2211 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 2212 2213 kfree(tx_q->tx_skbuff_dma); 2214 kfree(tx_q->tx_skbuff); 2215 } 2216 2217 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2218 struct stmmac_dma_conf *dma_conf) 2219 { 2220 u8 tx_count = priv->plat->tx_queues_to_use; 2221 u8 queue; 2222 2223 /* Free TX queue resources */ 2224 for (queue = 0; queue < tx_count; queue++) 2225 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2226 } 2227 2228 /** 2229 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2230 * @priv: private structure 2231 * @dma_conf: structure to take the dma data 2232 * @queue: RX queue index 2233 * Description: according to which descriptor can be used (extend or basic) 2234 * this function allocates the resources for TX and RX paths. In case of 2235 * reception, for example, it pre-allocated the RX socket buffer in order to 2236 * allow zero-copy mechanism. 2237 */ 2238 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2239 struct stmmac_dma_conf *dma_conf, 2240 u32 queue) 2241 { 2242 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2243 struct stmmac_channel *ch = &priv->channel[queue]; 2244 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2245 struct page_pool_params pp_params = { 0 }; 2246 unsigned int dma_buf_sz_pad, num_pages; 2247 unsigned int napi_id; 2248 size_t size; 2249 void *addr; 2250 int ret; 2251 2252 dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + 2253 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2254 num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE); 2255 2256 rx_q->queue_index = queue; 2257 rx_q->priv_data = priv; 2258 rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE; 2259 2260 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2261 pp_params.pool_size = dma_conf->dma_rx_size; 2262 pp_params.order = order_base_2(num_pages); 2263 pp_params.nid = dev_to_node(priv->device); 2264 pp_params.dev = priv->device; 2265 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2266 pp_params.offset = 0; 2267 pp_params.max_len = dma_conf->dma_buf_sz + stmmac_rx_offset(priv); 2268 2269 rx_q->page_pool = page_pool_create(&pp_params); 2270 if (IS_ERR(rx_q->page_pool)) { 2271 ret = PTR_ERR(rx_q->page_pool); 2272 rx_q->page_pool = NULL; 2273 return ret; 2274 } 2275 2276 rx_q->buf_pool = kzalloc_objs(*rx_q->buf_pool, dma_conf->dma_rx_size); 2277 if (!rx_q->buf_pool) 2278 return -ENOMEM; 2279 2280 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2281 2282 addr = dma_alloc_coherent(priv->device, size, &rx_q->dma_rx_phy, 2283 GFP_KERNEL); 2284 if (!addr) 2285 return -ENOMEM; 2286 2287 if (priv->extend_desc) 2288 rx_q->dma_erx = addr; 2289 else 2290 rx_q->dma_rx = addr; 2291 2292 if (stmmac_xdp_is_enabled(priv) && 2293 test_bit(queue, priv->af_xdp_zc_qps)) 2294 napi_id = ch->rxtx_napi.napi_id; 2295 else 2296 napi_id = ch->rx_napi.napi_id; 2297 2298 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, queue, napi_id); 2299 if (ret) { 2300 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2301 return -EINVAL; 2302 } 2303 2304 return 0; 2305 } 2306 2307 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2308 struct stmmac_dma_conf *dma_conf) 2309 { 2310 u8 rx_count = priv->plat->rx_queues_to_use; 2311 u8 queue; 2312 int ret; 2313 2314 /* RX queues buffers and DMA */ 2315 for (queue = 0; queue < rx_count; queue++) { 2316 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2317 if (ret) 2318 goto err_dma; 2319 } 2320 2321 return 0; 2322 2323 err_dma: 2324 free_dma_rx_desc_resources(priv, dma_conf); 2325 2326 return ret; 2327 } 2328 2329 /** 2330 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2331 * @priv: private structure 2332 * @dma_conf: structure to take the dma data 2333 * @queue: TX queue index 2334 * Description: according to which descriptor can be used (extend or basic) 2335 * this function allocates the resources for TX and RX paths. In case of 2336 * reception, for example, it pre-allocated the RX socket buffer in order to 2337 * allow zero-copy mechanism. 2338 */ 2339 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2340 struct stmmac_dma_conf *dma_conf, 2341 u32 queue) 2342 { 2343 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2344 size_t size; 2345 void *addr; 2346 2347 tx_q->queue_index = queue; 2348 tx_q->priv_data = priv; 2349 2350 tx_q->tx_skbuff_dma = kzalloc_objs(*tx_q->tx_skbuff_dma, 2351 dma_conf->dma_tx_size); 2352 if (!tx_q->tx_skbuff_dma) 2353 return -ENOMEM; 2354 2355 tx_q->tx_skbuff = kzalloc_objs(struct sk_buff *, dma_conf->dma_tx_size); 2356 if (!tx_q->tx_skbuff) 2357 return -ENOMEM; 2358 2359 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2360 2361 addr = dma_alloc_coherent(priv->device, size, 2362 &tx_q->dma_tx_phy, GFP_KERNEL); 2363 if (!addr) 2364 return -ENOMEM; 2365 2366 if (priv->extend_desc) 2367 tx_q->dma_etx = addr; 2368 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2369 tx_q->dma_entx = addr; 2370 else 2371 tx_q->dma_tx = addr; 2372 2373 return 0; 2374 } 2375 2376 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2377 struct stmmac_dma_conf *dma_conf) 2378 { 2379 u8 tx_count = priv->plat->tx_queues_to_use; 2380 u8 queue; 2381 int ret; 2382 2383 /* TX queues buffers and DMA */ 2384 for (queue = 0; queue < tx_count; queue++) { 2385 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2386 if (ret) 2387 goto err_dma; 2388 } 2389 2390 return 0; 2391 2392 err_dma: 2393 free_dma_tx_desc_resources(priv, dma_conf); 2394 return ret; 2395 } 2396 2397 /** 2398 * alloc_dma_desc_resources - alloc TX/RX resources. 2399 * @priv: private structure 2400 * @dma_conf: structure to take the dma data 2401 * Description: according to which descriptor can be used (extend or basic) 2402 * this function allocates the resources for TX and RX paths. In case of 2403 * reception, for example, it pre-allocated the RX socket buffer in order to 2404 * allow zero-copy mechanism. 2405 */ 2406 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2407 struct stmmac_dma_conf *dma_conf) 2408 { 2409 /* RX Allocation */ 2410 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2411 2412 if (ret) 2413 return ret; 2414 2415 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2416 2417 return ret; 2418 } 2419 2420 /** 2421 * free_dma_desc_resources - free dma desc resources 2422 * @priv: private structure 2423 * @dma_conf: structure to take the dma data 2424 */ 2425 static void free_dma_desc_resources(struct stmmac_priv *priv, 2426 struct stmmac_dma_conf *dma_conf) 2427 { 2428 /* Release the DMA TX socket buffers */ 2429 free_dma_tx_desc_resources(priv, dma_conf); 2430 2431 /* Release the DMA RX socket buffers later 2432 * to ensure all pending XDP_TX buffers are returned. 2433 */ 2434 free_dma_rx_desc_resources(priv, dma_conf); 2435 } 2436 2437 /** 2438 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2439 * @priv: driver private structure 2440 * Description: It is used for enabling the rx queues in the MAC 2441 */ 2442 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2443 { 2444 u8 rx_queues_count = priv->plat->rx_queues_to_use; 2445 u8 queue; 2446 u8 mode; 2447 2448 for (queue = 0; queue < rx_queues_count; queue++) { 2449 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2450 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2451 } 2452 } 2453 2454 /** 2455 * stmmac_start_rx_dma - start RX DMA channel 2456 * @priv: driver private structure 2457 * @chan: RX channel index 2458 * Description: 2459 * This starts a RX DMA channel 2460 */ 2461 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2462 { 2463 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2464 stmmac_start_rx(priv, priv->ioaddr, chan); 2465 } 2466 2467 /** 2468 * stmmac_start_tx_dma - start TX DMA channel 2469 * @priv: driver private structure 2470 * @chan: TX channel index 2471 * Description: 2472 * This starts a TX DMA channel 2473 */ 2474 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2475 { 2476 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2477 stmmac_start_tx(priv, priv->ioaddr, chan); 2478 } 2479 2480 /** 2481 * stmmac_stop_rx_dma - stop RX DMA channel 2482 * @priv: driver private structure 2483 * @chan: RX channel index 2484 * Description: 2485 * This stops a RX DMA channel 2486 */ 2487 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2488 { 2489 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2490 stmmac_stop_rx(priv, priv->ioaddr, chan); 2491 } 2492 2493 /** 2494 * stmmac_stop_tx_dma - stop TX DMA channel 2495 * @priv: driver private structure 2496 * @chan: TX channel index 2497 * Description: 2498 * This stops a TX DMA channel 2499 */ 2500 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2501 { 2502 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2503 stmmac_stop_tx(priv, priv->ioaddr, chan); 2504 } 2505 2506 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2507 { 2508 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2509 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2510 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2511 u8 chan; 2512 2513 for (chan = 0; chan < dma_csr_ch; chan++) { 2514 struct stmmac_channel *ch = &priv->channel[chan]; 2515 unsigned long flags; 2516 2517 spin_lock_irqsave(&ch->lock, flags); 2518 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2519 spin_unlock_irqrestore(&ch->lock, flags); 2520 } 2521 } 2522 2523 /** 2524 * stmmac_start_all_dma - start all RX and TX DMA channels 2525 * @priv: driver private structure 2526 * Description: 2527 * This starts all the RX and TX DMA channels 2528 */ 2529 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2530 { 2531 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2532 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2533 u8 chan; 2534 2535 for (chan = 0; chan < rx_channels_count; chan++) 2536 stmmac_start_rx_dma(priv, chan); 2537 2538 for (chan = 0; chan < tx_channels_count; chan++) 2539 stmmac_start_tx_dma(priv, chan); 2540 } 2541 2542 /** 2543 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2544 * @priv: driver private structure 2545 * Description: 2546 * This stops the RX and TX DMA channels 2547 */ 2548 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2549 { 2550 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2551 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2552 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2553 u8 chan; 2554 2555 for (chan = 0; chan < rx_channels_count; chan++) 2556 stmmac_stop_rx_dma(priv, chan); 2557 2558 for (chan = 0; chan < tx_channels_count; chan++) 2559 stmmac_stop_tx_dma(priv, chan); 2560 2561 for (chan = 0; chan < dma_csr_ch; chan++) 2562 stmmac_deinit_chan(priv, priv->ioaddr, chan); 2563 } 2564 2565 /** 2566 * stmmac_dma_operation_mode - HW DMA operation mode 2567 * @priv: driver private structure 2568 * Description: it is used for configuring the DMA operation mode register in 2569 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2570 */ 2571 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2572 { 2573 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2574 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2575 int rxfifosz = priv->plat->rx_fifo_size; 2576 int txfifosz = priv->plat->tx_fifo_size; 2577 u32 txmode = 0; 2578 u32 rxmode = 0; 2579 u8 qmode = 0; 2580 u8 chan; 2581 2582 if (rxfifosz == 0) 2583 rxfifosz = priv->dma_cap.rx_fifo_size; 2584 if (txfifosz == 0) 2585 txfifosz = priv->dma_cap.tx_fifo_size; 2586 2587 /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ 2588 if (dwmac_is_xmac(priv->plat->core_type)) { 2589 rxfifosz /= rx_channels_count; 2590 txfifosz /= tx_channels_count; 2591 } 2592 2593 if (priv->plat->force_thresh_dma_mode) { 2594 txmode = tc; 2595 rxmode = tc; 2596 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2597 /* 2598 * In case of GMAC, SF mode can be enabled 2599 * to perform the TX COE in HW. This depends on: 2600 * 1) TX COE if actually supported 2601 * 2) There is no bugged Jumbo frame support 2602 * that needs to not insert csum in the TDES. 2603 */ 2604 txmode = SF_DMA_MODE; 2605 rxmode = SF_DMA_MODE; 2606 priv->xstats.threshold = SF_DMA_MODE; 2607 } else { 2608 txmode = tc; 2609 rxmode = SF_DMA_MODE; 2610 } 2611 2612 /* configure all channels */ 2613 for (chan = 0; chan < rx_channels_count; chan++) { 2614 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2615 2616 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2617 2618 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2619 rxfifosz, qmode); 2620 2621 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 2622 } 2623 2624 for (chan = 0; chan < tx_channels_count; chan++) { 2625 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2626 2627 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2628 txfifosz, qmode); 2629 } 2630 } 2631 2632 static void stmmac_xsk_request_timestamp(void *_priv) 2633 { 2634 struct stmmac_metadata_request *meta_req = _priv; 2635 2636 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2637 *meta_req->set_ic = true; 2638 } 2639 2640 static u64 stmmac_xsk_fill_timestamp(void *_priv) 2641 { 2642 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2643 struct stmmac_priv *priv = tx_compl->priv; 2644 struct dma_desc *desc = tx_compl->desc; 2645 bool found = false; 2646 u64 ns = 0; 2647 2648 if (!priv->hwts_tx_en) 2649 return 0; 2650 2651 /* check tx tstamp status */ 2652 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2653 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2654 found = true; 2655 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2656 found = true; 2657 } 2658 2659 if (found) { 2660 ns -= priv->plat->cdc_error_adj; 2661 return ns_to_ktime(ns); 2662 } 2663 2664 return 0; 2665 } 2666 2667 static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv) 2668 { 2669 struct timespec64 ts = ns_to_timespec64(launch_time); 2670 struct stmmac_metadata_request *meta_req = _priv; 2671 2672 if (meta_req->tbs & STMMAC_TBS_EN) 2673 stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec, 2674 ts.tv_nsec); 2675 } 2676 2677 static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2678 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2679 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2680 .tmo_request_launch_time = stmmac_xsk_request_launch_time, 2681 }; 2682 2683 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2684 { 2685 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2686 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2687 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2688 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 2689 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2690 unsigned int entry = tx_q->cur_tx; 2691 struct dma_desc *tx_desc = NULL; 2692 struct xdp_desc xdp_desc; 2693 bool work_done = true; 2694 u32 tx_set_ic_bit = 0; 2695 2696 /* Avoids TX time-out as we are sharing with slow path */ 2697 txq_trans_cond_update(nq); 2698 2699 budget = min(budget, stmmac_tx_avail(priv, queue)); 2700 2701 for (; budget > 0; budget--) { 2702 struct stmmac_metadata_request meta_req; 2703 struct xsk_tx_metadata *meta = NULL; 2704 dma_addr_t dma_addr; 2705 bool set_ic; 2706 2707 /* We are sharing with slow path and stop XSK TX desc submission when 2708 * available TX ring is less than threshold. 2709 */ 2710 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2711 !netif_carrier_ok(priv->dev)) { 2712 work_done = false; 2713 break; 2714 } 2715 2716 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2717 break; 2718 2719 if (priv->est && priv->est->enable && 2720 priv->est->max_sdu[queue] && 2721 xdp_desc.len > priv->est->max_sdu[queue]) { 2722 priv->xstats.max_sdu_txq_drop[queue]++; 2723 continue; 2724 } 2725 2726 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 2727 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2728 meta = xsk_buff_get_metadata(pool, xdp_desc.addr, 2729 xdp_desc.options); 2730 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2731 2732 /* To return XDP buffer to XSK pool, we simple call 2733 * xsk_tx_completed(), so we don't need to fill up 2734 * 'buf' and 'xdpf'. 2735 */ 2736 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_XSK_TX, 2737 0, xdp_desc.len, false); 2738 stmmac_set_tx_dma_last_segment(tx_q, entry); 2739 2740 tx_q->xdpf[entry] = NULL; 2741 2742 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2743 2744 tx_q->tx_count_frames++; 2745 2746 if (!priv->tx_coal_frames[queue]) 2747 set_ic = false; 2748 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2749 set_ic = true; 2750 else 2751 set_ic = false; 2752 2753 meta_req.priv = priv; 2754 meta_req.tx_desc = tx_desc; 2755 meta_req.set_ic = &set_ic; 2756 meta_req.tbs = tx_q->tbs; 2757 meta_req.edesc = &tx_q->dma_entx[entry]; 2758 xsk_tx_metadata_request(pool, &meta, 2759 &stmmac_xsk_tx_metadata_ops, &meta_req); 2760 if (set_ic) { 2761 tx_q->tx_count_frames = 0; 2762 stmmac_set_tx_ic(priv, tx_desc); 2763 tx_set_ic_bit++; 2764 } 2765 2766 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2767 csum, priv->descriptor_mode, true, true, 2768 xdp_desc.len); 2769 2770 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 2771 2772 xsk_tx_metadata_to_compl(meta, 2773 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2774 2775 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2776 entry = tx_q->cur_tx; 2777 } 2778 u64_stats_update_begin(&txq_stats->napi_syncp); 2779 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2780 u64_stats_update_end(&txq_stats->napi_syncp); 2781 2782 if (tx_desc) { 2783 stmmac_flush_tx_descriptors(priv, queue); 2784 xsk_tx_release(pool); 2785 } 2786 2787 /* Return true if all of the 3 conditions are met 2788 * a) TX Budget is still available 2789 * b) work_done = true when XSK TX desc peek is empty (no more 2790 * pending XSK TX for transmission) 2791 */ 2792 return !!budget && work_done; 2793 } 2794 2795 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2796 { 2797 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2798 tc += 64; 2799 2800 if (priv->plat->force_thresh_dma_mode) 2801 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2802 else 2803 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2804 chan); 2805 2806 priv->xstats.threshold = tc; 2807 } 2808 } 2809 2810 /** 2811 * stmmac_tx_clean - to manage the transmission completion 2812 * @priv: driver private structure 2813 * @budget: napi budget limiting this functions packet handling 2814 * @queue: TX queue index 2815 * @pending_packets: signal to arm the TX coal timer 2816 * Description: it reclaims the transmit resources after transmission completes. 2817 * If some packets still needs to be handled, due to TX coalesce, set 2818 * pending_packets to true to make NAPI arm the TX coal timer. 2819 */ 2820 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2821 bool *pending_packets) 2822 { 2823 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2824 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2825 unsigned int bytes_compl = 0, pkts_compl = 0; 2826 unsigned int entry, xmits = 0, count = 0; 2827 u32 tx_packets = 0, tx_errors = 0; 2828 2829 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2830 2831 tx_q->xsk_frames_done = 0; 2832 2833 entry = tx_q->dirty_tx; 2834 2835 /* Try to clean all TX complete frame in 1 shot */ 2836 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2837 struct xdp_frame *xdpf; 2838 struct sk_buff *skb; 2839 struct dma_desc *p; 2840 int status; 2841 2842 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2843 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2844 xdpf = tx_q->xdpf[entry]; 2845 skb = NULL; 2846 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2847 xdpf = NULL; 2848 skb = tx_q->tx_skbuff[entry]; 2849 } else { 2850 xdpf = NULL; 2851 skb = NULL; 2852 } 2853 2854 p = stmmac_get_tx_desc(priv, tx_q, entry); 2855 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2856 /* Check if the descriptor is owned by the DMA */ 2857 if (unlikely(status & tx_dma_own)) 2858 break; 2859 2860 count++; 2861 2862 /* Make sure descriptor fields are read after reading 2863 * the own bit. 2864 */ 2865 dma_rmb(); 2866 2867 /* Just consider the last segment and ...*/ 2868 if (likely(!(status & tx_not_ls))) { 2869 /* ... verify the status error condition */ 2870 if (unlikely(status & tx_err)) { 2871 tx_errors++; 2872 if (unlikely(status & tx_err_bump_tc)) 2873 stmmac_bump_dma_threshold(priv, queue); 2874 } else { 2875 tx_packets++; 2876 } 2877 if (skb) { 2878 stmmac_get_tx_hwtstamp(priv, p, skb); 2879 } else if (tx_q->xsk_pool && 2880 xp_tx_metadata_enabled(tx_q->xsk_pool)) { 2881 struct stmmac_xsk_tx_complete tx_compl = { 2882 .priv = priv, 2883 .desc = p, 2884 }; 2885 2886 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2887 &stmmac_xsk_tx_metadata_ops, 2888 &tx_compl); 2889 } 2890 } 2891 2892 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2893 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2894 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2895 dma_unmap_page(priv->device, 2896 tx_q->tx_skbuff_dma[entry].buf, 2897 tx_q->tx_skbuff_dma[entry].len, 2898 DMA_TO_DEVICE); 2899 else 2900 dma_unmap_single(priv->device, 2901 tx_q->tx_skbuff_dma[entry].buf, 2902 tx_q->tx_skbuff_dma[entry].len, 2903 DMA_TO_DEVICE); 2904 tx_q->tx_skbuff_dma[entry].buf = 0; 2905 tx_q->tx_skbuff_dma[entry].len = 0; 2906 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2907 } 2908 2909 /* This looks at tx_q->tx_skbuff_dma[tx_q->dirty_tx].is_jumbo 2910 * and tx_q->tx_skbuff_dma[tx_q->dirty_tx].last_segment 2911 */ 2912 stmmac_clean_desc3(priv, tx_q, p); 2913 2914 tx_q->tx_skbuff_dma[entry].last_segment = false; 2915 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2916 2917 if (xdpf && 2918 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2919 xdp_return_frame_rx_napi(xdpf); 2920 tx_q->xdpf[entry] = NULL; 2921 } 2922 2923 if (xdpf && 2924 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2925 xdp_return_frame(xdpf); 2926 tx_q->xdpf[entry] = NULL; 2927 } 2928 2929 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2930 tx_q->xsk_frames_done++; 2931 2932 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2933 if (likely(skb)) { 2934 pkts_compl++; 2935 bytes_compl += skb->len; 2936 dev_consume_skb_any(skb); 2937 tx_q->tx_skbuff[entry] = NULL; 2938 } 2939 } 2940 2941 stmmac_release_tx_desc(priv, p, priv->descriptor_mode); 2942 2943 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 2944 } 2945 tx_q->dirty_tx = entry; 2946 2947 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2948 pkts_compl, bytes_compl); 2949 2950 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2951 queue))) && 2952 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2953 2954 netif_dbg(priv, tx_done, priv->dev, 2955 "%s: restart transmit\n", __func__); 2956 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2957 } 2958 2959 if (tx_q->xsk_pool) { 2960 bool work_done; 2961 2962 if (tx_q->xsk_frames_done) 2963 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2964 2965 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2966 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2967 2968 /* For XSK TX, we try to send as many as possible. 2969 * If XSK work done (XSK TX desc empty and budget still 2970 * available), return "budget - 1" to reenable TX IRQ. 2971 * Else, return "budget" to make NAPI continue polling. 2972 */ 2973 work_done = stmmac_xdp_xmit_zc(priv, queue, 2974 STMMAC_XSK_TX_BUDGET_MAX); 2975 if (work_done) 2976 xmits = budget - 1; 2977 else 2978 xmits = budget; 2979 } 2980 2981 if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode) 2982 stmmac_restart_sw_lpi_timer(priv); 2983 2984 /* We still have pending packets, let's call for a new scheduling */ 2985 if (tx_q->dirty_tx != tx_q->cur_tx) 2986 *pending_packets = true; 2987 2988 u64_stats_update_begin(&txq_stats->napi_syncp); 2989 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2990 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2991 u64_stats_inc(&txq_stats->napi.tx_clean); 2992 u64_stats_update_end(&txq_stats->napi_syncp); 2993 2994 priv->xstats.tx_errors += tx_errors; 2995 2996 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2997 2998 /* Combine decisions from TX clean and XSK TX */ 2999 return max(count, xmits); 3000 } 3001 3002 /** 3003 * stmmac_tx_err - to manage the tx error 3004 * @priv: driver private structure 3005 * @chan: channel index 3006 * Description: it cleans the descriptors and restarts the transmission 3007 * in case of transmission errors. 3008 */ 3009 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 3010 { 3011 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3012 3013 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 3014 3015 stmmac_stop_tx_dma(priv, chan); 3016 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 3017 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 3018 stmmac_reset_tx_queue(priv, chan); 3019 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3020 tx_q->dma_tx_phy, chan); 3021 stmmac_start_tx_dma(priv, chan); 3022 3023 priv->xstats.tx_errors++; 3024 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 3025 } 3026 3027 /** 3028 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 3029 * @priv: driver private structure 3030 * @txmode: TX operating mode 3031 * @rxmode: RX operating mode 3032 * @chan: channel index 3033 * Description: it is used for configuring of the DMA operation mode in 3034 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 3035 * mode. 3036 */ 3037 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 3038 u32 rxmode, u32 chan) 3039 { 3040 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 3041 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 3042 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3043 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3044 int rxfifosz = priv->plat->rx_fifo_size; 3045 int txfifosz = priv->plat->tx_fifo_size; 3046 3047 if (rxfifosz == 0) 3048 rxfifosz = priv->dma_cap.rx_fifo_size; 3049 if (txfifosz == 0) 3050 txfifosz = priv->dma_cap.tx_fifo_size; 3051 3052 /* Adjust for real per queue fifo size */ 3053 rxfifosz /= rx_channels_count; 3054 txfifosz /= tx_channels_count; 3055 3056 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 3057 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 3058 } 3059 3060 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 3061 { 3062 int ret; 3063 3064 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 3065 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 3066 if (ret && (ret != -EINVAL)) { 3067 stmmac_global_err(priv); 3068 return true; 3069 } 3070 3071 return false; 3072 } 3073 3074 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 3075 { 3076 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 3077 &priv->xstats, chan, dir); 3078 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 3079 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3080 struct stmmac_channel *ch = &priv->channel[chan]; 3081 struct napi_struct *rx_napi; 3082 struct napi_struct *tx_napi; 3083 unsigned long flags; 3084 3085 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 3086 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3087 3088 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 3089 if (napi_schedule_prep(rx_napi)) { 3090 spin_lock_irqsave(&ch->lock, flags); 3091 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 3092 spin_unlock_irqrestore(&ch->lock, flags); 3093 __napi_schedule(rx_napi); 3094 } 3095 } 3096 3097 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 3098 if (napi_schedule_prep(tx_napi)) { 3099 spin_lock_irqsave(&ch->lock, flags); 3100 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 3101 spin_unlock_irqrestore(&ch->lock, flags); 3102 __napi_schedule(tx_napi); 3103 } 3104 } 3105 3106 return status; 3107 } 3108 3109 /** 3110 * stmmac_dma_interrupt - DMA ISR 3111 * @priv: driver private structure 3112 * Description: this is the DMA ISR. It is called by the main ISR. 3113 * It calls the dwmac dma routine and schedule poll method in case of some 3114 * work can be done. 3115 */ 3116 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 3117 { 3118 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3119 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3120 u8 channels_to_check = tx_channel_count > rx_channel_count ? 3121 tx_channel_count : rx_channel_count; 3122 int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 3123 u8 chan; 3124 3125 /* Make sure we never check beyond our status buffer. */ 3126 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 3127 channels_to_check = ARRAY_SIZE(status); 3128 3129 for (chan = 0; chan < channels_to_check; chan++) 3130 status[chan] = stmmac_napi_check(priv, chan, 3131 DMA_DIR_RXTX); 3132 3133 for (chan = 0; chan < tx_channel_count; chan++) { 3134 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 3135 /* Try to bump up the dma threshold on this failure */ 3136 stmmac_bump_dma_threshold(priv, chan); 3137 } else if (unlikely(status[chan] == tx_hard_error)) { 3138 stmmac_tx_err(priv, chan); 3139 } 3140 } 3141 } 3142 3143 /** 3144 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 3145 * @priv: driver private structure 3146 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 3147 */ 3148 static void stmmac_mmc_setup(struct stmmac_priv *priv) 3149 { 3150 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 3151 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 3152 3153 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 3154 3155 if (priv->dma_cap.rmon) { 3156 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 3157 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 3158 } else 3159 netdev_info(priv->dev, "No MAC Management Counters available\n"); 3160 } 3161 3162 /** 3163 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 3164 * @priv: driver private structure 3165 * Description: 3166 * new GMAC chip generations have a new register to indicate the 3167 * presence of the optional feature/functions. 3168 * This can be also used to override the value passed through the 3169 * platform and necessary for old MAC10/100 and GMAC chips. 3170 */ 3171 static int stmmac_get_hw_features(struct stmmac_priv *priv) 3172 { 3173 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 3174 } 3175 3176 /** 3177 * stmmac_check_ether_addr - check if the MAC addr is valid 3178 * @priv: driver private structure 3179 * Description: 3180 * it is to verify if the MAC address is valid, in case of failures it 3181 * generates a random MAC address 3182 */ 3183 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 3184 { 3185 u8 addr[ETH_ALEN]; 3186 3187 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 3188 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 3189 if (is_valid_ether_addr(addr)) 3190 eth_hw_addr_set(priv->dev, addr); 3191 else 3192 eth_hw_addr_random(priv->dev); 3193 dev_info(priv->device, "device MAC address %pM\n", 3194 priv->dev->dev_addr); 3195 } 3196 } 3197 3198 int stmmac_get_phy_intf_sel(phy_interface_t interface) 3199 { 3200 int phy_intf_sel = -EINVAL; 3201 3202 if (interface == PHY_INTERFACE_MODE_MII || 3203 interface == PHY_INTERFACE_MODE_GMII) 3204 phy_intf_sel = PHY_INTF_SEL_GMII_MII; 3205 else if (phy_interface_mode_is_rgmii(interface)) 3206 phy_intf_sel = PHY_INTF_SEL_RGMII; 3207 else if (interface == PHY_INTERFACE_MODE_RMII) 3208 phy_intf_sel = PHY_INTF_SEL_RMII; 3209 else if (interface == PHY_INTERFACE_MODE_REVMII) 3210 phy_intf_sel = PHY_INTF_SEL_REVMII; 3211 3212 return phy_intf_sel; 3213 } 3214 EXPORT_SYMBOL_GPL(stmmac_get_phy_intf_sel); 3215 3216 static int stmmac_prereset_configure(struct stmmac_priv *priv) 3217 { 3218 struct plat_stmmacenet_data *plat_dat = priv->plat; 3219 phy_interface_t interface; 3220 struct phylink_pcs *pcs; 3221 int phy_intf_sel, ret; 3222 3223 if (!plat_dat->set_phy_intf_sel) 3224 return 0; 3225 3226 interface = plat_dat->phy_interface; 3227 3228 /* Check whether this mode uses a PCS */ 3229 pcs = stmmac_mac_select_pcs(&priv->phylink_config, interface); 3230 if (priv->integrated_pcs && pcs == &priv->integrated_pcs->pcs) { 3231 /* Request the phy_intf_sel from the integrated PCS */ 3232 phy_intf_sel = stmmac_integrated_pcs_get_phy_intf_sel(pcs, 3233 interface); 3234 } else { 3235 phy_intf_sel = stmmac_get_phy_intf_sel(interface); 3236 } 3237 3238 if (phy_intf_sel < 0) { 3239 netdev_err(priv->dev, 3240 "failed to get phy_intf_sel for %s: %pe\n", 3241 phy_modes(interface), ERR_PTR(phy_intf_sel)); 3242 return phy_intf_sel; 3243 } 3244 3245 ret = plat_dat->set_phy_intf_sel(plat_dat->bsp_priv, phy_intf_sel); 3246 if (ret == -EINVAL) 3247 netdev_err(priv->dev, "platform does not support %s\n", 3248 phy_modes(interface)); 3249 else if (ret < 0) 3250 netdev_err(priv->dev, 3251 "platform failed to set interface %s: %pe\n", 3252 phy_modes(interface), ERR_PTR(ret)); 3253 3254 return ret; 3255 } 3256 3257 /** 3258 * stmmac_init_dma_engine - DMA init. 3259 * @priv: driver private structure 3260 * Description: 3261 * It inits the DMA invoking the specific MAC/GMAC callback. 3262 * Some DMA parameters can be passed from the platform; 3263 * in case of these are not passed a default is kept for the MAC or GMAC. 3264 */ 3265 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 3266 { 3267 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3268 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3269 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 3270 struct stmmac_rx_queue *rx_q; 3271 struct stmmac_tx_queue *tx_q; 3272 int ret = 0; 3273 u8 chan; 3274 3275 ret = stmmac_prereset_configure(priv); 3276 if (ret) 3277 return ret; 3278 3279 ret = stmmac_reset(priv); 3280 if (ret) { 3281 netdev_err(priv->dev, "Failed to reset the dma\n"); 3282 return ret; 3283 } 3284 3285 /* DMA Configuration */ 3286 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); 3287 3288 if (priv->plat->axi) 3289 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3290 3291 /* DMA CSR Channel configuration */ 3292 for (chan = 0; chan < dma_csr_ch; chan++) { 3293 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3294 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3295 } 3296 3297 /* DMA RX Channel Configuration */ 3298 for (chan = 0; chan < rx_channels_count; chan++) { 3299 rx_q = &priv->dma_conf.rx_queue[chan]; 3300 3301 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3302 rx_q->dma_rx_phy, chan); 3303 3304 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 3305 rx_q->buf_alloc_num); 3306 } 3307 3308 /* DMA TX Channel Configuration */ 3309 for (chan = 0; chan < tx_channels_count; chan++) { 3310 tx_q = &priv->dma_conf.tx_queue[chan]; 3311 3312 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3313 tx_q->dma_tx_phy, chan); 3314 3315 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 3316 } 3317 3318 return ret; 3319 } 3320 3321 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3322 { 3323 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3324 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3325 struct stmmac_channel *ch; 3326 struct napi_struct *napi; 3327 3328 if (!tx_coal_timer) 3329 return; 3330 3331 ch = &priv->channel[queue]; 3332 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3333 3334 /* Arm timer only if napi is not already scheduled. 3335 * Try to cancel any timer if napi is scheduled, timer will be armed 3336 * again in the next scheduled napi. 3337 */ 3338 if (unlikely(!napi_is_scheduled(napi))) { 3339 if (unlikely(!(hrtimer_active(&tx_q->txtimer)))) 3340 hrtimer_start(&tx_q->txtimer, 3341 STMMAC_COAL_TIMER(tx_coal_timer), 3342 HRTIMER_MODE_REL); 3343 } else { 3344 hrtimer_try_to_cancel(&tx_q->txtimer); 3345 } 3346 } 3347 3348 /** 3349 * stmmac_tx_timer - mitigation sw timer for tx. 3350 * @t: data pointer 3351 * Description: 3352 * This is the timer handler to directly invoke the stmmac_tx_clean. 3353 */ 3354 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3355 { 3356 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3357 struct stmmac_priv *priv = tx_q->priv_data; 3358 struct stmmac_channel *ch; 3359 struct napi_struct *napi; 3360 3361 ch = &priv->channel[tx_q->queue_index]; 3362 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3363 3364 if (likely(napi_schedule_prep(napi))) { 3365 unsigned long flags; 3366 3367 spin_lock_irqsave(&ch->lock, flags); 3368 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3369 spin_unlock_irqrestore(&ch->lock, flags); 3370 __napi_schedule(napi); 3371 } 3372 3373 return HRTIMER_NORESTART; 3374 } 3375 3376 /** 3377 * stmmac_init_coalesce - init mitigation options. 3378 * @priv: driver private structure 3379 * Description: 3380 * This inits the coalesce parameters: i.e. timer rate, 3381 * timer handler and default threshold used for enabling the 3382 * interrupt on completion bit. 3383 */ 3384 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3385 { 3386 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3387 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3388 u8 chan; 3389 3390 for (chan = 0; chan < tx_channel_count; chan++) { 3391 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3392 3393 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3394 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3395 3396 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3397 } 3398 3399 for (chan = 0; chan < rx_channel_count; chan++) 3400 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3401 } 3402 3403 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3404 { 3405 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3406 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3407 u8 chan; 3408 3409 /* set TX ring length */ 3410 for (chan = 0; chan < tx_channels_count; chan++) 3411 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3412 (priv->dma_conf.dma_tx_size - 1), chan); 3413 3414 /* set RX ring length */ 3415 for (chan = 0; chan < rx_channels_count; chan++) 3416 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3417 (priv->dma_conf.dma_rx_size - 1), chan); 3418 } 3419 3420 /** 3421 * stmmac_set_tx_queue_weight - Set TX queue weight 3422 * @priv: driver private structure 3423 * Description: It is used for setting TX queues weight 3424 */ 3425 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3426 { 3427 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3428 u32 weight; 3429 u8 queue; 3430 3431 for (queue = 0; queue < tx_queues_count; queue++) { 3432 weight = priv->plat->tx_queues_cfg[queue].weight; 3433 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3434 } 3435 } 3436 3437 /** 3438 * stmmac_configure_cbs - Configure CBS in TX queue 3439 * @priv: driver private structure 3440 * Description: It is used for configuring CBS in AVB TX queues 3441 */ 3442 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3443 { 3444 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3445 u32 mode_to_use; 3446 u8 queue; 3447 3448 /* queue 0 is reserved for legacy traffic */ 3449 for (queue = 1; queue < tx_queues_count; queue++) { 3450 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3451 if (mode_to_use == MTL_QUEUE_DCB) 3452 continue; 3453 3454 stmmac_config_cbs(priv, priv->hw, 3455 priv->plat->tx_queues_cfg[queue].send_slope, 3456 priv->plat->tx_queues_cfg[queue].idle_slope, 3457 priv->plat->tx_queues_cfg[queue].high_credit, 3458 priv->plat->tx_queues_cfg[queue].low_credit, 3459 queue); 3460 } 3461 } 3462 3463 /** 3464 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3465 * @priv: driver private structure 3466 * Description: It is used for mapping RX queues to RX dma channels 3467 */ 3468 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3469 { 3470 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3471 u8 queue; 3472 u32 chan; 3473 3474 for (queue = 0; queue < rx_queues_count; queue++) { 3475 chan = priv->plat->rx_queues_cfg[queue].chan; 3476 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3477 } 3478 } 3479 3480 /** 3481 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3482 * @priv: driver private structure 3483 * Description: It is used for configuring the RX Queue Priority 3484 */ 3485 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3486 { 3487 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3488 u8 queue; 3489 u32 prio; 3490 3491 for (queue = 0; queue < rx_queues_count; queue++) { 3492 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3493 continue; 3494 3495 prio = priv->plat->rx_queues_cfg[queue].prio; 3496 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3497 } 3498 } 3499 3500 /** 3501 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3502 * @priv: driver private structure 3503 * Description: It is used for configuring the TX Queue Priority 3504 */ 3505 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3506 { 3507 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3508 u8 queue; 3509 u32 prio; 3510 3511 for (queue = 0; queue < tx_queues_count; queue++) { 3512 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3513 continue; 3514 3515 prio = priv->plat->tx_queues_cfg[queue].prio; 3516 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3517 } 3518 } 3519 3520 /** 3521 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3522 * @priv: driver private structure 3523 * Description: It is used for configuring the RX queue routing 3524 */ 3525 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3526 { 3527 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3528 u8 packet; 3529 u8 queue; 3530 3531 for (queue = 0; queue < rx_queues_count; queue++) { 3532 /* no specific packet type routing specified for the queue */ 3533 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3534 continue; 3535 3536 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3537 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3538 } 3539 } 3540 3541 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3542 { 3543 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3544 priv->rss.enable = false; 3545 return; 3546 } 3547 3548 if (priv->dev->features & NETIF_F_RXHASH) 3549 priv->rss.enable = true; 3550 else 3551 priv->rss.enable = false; 3552 3553 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3554 priv->plat->rx_queues_to_use); 3555 } 3556 3557 /** 3558 * stmmac_mtl_configuration - Configure MTL 3559 * @priv: driver private structure 3560 * Description: It is used for configuring MTL 3561 */ 3562 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3563 { 3564 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3565 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3566 3567 if (tx_queues_count > 1) 3568 stmmac_set_tx_queue_weight(priv); 3569 3570 /* Configure MTL RX algorithms */ 3571 if (rx_queues_count > 1) 3572 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3573 priv->plat->rx_sched_algorithm); 3574 3575 /* Configure MTL TX algorithms */ 3576 if (tx_queues_count > 1) 3577 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3578 priv->plat->tx_sched_algorithm); 3579 3580 /* Configure CBS in AVB TX queues */ 3581 if (tx_queues_count > 1) 3582 stmmac_configure_cbs(priv); 3583 3584 /* Map RX MTL to DMA channels */ 3585 stmmac_rx_queue_dma_chan_map(priv); 3586 3587 /* Enable MAC RX Queues */ 3588 stmmac_mac_enable_rx_queues(priv); 3589 3590 /* Set RX priorities */ 3591 if (rx_queues_count > 1) 3592 stmmac_mac_config_rx_queues_prio(priv); 3593 3594 /* Set TX priorities */ 3595 if (tx_queues_count > 1) 3596 stmmac_mac_config_tx_queues_prio(priv); 3597 3598 /* Set RX routing */ 3599 if (rx_queues_count > 1) 3600 stmmac_mac_config_rx_queues_routing(priv); 3601 3602 /* Receive Side Scaling */ 3603 if (rx_queues_count > 1) 3604 stmmac_mac_config_rss(priv); 3605 } 3606 3607 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3608 { 3609 if (priv->dma_cap.asp) { 3610 netdev_info(priv->dev, "Enabling Safety Features\n"); 3611 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3612 priv->plat->safety_feat_cfg); 3613 } else { 3614 netdev_info(priv->dev, "No Safety Features support found\n"); 3615 } 3616 } 3617 3618 /* STM32MP25xx (dwmac v5.3) states "Do not enable time-based scheduling for 3619 * channels on which the TSO feature is enabled." If we have a skb for a 3620 * channel which has TBS enabled, fall back to software GSO. 3621 */ 3622 static bool stmmac_tso_channel_permitted(struct stmmac_priv *priv, 3623 unsigned int chan) 3624 { 3625 /* TSO and TBS cannot co-exist */ 3626 return !(priv->dma_conf.tx_queue[chan].tbs & STMMAC_TBS_AVAIL); 3627 } 3628 3629 /** 3630 * stmmac_hw_setup - setup mac in a usable state. 3631 * @dev : pointer to the device structure. 3632 * Description: 3633 * this is the main function to setup the HW in a usable state because the 3634 * dma engine is reset, the core registers are configured (e.g. AXI, 3635 * Checksum features, timers). The DMA is ready to start receiving and 3636 * transmitting. 3637 * Return value: 3638 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3639 * file on failure. 3640 */ 3641 static int stmmac_hw_setup(struct net_device *dev) 3642 { 3643 struct stmmac_priv *priv = netdev_priv(dev); 3644 u8 rx_cnt = priv->plat->rx_queues_to_use; 3645 u8 tx_cnt = priv->plat->tx_queues_to_use; 3646 bool sph_en; 3647 u8 chan; 3648 int ret; 3649 3650 /* Make sure RX clock is enabled */ 3651 if (priv->hw->phylink_pcs) 3652 phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs); 3653 3654 /* Note that clk_rx_i must be running for reset to complete. This 3655 * clock may also be required when setting the MAC address. 3656 * 3657 * Block the receive clock stop for LPI mode at the PHY in case 3658 * the link is established with EEE mode active. 3659 */ 3660 phylink_rx_clk_stop_block(priv->phylink); 3661 3662 /* DMA initialization and SW reset */ 3663 ret = stmmac_init_dma_engine(priv); 3664 if (ret < 0) { 3665 phylink_rx_clk_stop_unblock(priv->phylink); 3666 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3667 __func__); 3668 return ret; 3669 } 3670 3671 /* Copy the MAC addr into the HW */ 3672 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3673 phylink_rx_clk_stop_unblock(priv->phylink); 3674 3675 /* Initialize the MAC Core */ 3676 stmmac_core_init(priv, priv->hw, dev); 3677 3678 /* Initialize MTL*/ 3679 stmmac_mtl_configuration(priv); 3680 3681 /* Apply the RX packet parser table */ 3682 if (priv->tc_entries) { 3683 ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries, 3684 priv->tc_entries_max); 3685 if (ret) 3686 return ret; 3687 } 3688 3689 /* Initialize Safety Features */ 3690 stmmac_safety_feat_configuration(priv); 3691 3692 ret = stmmac_rx_ipc(priv, priv->hw); 3693 if (!ret) { 3694 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3695 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3696 priv->hw->rx_csum = 0; 3697 } 3698 3699 /* Enable the MAC Rx/Tx */ 3700 stmmac_mac_set(priv, priv->ioaddr, true); 3701 3702 /* Set the HW DMA mode and the COE */ 3703 stmmac_dma_operation_mode(priv); 3704 3705 stmmac_mmc_setup(priv); 3706 3707 if (priv->use_riwt) { 3708 u32 queue; 3709 3710 for (queue = 0; queue < rx_cnt; queue++) { 3711 if (!priv->rx_riwt[queue]) 3712 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3713 3714 stmmac_rx_watchdog(priv, priv->ioaddr, 3715 priv->rx_riwt[queue], queue); 3716 } 3717 } 3718 3719 /* set TX and RX rings length */ 3720 stmmac_set_rings_length(priv); 3721 3722 /* Enable TSO */ 3723 if (priv->dma_cap.tsoen && priv->plat->flags & STMMAC_FLAG_TSO_EN) { 3724 for (chan = 0; chan < tx_cnt; chan++) { 3725 if (!stmmac_tso_channel_permitted(priv, chan)) 3726 continue; 3727 3728 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3729 } 3730 } 3731 3732 /* Enable Split Header */ 3733 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 3734 for (chan = 0; chan < rx_cnt; chan++) 3735 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3736 3737 3738 /* VLAN Tag Insertion */ 3739 if (priv->dma_cap.vlins) 3740 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3741 3742 /* TBS */ 3743 for (chan = 0; chan < tx_cnt; chan++) { 3744 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3745 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3746 3747 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3748 } 3749 3750 /* Configure real RX and TX queues */ 3751 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3752 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3753 3754 /* Start the ball rolling... */ 3755 stmmac_start_all_dma(priv); 3756 3757 phylink_rx_clk_stop_block(priv->phylink); 3758 stmmac_set_hw_vlan_mode(priv, priv->hw); 3759 phylink_rx_clk_stop_unblock(priv->phylink); 3760 3761 return 0; 3762 } 3763 3764 static void stmmac_free_irq(struct net_device *dev, 3765 enum request_irq_err irq_err, int irq_idx) 3766 { 3767 struct stmmac_priv *priv = netdev_priv(dev); 3768 struct stmmac_msi *msi = priv->msi; 3769 int j; 3770 3771 switch (irq_err) { 3772 case REQ_IRQ_ERR_ALL: 3773 irq_idx = priv->plat->tx_queues_to_use; 3774 fallthrough; 3775 case REQ_IRQ_ERR_TX: 3776 for (j = irq_idx - 1; msi && j >= 0; j--) { 3777 if (msi->tx_irq[j] > 0) { 3778 irq_set_affinity_hint(msi->tx_irq[j], NULL); 3779 free_irq(msi->tx_irq[j], 3780 &priv->dma_conf.tx_queue[j]); 3781 } 3782 } 3783 irq_idx = priv->plat->rx_queues_to_use; 3784 fallthrough; 3785 case REQ_IRQ_ERR_RX: 3786 for (j = irq_idx - 1; msi && j >= 0; j--) { 3787 if (msi->rx_irq[j] > 0) { 3788 irq_set_affinity_hint(msi->rx_irq[j], NULL); 3789 free_irq(msi->rx_irq[j], 3790 &priv->dma_conf.rx_queue[j]); 3791 } 3792 } 3793 3794 if (msi && msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) 3795 free_irq(msi->sfty_ue_irq, dev); 3796 fallthrough; 3797 case REQ_IRQ_ERR_SFTY_UE: 3798 if (msi && msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) 3799 free_irq(msi->sfty_ce_irq, dev); 3800 fallthrough; 3801 case REQ_IRQ_ERR_SFTY_CE: 3802 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3803 free_irq(priv->wol_irq, dev); 3804 fallthrough; 3805 case REQ_IRQ_ERR_SFTY: 3806 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) 3807 free_irq(priv->sfty_irq, dev); 3808 fallthrough; 3809 case REQ_IRQ_ERR_WOL: 3810 free_irq(dev->irq, dev); 3811 fallthrough; 3812 case REQ_IRQ_ERR_MAC: 3813 case REQ_IRQ_ERR_NO: 3814 /* If MAC IRQ request error, no more IRQ to free */ 3815 break; 3816 } 3817 } 3818 3819 static int stmmac_msi_init(struct stmmac_priv *priv, 3820 struct stmmac_resources *res) 3821 { 3822 int i; 3823 3824 priv->msi = devm_kmalloc(priv->device, sizeof(*priv->msi), GFP_KERNEL); 3825 if (!priv->msi) 3826 return -ENOMEM; 3827 3828 priv->msi->sfty_ce_irq = res->sfty_ce_irq; 3829 priv->msi->sfty_ue_irq = res->sfty_ue_irq; 3830 3831 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 3832 priv->msi->rx_irq[i] = res->rx_irq[i]; 3833 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 3834 priv->msi->tx_irq[i] = res->tx_irq[i]; 3835 3836 return 0; 3837 } 3838 3839 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3840 { 3841 struct stmmac_priv *priv = netdev_priv(dev); 3842 struct stmmac_msi *msi = priv->msi; 3843 enum request_irq_err irq_err; 3844 int irq_idx = 0; 3845 char *int_name; 3846 int ret; 3847 int i; 3848 3849 /* For common interrupt */ 3850 int_name = msi->int_name_mac; 3851 sprintf(int_name, "%s:%s", dev->name, "mac"); 3852 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3853 0, int_name, dev); 3854 if (unlikely(ret < 0)) { 3855 netdev_err(priv->dev, 3856 "%s: alloc mac MSI %d (error: %d)\n", 3857 __func__, dev->irq, ret); 3858 irq_err = REQ_IRQ_ERR_MAC; 3859 goto irq_error; 3860 } 3861 3862 /* Request the Wake IRQ in case of another line 3863 * is used for WoL 3864 */ 3865 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3866 int_name = msi->int_name_wol; 3867 sprintf(int_name, "%s:%s", dev->name, "wol"); 3868 ret = request_irq(priv->wol_irq, 3869 stmmac_mac_interrupt, 3870 0, int_name, dev); 3871 if (unlikely(ret < 0)) { 3872 netdev_err(priv->dev, 3873 "%s: alloc wol MSI %d (error: %d)\n", 3874 __func__, priv->wol_irq, ret); 3875 irq_err = REQ_IRQ_ERR_WOL; 3876 goto irq_error; 3877 } 3878 } 3879 3880 /* Request the common Safety Feature Correctible/Uncorrectible 3881 * Error line in case of another line is used 3882 */ 3883 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3884 int_name = msi->int_name_sfty; 3885 sprintf(int_name, "%s:%s", dev->name, "safety"); 3886 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3887 0, int_name, dev); 3888 if (unlikely(ret < 0)) { 3889 netdev_err(priv->dev, 3890 "%s: alloc sfty MSI %d (error: %d)\n", 3891 __func__, priv->sfty_irq, ret); 3892 irq_err = REQ_IRQ_ERR_SFTY; 3893 goto irq_error; 3894 } 3895 } 3896 3897 /* Request the Safety Feature Correctible Error line in 3898 * case of another line is used 3899 */ 3900 if (msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) { 3901 int_name = msi->int_name_sfty_ce; 3902 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3903 ret = request_irq(msi->sfty_ce_irq, 3904 stmmac_safety_interrupt, 3905 0, int_name, dev); 3906 if (unlikely(ret < 0)) { 3907 netdev_err(priv->dev, 3908 "%s: alloc sfty ce MSI %d (error: %d)\n", 3909 __func__, msi->sfty_ce_irq, ret); 3910 irq_err = REQ_IRQ_ERR_SFTY_CE; 3911 goto irq_error; 3912 } 3913 } 3914 3915 /* Request the Safety Feature Uncorrectible Error line in 3916 * case of another line is used 3917 */ 3918 if (msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) { 3919 int_name = msi->int_name_sfty_ue; 3920 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3921 ret = request_irq(msi->sfty_ue_irq, 3922 stmmac_safety_interrupt, 3923 0, int_name, dev); 3924 if (unlikely(ret < 0)) { 3925 netdev_err(priv->dev, 3926 "%s: alloc sfty ue MSI %d (error: %d)\n", 3927 __func__, msi->sfty_ue_irq, ret); 3928 irq_err = REQ_IRQ_ERR_SFTY_UE; 3929 goto irq_error; 3930 } 3931 } 3932 3933 /* Request Rx MSI irq */ 3934 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3935 if (i >= MTL_MAX_RX_QUEUES) 3936 break; 3937 if (msi->rx_irq[i] == 0) 3938 continue; 3939 3940 int_name = msi->int_name_rx_irq[i]; 3941 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3942 ret = request_irq(msi->rx_irq[i], 3943 stmmac_msi_intr_rx, 3944 0, int_name, &priv->dma_conf.rx_queue[i]); 3945 if (unlikely(ret < 0)) { 3946 netdev_err(priv->dev, 3947 "%s: alloc rx-%d MSI %d (error: %d)\n", 3948 __func__, i, msi->rx_irq[i], ret); 3949 irq_err = REQ_IRQ_ERR_RX; 3950 irq_idx = i; 3951 goto irq_error; 3952 } 3953 irq_set_affinity_hint(msi->rx_irq[i], 3954 cpumask_of(i % num_online_cpus())); 3955 } 3956 3957 /* Request Tx MSI irq */ 3958 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3959 if (i >= MTL_MAX_TX_QUEUES) 3960 break; 3961 if (msi->tx_irq[i] == 0) 3962 continue; 3963 3964 int_name = msi->int_name_tx_irq[i]; 3965 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3966 ret = request_irq(msi->tx_irq[i], 3967 stmmac_msi_intr_tx, 3968 0, int_name, &priv->dma_conf.tx_queue[i]); 3969 if (unlikely(ret < 0)) { 3970 netdev_err(priv->dev, 3971 "%s: alloc tx-%d MSI %d (error: %d)\n", 3972 __func__, i, msi->tx_irq[i], ret); 3973 irq_err = REQ_IRQ_ERR_TX; 3974 irq_idx = i; 3975 goto irq_error; 3976 } 3977 irq_set_affinity_hint(msi->tx_irq[i], 3978 cpumask_of(i % num_online_cpus())); 3979 } 3980 3981 return 0; 3982 3983 irq_error: 3984 stmmac_free_irq(dev, irq_err, irq_idx); 3985 return ret; 3986 } 3987 3988 static int stmmac_request_irq_single(struct net_device *dev) 3989 { 3990 struct stmmac_priv *priv = netdev_priv(dev); 3991 enum request_irq_err irq_err; 3992 int ret; 3993 3994 ret = request_irq(dev->irq, stmmac_interrupt, 3995 IRQF_SHARED, dev->name, dev); 3996 if (unlikely(ret < 0)) { 3997 netdev_err(priv->dev, 3998 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3999 __func__, dev->irq, ret); 4000 irq_err = REQ_IRQ_ERR_MAC; 4001 goto irq_error; 4002 } 4003 4004 /* Request the Wake IRQ in case of another line 4005 * is used for WoL 4006 */ 4007 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 4008 ret = request_irq(priv->wol_irq, stmmac_interrupt, 4009 IRQF_SHARED, dev->name, dev); 4010 if (unlikely(ret < 0)) { 4011 netdev_err(priv->dev, 4012 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 4013 __func__, priv->wol_irq, ret); 4014 irq_err = REQ_IRQ_ERR_WOL; 4015 goto irq_error; 4016 } 4017 } 4018 4019 /* Request the common Safety Feature Correctible/Uncorrectible 4020 * Error line in case of another line is used 4021 */ 4022 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 4023 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 4024 IRQF_SHARED, dev->name, dev); 4025 if (unlikely(ret < 0)) { 4026 netdev_err(priv->dev, 4027 "%s: ERROR: allocating the sfty IRQ %d (%d)\n", 4028 __func__, priv->sfty_irq, ret); 4029 irq_err = REQ_IRQ_ERR_SFTY; 4030 goto irq_error; 4031 } 4032 } 4033 4034 return 0; 4035 4036 irq_error: 4037 stmmac_free_irq(dev, irq_err, 0); 4038 return ret; 4039 } 4040 4041 static int stmmac_request_irq(struct net_device *dev) 4042 { 4043 struct stmmac_priv *priv = netdev_priv(dev); 4044 int ret; 4045 4046 /* Request the IRQ lines */ 4047 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 4048 ret = stmmac_request_irq_multi_msi(dev); 4049 else 4050 ret = stmmac_request_irq_single(dev); 4051 4052 return ret; 4053 } 4054 4055 /** 4056 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 4057 * @priv: driver private structure 4058 * @mtu: MTU to setup the dma queue and buf with 4059 * Description: Allocate and generate a dma_conf based on the provided MTU. 4060 * Allocate the Tx/Rx DMA queue and init them. 4061 * Return value: 4062 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 4063 */ 4064 static struct stmmac_dma_conf * 4065 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 4066 { 4067 struct stmmac_dma_conf *dma_conf; 4068 int bfsize, len, ret; 4069 u8 chan; 4070 4071 dma_conf = kzalloc_obj(*dma_conf); 4072 if (!dma_conf) { 4073 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 4074 __func__); 4075 return ERR_PTR(-ENOMEM); 4076 } 4077 4078 len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; 4079 4080 /* Returns 0 or BUF_SIZE_16KiB if len > 8KiB and dwmac4 or ring mode */ 4081 bfsize = stmmac_set_16kib_bfsize(priv, len); 4082 if (bfsize < 0) 4083 bfsize = 0; 4084 4085 if (bfsize < BUF_SIZE_16KiB) 4086 bfsize = stmmac_set_bfsize(len); 4087 4088 dma_conf->dma_buf_sz = bfsize; 4089 /* Chose the tx/rx size from the already defined one in the 4090 * priv struct. (if defined) 4091 */ 4092 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 4093 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 4094 4095 if (!dma_conf->dma_tx_size) 4096 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 4097 if (!dma_conf->dma_rx_size) 4098 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 4099 4100 /* Earlier check for TBS */ 4101 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 4102 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 4103 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 4104 4105 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 4106 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 4107 } 4108 4109 ret = alloc_dma_desc_resources(priv, dma_conf); 4110 if (ret < 0) { 4111 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 4112 __func__); 4113 goto alloc_error; 4114 } 4115 4116 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 4117 if (ret < 0) { 4118 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 4119 __func__); 4120 goto init_error; 4121 } 4122 4123 return dma_conf; 4124 4125 init_error: 4126 free_dma_desc_resources(priv, dma_conf); 4127 alloc_error: 4128 kfree(dma_conf); 4129 return ERR_PTR(ret); 4130 } 4131 4132 /** 4133 * __stmmac_open - open entry point of the driver 4134 * @dev : pointer to the device structure. 4135 * @dma_conf : structure to take the dma data 4136 * Description: 4137 * This function is the open entry point of the driver. 4138 * Return value: 4139 * 0 on success and an appropriate (-)ve integer as defined in errno.h 4140 * file on failure. 4141 */ 4142 static int __stmmac_open(struct net_device *dev, 4143 struct stmmac_dma_conf *dma_conf) 4144 { 4145 struct stmmac_priv *priv = netdev_priv(dev); 4146 u8 chan; 4147 int ret; 4148 4149 for (int i = 0; i < priv->plat->tx_queues_to_use; i++) 4150 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 4151 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 4152 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 4153 4154 /* The PHY is suspended when the interface is reopened without 4155 * disconnecting the PHY, e.g. on MTU change. IEEE 802.3 allows PHYs 4156 * to stop their receive clock while powered down, but the DMA 4157 * software reset in stmmac_hw_setup() requires a running receive 4158 * clock, and phylink_start() below resumes the PHY only after the 4159 * hardware setup. Resume a suspended PHY here first. 4160 */ 4161 phylink_prepare_resume(priv->phylink); 4162 4163 stmmac_reset_queues_param(priv); 4164 4165 ret = stmmac_hw_setup(dev); 4166 if (ret < 0) { 4167 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 4168 goto init_error; 4169 } 4170 4171 stmmac_setup_ptp(priv); 4172 4173 stmmac_init_coalesce(priv); 4174 4175 phylink_start(priv->phylink); 4176 4177 stmmac_vlan_restore(priv); 4178 4179 ret = stmmac_request_irq(dev); 4180 if (ret) 4181 goto irq_error; 4182 4183 stmmac_enable_all_queues(priv); 4184 netif_tx_start_all_queues(priv->dev); 4185 stmmac_enable_all_dma_irq(priv); 4186 4187 return 0; 4188 4189 irq_error: 4190 phylink_stop(priv->phylink); 4191 4192 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4193 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4194 4195 stmmac_release_ptp(priv); 4196 init_error: 4197 return ret; 4198 } 4199 4200 static int stmmac_open(struct net_device *dev) 4201 { 4202 struct stmmac_priv *priv = netdev_priv(dev); 4203 struct stmmac_dma_conf *dma_conf; 4204 int ret; 4205 4206 /* Initialise the tx lpi timer, converting from msec to usec */ 4207 if (!priv->tx_lpi_timer) 4208 priv->tx_lpi_timer = eee_timer * 1000; 4209 4210 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 4211 if (IS_ERR(dma_conf)) 4212 return PTR_ERR(dma_conf); 4213 4214 ret = pm_runtime_resume_and_get(priv->device); 4215 if (ret < 0) 4216 goto err_dma_resources; 4217 4218 ret = stmmac_init_phy(dev); 4219 if (ret) 4220 goto err_runtime_pm; 4221 4222 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 4223 ret = stmmac_legacy_serdes_power_up(priv); 4224 if (ret < 0) 4225 goto err_disconnect_phy; 4226 } 4227 4228 ret = __stmmac_open(dev, dma_conf); 4229 if (ret) 4230 goto err_serdes; 4231 4232 kfree(dma_conf); 4233 4234 /* We may have called phylink_speed_down before */ 4235 phylink_speed_up(priv->phylink); 4236 4237 return ret; 4238 4239 err_serdes: 4240 stmmac_legacy_serdes_power_down(priv); 4241 err_disconnect_phy: 4242 phylink_disconnect_phy(priv->phylink); 4243 err_runtime_pm: 4244 pm_runtime_put(priv->device); 4245 err_dma_resources: 4246 free_dma_desc_resources(priv, dma_conf); 4247 kfree(dma_conf); 4248 return ret; 4249 } 4250 4251 static void __stmmac_release(struct net_device *dev) 4252 { 4253 struct stmmac_priv *priv = netdev_priv(dev); 4254 u8 chan; 4255 4256 /* Stop and disconnect the PHY */ 4257 phylink_stop(priv->phylink); 4258 4259 stmmac_disable_all_queues(priv); 4260 4261 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4262 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4263 4264 netif_tx_disable(dev); 4265 4266 /* Free the IRQ lines */ 4267 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4268 4269 /* Stop TX/RX DMA and clear the descriptors */ 4270 stmmac_stop_all_dma(priv); 4271 4272 /* Release and free the Rx/Tx resources */ 4273 free_dma_desc_resources(priv, &priv->dma_conf); 4274 4275 stmmac_release_ptp(priv); 4276 4277 if (stmmac_fpe_supported(priv)) 4278 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 4279 } 4280 4281 /** 4282 * stmmac_release - close entry point of the driver 4283 * @dev : device pointer. 4284 * Description: 4285 * This is the stop entry point of the driver. 4286 */ 4287 static int stmmac_release(struct net_device *dev) 4288 { 4289 struct stmmac_priv *priv = netdev_priv(dev); 4290 4291 /* If the PHY or MAC has WoL enabled, then the PHY will not be 4292 * suspended when phylink_stop() is called below. Set the PHY 4293 * to its slowest speed to save power. 4294 */ 4295 if (device_may_wakeup(priv->device)) 4296 phylink_speed_down(priv->phylink, false); 4297 4298 __stmmac_release(dev); 4299 4300 stmmac_legacy_serdes_power_down(priv); 4301 phylink_disconnect_phy(priv->phylink); 4302 pm_runtime_put(priv->device); 4303 4304 return 0; 4305 } 4306 4307 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4308 struct stmmac_tx_queue *tx_q) 4309 { 4310 struct dma_desc *p; 4311 u16 tag = 0x0; 4312 4313 if (!priv->dma_cap.vlins || !skb_vlan_tag_present(skb)) 4314 return false; 4315 4316 tag = skb_vlan_tag_get(skb); 4317 4318 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4319 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4320 else 4321 p = &tx_q->dma_tx[tx_q->cur_tx]; 4322 4323 if (stmmac_set_desc_vlan_tag(priv, p, tag, 0x0, 0x0)) 4324 return false; 4325 4326 stmmac_set_tx_owner(priv, p); 4327 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4328 return true; 4329 } 4330 4331 /** 4332 * stmmac_tso_allocator - close entry point of the driver 4333 * @priv: driver private structure 4334 * @entry: TX queue buffer index 4335 * @des: buffer start address 4336 * @total_len: total length to fill in descriptors 4337 * @last_segment: condition for the last descriptor 4338 * @queue: TX queue index 4339 * Description: 4340 * This function fills descriptor and request new descriptors according to 4341 * buffer length to fill 4342 */ 4343 static void stmmac_tso_allocator(struct stmmac_priv *priv, u32 *entry, 4344 dma_addr_t des, int total_len, 4345 bool last_segment, u32 queue) 4346 { 4347 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4348 struct dma_desc *desc; 4349 u32 buff_size; 4350 int tmp_len; 4351 4352 tmp_len = total_len; 4353 4354 while (tmp_len > 0) { 4355 dma_addr_t curr_addr; 4356 4357 *entry = STMMAC_NEXT_ENTRY(*entry, priv->dma_conf.dma_tx_size); 4358 WARN_ON(tx_q->tx_skbuff[*entry]); 4359 4360 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4361 desc = &tx_q->dma_entx[*entry].basic; 4362 else 4363 desc = &tx_q->dma_tx[*entry]; 4364 4365 curr_addr = des + (total_len - tmp_len); 4366 stmmac_set_desc_addr(priv, desc, curr_addr); 4367 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4368 TSO_MAX_BUFF_SIZE : tmp_len; 4369 4370 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4371 0, 1, 4372 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4373 0, 0); 4374 4375 tmp_len -= TSO_MAX_BUFF_SIZE; 4376 } 4377 } 4378 4379 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4380 { 4381 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4382 4383 /* The own bit must be the latest setting done when prepare the 4384 * descriptor and then barrier is needed to make sure that 4385 * all is coherent before granting the DMA engine. 4386 */ 4387 wmb(); 4388 4389 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx); 4390 } 4391 4392 static void stmmac_set_gso_features(struct net_device *ndev) 4393 { 4394 struct stmmac_priv *priv = netdev_priv(ndev); 4395 const struct stmmac_dma_cfg *dma_cfg; 4396 int txpbl; 4397 4398 if (priv->dma_cap.tsoen) 4399 dev_info(priv->device, "TSO supported\n"); 4400 4401 if (!(priv->plat->flags & STMMAC_FLAG_TSO_EN)) 4402 return; 4403 4404 if (!priv->dma_cap.tsoen) { 4405 dev_warn(priv->device, "platform requests unsupported TSO\n"); 4406 return; 4407 } 4408 4409 /* FIXME: 4410 * STM32MP151 (v4.2 userver v4.0) states that TxPBL must be >= 4. It 4411 * is not clear whether PBLx8 (which multiplies the PBL value by 8) 4412 * influences this. 4413 */ 4414 dma_cfg = priv->plat->dma_cfg; 4415 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl; 4416 if (txpbl < 4) { 4417 dev_warn(priv->device, "txpbl(%d) is too low for TSO\n", txpbl); 4418 return; 4419 } 4420 4421 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4422 if (priv->plat->core_type == DWMAC_CORE_GMAC4) 4423 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 4424 4425 dev_info(priv->device, "TSO feature enabled\n"); 4426 } 4427 4428 static size_t stmmac_tso_header_size(struct sk_buff *skb) 4429 { 4430 size_t size; 4431 4432 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4433 size = skb_transport_offset(skb) + sizeof(struct udphdr); 4434 else 4435 size = skb_tcp_all_headers(skb); 4436 4437 return size; 4438 } 4439 4440 /* STM32MP151 (dwmac v4.2) and STM32MP25xx (dwmac v5.3) states for TDES2 normal 4441 * (read format) descriptor that the maximum header length supported for the 4442 * TSO feature is 1023 bytes. 4443 * 4444 * While IPv4 is limited to MAC+VLAN+IPv4+ext+TCP+ext = 138 bytes, the IPv6 4445 * extension headers aren't similarly limited. 4446 * 4447 * Fall back to software GSO for these skbs. Also check that the MSS is >= 4448 * the recommended 64 bytes (documented in ETH_DMACxCR register description), 4449 * and that a the header plus MSS is not larger than 16383 (documented in 4450 * "Building the Descriptor and the packet for the TSO feature"). 4451 */ 4452 static bool stmmac_tso_valid_packet(struct sk_buff *skb) 4453 { 4454 size_t header_len = stmmac_tso_header_size(skb); 4455 unsigned int gso_size = skb_shinfo(skb)->gso_size; 4456 4457 return header_len <= 1023 && gso_size >= 64 && 4458 header_len + gso_size < 16383; 4459 } 4460 4461 static int stmmac_tso_get_num_desc(struct stmmac_tx_queue *tx_q, 4462 struct sk_buff *skb, u32 pay_len) 4463 { 4464 int i, ndesc = 1; 4465 4466 /* head payload */ 4467 ndesc += DIV_ROUND_UP(pay_len, TSO_MAX_BUFF_SIZE); 4468 /* frag payload */ 4469 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4470 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4471 4472 ndesc += DIV_ROUND_UP(skb_frag_size(frag), 4473 TSO_MAX_BUFF_SIZE); 4474 } 4475 /* MSS update requires a new descriptor */ 4476 ndesc += !!(skb_shinfo(skb)->gso_size != tx_q->mss); 4477 4478 return ndesc; 4479 } 4480 4481 /** 4482 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4483 * @skb : the socket buffer 4484 * @dev : device pointer 4485 * Description: this is the transmit function that is called on TSO frames 4486 * (support available on GMAC4 and newer chips). 4487 * Diagram below show the ring programming in case of TSO frames: 4488 * 4489 * First Descriptor 4490 * -------- 4491 * | DES0 |---> buffer1 = L2/L3/L4 header 4492 * | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address 4493 * | | width is 32-bit, but we never use it. 4494 * | | Also can be used as the most-significant 8-bits or 16-bits of 4495 * | | buffer1 address pointer if the DMA AXI address width is 40-bit 4496 * | | or 48-bit, and we always use it. 4497 * | DES2 |---> buffer1 len 4498 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4499 * -------- 4500 * -------- 4501 * | DES0 |---> buffer1 = TCP Payload (can continue on next descr...) 4502 * | DES1 |---> same as the First Descriptor 4503 * | DES2 |---> buffer1 len 4504 * | DES3 | 4505 * -------- 4506 * | 4507 * ... 4508 * | 4509 * -------- 4510 * | DES0 |---> buffer1 = Split TCP Payload 4511 * | DES1 |---> same as the First Descriptor 4512 * | DES2 |---> buffer1 len 4513 * | DES3 | 4514 * -------- 4515 * 4516 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4517 */ 4518 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4519 { 4520 unsigned int first_entry, entry, tx_packets, proto_hdr_len; 4521 struct dma_desc *desc, *first, *mss_desc = NULL; 4522 struct stmmac_priv *priv = netdev_priv(dev); 4523 struct stmmac_txq_stats *txq_stats; 4524 int i, first_tx, nfrags, ndesc; 4525 struct stmmac_tx_queue *tx_q; 4526 bool set_ic, is_last_segment; 4527 u32 pay_len, mss, queue; 4528 dma_addr_t des; 4529 u8 hdr; 4530 4531 nfrags = skb_shinfo(skb)->nr_frags; 4532 queue = skb_get_queue_mapping(skb); 4533 4534 tx_q = &priv->dma_conf.tx_queue[queue]; 4535 txq_stats = &priv->xstats.txq_stats[queue]; 4536 first_tx = tx_q->cur_tx; 4537 4538 /* Compute header lengths */ 4539 proto_hdr_len = stmmac_tso_header_size(skb); 4540 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4541 4542 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4543 hdr = sizeof(struct udphdr); 4544 else 4545 hdr = tcp_hdrlen(skb); 4546 4547 ndesc = stmmac_tso_get_num_desc(tx_q, skb, pay_len); 4548 if (unlikely(stmmac_tx_avail(priv, queue) < ndesc)) { 4549 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4550 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4551 queue)); 4552 /* This is a hard error, log it. */ 4553 netdev_err(priv->dev, 4554 "%s: Tx Ring full when queue awake\n", 4555 __func__); 4556 } 4557 return NETDEV_TX_BUSY; 4558 } 4559 4560 mss = skb_shinfo(skb)->gso_size; 4561 4562 /* set new MSS value if needed */ 4563 if (mss != tx_q->mss) { 4564 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4565 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4566 else 4567 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4568 4569 stmmac_set_mss(priv, mss_desc, mss); 4570 tx_q->mss = mss; 4571 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, 4572 priv->dma_conf.dma_tx_size); 4573 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4574 } 4575 4576 if (netif_msg_tx_queued(priv)) { 4577 pr_info("%s: hdrlen %d, hdr_len %u, pay_len %d, mss %d\n", 4578 __func__, hdr, proto_hdr_len, pay_len, mss); 4579 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4580 skb->data_len); 4581 } 4582 4583 first_entry = tx_q->cur_tx; 4584 entry = first_entry; 4585 4586 WARN_ON(tx_q->tx_skbuff[entry]); 4587 4588 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4589 desc = &tx_q->dma_entx[entry].basic; 4590 else 4591 desc = &tx_q->dma_tx[entry]; 4592 first = desc; 4593 4594 /* first descriptor: fill Headers on Buf1 */ 4595 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4596 DMA_TO_DEVICE); 4597 if (dma_mapping_error(priv->device, des)) 4598 goto error; 4599 4600 stmmac_set_desc_addr(priv, first, des); 4601 stmmac_tso_allocator(priv, &entry, des + proto_hdr_len, pay_len, 4602 (nfrags == 0), queue); 4603 4604 /* In case two or more DMA transmit descriptors are allocated for this 4605 * non-paged SKB data, the DMA buffer address should be saved to 4606 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4607 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4608 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4609 * since the tail areas of the DMA buffer can be accessed by DMA engine 4610 * sooner or later. 4611 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4612 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4613 * this DMA buffer right after the DMA engine completely finishes the 4614 * full buffer transmission. 4615 */ 4616 stmmac_set_tx_skb_dma_entry(tx_q, entry, des, skb_headlen(skb), false); 4617 4618 /* Prepare fragments */ 4619 for (i = 0; i < nfrags; i++) { 4620 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4621 4622 des = skb_frag_dma_map(priv->device, frag, 0, 4623 skb_frag_size(frag), 4624 DMA_TO_DEVICE); 4625 if (dma_mapping_error(priv->device, des)) 4626 goto error_dma_unmap; 4627 4628 stmmac_tso_allocator(priv, &entry, des, skb_frag_size(frag), 4629 (i == nfrags - 1), queue); 4630 4631 stmmac_set_tx_skb_dma_entry(tx_q, entry, des, 4632 skb_frag_size(frag), true); 4633 } 4634 tx_q->cur_tx = entry; 4635 4636 stmmac_set_tx_dma_last_segment(tx_q, tx_q->cur_tx); 4637 4638 /* Only the last descriptor gets to point to the skb. */ 4639 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4640 4641 /* Manage tx mitigation */ 4642 tx_packets = CIRC_CNT(tx_q->cur_tx + 1, first_tx, 4643 priv->dma_conf.dma_tx_size); 4644 tx_q->tx_count_frames += tx_packets; 4645 4646 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4647 set_ic = true; 4648 else if (!priv->tx_coal_frames[queue]) 4649 set_ic = false; 4650 else if (!netdev_xmit_more()) 4651 set_ic = true; 4652 else if (tx_packets > priv->tx_coal_frames[queue]) 4653 set_ic = true; 4654 else if ((tx_q->tx_count_frames % 4655 priv->tx_coal_frames[queue]) < tx_packets) 4656 set_ic = true; 4657 else 4658 set_ic = false; 4659 4660 if (set_ic) { 4661 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4662 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4663 else 4664 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4665 4666 tx_q->tx_count_frames = 0; 4667 stmmac_set_tx_ic(priv, desc); 4668 } 4669 4670 /* We've used all descriptors we need for this skb, however, 4671 * advance cur_tx so that it references a fresh descriptor. 4672 * ndo_start_xmit will fill this descriptor the next time it's 4673 * called and stmmac_tx_clean may clean up to this descriptor. 4674 */ 4675 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4676 4677 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4678 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4679 __func__); 4680 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4681 } 4682 4683 u64_stats_update_begin(&txq_stats->q_syncp); 4684 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4685 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4686 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4687 if (set_ic) 4688 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4689 u64_stats_update_end(&txq_stats->q_syncp); 4690 4691 if (priv->sarc_type) 4692 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4693 4694 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4695 priv->hwts_tx_en)) { 4696 /* declare that device is doing timestamping */ 4697 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4698 stmmac_enable_tx_timestamp(priv, first); 4699 } 4700 4701 /* If we only have one entry used, then the first entry is the last 4702 * segment. 4703 */ 4704 is_last_segment = CIRC_CNT(tx_q->cur_tx, first_entry, 4705 priv->dma_conf.dma_tx_size) == 1; 4706 4707 /* Complete the first descriptor before granting the DMA */ 4708 stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, 4709 is_last_segment, hdr / 4, 4710 skb->len - proto_hdr_len); 4711 4712 /* If context desc is used to change MSS */ 4713 if (mss_desc) { 4714 /* Make sure that first descriptor has been completely 4715 * written, including its own bit. This is because MSS is 4716 * actually before first descriptor, so we need to make 4717 * sure that MSS's own bit is the last thing written. 4718 */ 4719 dma_wmb(); 4720 stmmac_set_tx_owner(priv, mss_desc); 4721 } 4722 4723 if (netif_msg_pktdata(priv)) { 4724 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4725 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4726 tx_q->cur_tx, first, nfrags); 4727 pr_info(">>> frame to be transmitted: "); 4728 print_pkt(skb->data, skb_headlen(skb)); 4729 } 4730 4731 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4732 skb_tx_timestamp(skb); 4733 4734 stmmac_flush_tx_descriptors(priv, queue); 4735 stmmac_tx_timer_arm(priv, queue); 4736 4737 return NETDEV_TX_OK; 4738 4739 error_dma_unmap: 4740 for (;;) { 4741 desc = stmmac_get_tx_desc(priv, tx_q, first_entry); 4742 stmmac_release_tx_desc(priv, desc, priv->descriptor_mode); 4743 stmmac_free_tx_buffer(priv, &priv->dma_conf, queue, 4744 first_entry); 4745 if (first_entry == entry) 4746 break; 4747 4748 first_entry = STMMAC_NEXT_ENTRY(first_entry, 4749 priv->dma_conf.dma_tx_size); 4750 } 4751 error: 4752 dev_err(priv->device, "Tx dma map failed\n"); 4753 dev_kfree_skb(skb); 4754 priv->xstats.tx_dropped++; 4755 return NETDEV_TX_OK; 4756 } 4757 4758 /** 4759 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4760 * @skb: socket buffer to check 4761 * 4762 * Check if a packet has an ethertype that will trigger the IP header checks 4763 * and IP/TCP checksum engine of the stmmac core. 4764 * 4765 * Return: true if the ethertype can trigger the checksum engine, false 4766 * otherwise 4767 */ 4768 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4769 { 4770 int depth = 0; 4771 __be16 proto; 4772 4773 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4774 &depth); 4775 4776 return (depth <= ETH_HLEN) && 4777 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4778 } 4779 4780 /** 4781 * stmmac_xmit - Tx entry point of the driver 4782 * @skb : the socket buffer 4783 * @dev : device pointer 4784 * Description : this is the tx entry point of the driver. 4785 * It programs the chain or the ring and supports oversized frames 4786 * and SG feature. 4787 */ 4788 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4789 { 4790 bool enh_desc, has_vlan, set_ic, is_jumbo = false; 4791 struct stmmac_priv *priv = netdev_priv(dev); 4792 unsigned int nopaged_len = skb_headlen(skb); 4793 u32 queue = skb_get_queue_mapping(skb); 4794 int nfrags = skb_shinfo(skb)->nr_frags; 4795 unsigned int first_entry, tx_packets; 4796 struct stmmac_txq_stats *txq_stats; 4797 struct dma_desc *desc, *first_desc; 4798 struct stmmac_tx_queue *tx_q; 4799 int i, csum_insertion = 0; 4800 int entry, first_tx; 4801 dma_addr_t dma_addr; 4802 u32 sdu_len; 4803 4804 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4805 stmmac_stop_sw_lpi(priv); 4806 4807 if (skb_is_gso(skb)) 4808 return stmmac_tso_xmit(skb, dev); 4809 4810 if (priv->est && priv->est->enable && 4811 priv->est->max_sdu[queue]) { 4812 sdu_len = skb->len; 4813 /* Add VLAN tag length if VLAN tag insertion offload is requested */ 4814 if (priv->dma_cap.vlins && skb_vlan_tag_present(skb)) 4815 sdu_len += VLAN_HLEN; 4816 if (sdu_len > priv->est->max_sdu[queue]) { 4817 priv->xstats.max_sdu_txq_drop[queue]++; 4818 goto max_sdu_err; 4819 } 4820 } 4821 4822 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4823 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4824 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4825 queue)); 4826 /* This is a hard error, log it. */ 4827 netdev_err(priv->dev, 4828 "%s: Tx Ring full when queue awake\n", 4829 __func__); 4830 } 4831 return NETDEV_TX_BUSY; 4832 } 4833 4834 tx_q = &priv->dma_conf.tx_queue[queue]; 4835 first_tx = tx_q->cur_tx; 4836 4837 /* Check if VLAN can be inserted by HW */ 4838 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4839 4840 entry = tx_q->cur_tx; 4841 first_entry = entry; 4842 WARN_ON(tx_q->tx_skbuff[first_entry]); 4843 4844 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4845 first_desc = desc; 4846 4847 if (has_vlan) 4848 stmmac_set_desc_vlan(priv, first_desc, STMMAC_VLAN_INSERT); 4849 4850 enh_desc = priv->plat->enh_desc; 4851 /* To program the descriptors according to the size of the frame */ 4852 if (enh_desc) 4853 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4854 4855 csum_insertion = skb->ip_summed == CHECKSUM_PARTIAL; 4856 4857 if (unlikely(is_jumbo)) { 4858 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4859 if (unlikely(entry < 0) && (entry != -EINVAL)) 4860 goto dma_map_err; 4861 } else { 4862 bool last_segment = (nfrags == 0); 4863 4864 dma_addr = dma_map_single(priv->device, skb->data, 4865 nopaged_len, DMA_TO_DEVICE); 4866 if (dma_mapping_error(priv->device, dma_addr)) 4867 goto dma_map_err; 4868 4869 stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr, 4870 nopaged_len, false); 4871 4872 stmmac_set_desc_addr(priv, first_desc, dma_addr); 4873 4874 if (last_segment) 4875 stmmac_set_tx_dma_last_segment(tx_q, first_entry); 4876 4877 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4878 priv->hwts_tx_en)) { 4879 /* declare that device is doing timestamping */ 4880 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4881 stmmac_enable_tx_timestamp(priv, first_desc); 4882 } 4883 4884 /* Prepare the first descriptor without setting the OWN bit */ 4885 stmmac_prepare_tx_desc(priv, first_desc, 1, nopaged_len, 4886 csum_insertion, priv->descriptor_mode, 4887 0, last_segment, skb->len); 4888 } 4889 4890 if (priv->sarc_type) 4891 stmmac_set_desc_sarc(priv, first_desc, priv->sarc_type); 4892 4893 /* STMMAC_TBS_EN can only be set if STMMAC_TBS_AVAIL has already 4894 * been set, which means the underlying type of the descriptors 4895 * will be struct stmmac_edesc. Therefore, it is safe to convert 4896 * the basic descriptor to the enhanced descriptor here. 4897 */ 4898 if (tx_q->tbs & STMMAC_TBS_EN) { 4899 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4900 4901 stmmac_set_desc_tbs(priv, dma_desc_to_edesc(first_desc), 4902 ts.tv_sec, ts.tv_nsec); 4903 } 4904 4905 for (i = 0; i < nfrags; i++) { 4906 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4907 unsigned int frag_size = skb_frag_size(frag); 4908 bool last_segment = (i == (nfrags - 1)); 4909 4910 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4911 WARN_ON(tx_q->tx_skbuff[entry]); 4912 4913 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4914 4915 dma_addr = skb_frag_dma_map(priv->device, frag, 0, frag_size, 4916 DMA_TO_DEVICE); 4917 if (dma_mapping_error(priv->device, dma_addr)) 4918 goto dma_map_err; /* should reuse desc w/o issues */ 4919 4920 stmmac_set_tx_skb_dma_entry(tx_q, entry, dma_addr, frag_size, 4921 true); 4922 stmmac_set_desc_addr(priv, desc, dma_addr); 4923 4924 /* Prepare the descriptor and set the own bit too */ 4925 stmmac_prepare_tx_desc(priv, desc, 0, frag_size, csum_insertion, 4926 priv->descriptor_mode, 1, last_segment, 4927 skb->len); 4928 } 4929 4930 stmmac_set_tx_dma_last_segment(tx_q, entry); 4931 4932 /* Only the last descriptor gets to point to the skb. */ 4933 tx_q->tx_skbuff[entry] = skb; 4934 4935 /* According to the coalesce parameter the IC bit for the latest 4936 * segment is reset and the timer re-started to clean the tx status. 4937 * This approach takes care about the fragments: desc is the first 4938 * element in case of no SG. 4939 */ 4940 tx_packets = CIRC_CNT(entry + 1, first_tx, priv->dma_conf.dma_tx_size); 4941 tx_q->tx_count_frames += tx_packets; 4942 4943 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4944 set_ic = true; 4945 else if (!priv->tx_coal_frames[queue]) 4946 set_ic = false; 4947 else if (!netdev_xmit_more()) 4948 set_ic = true; 4949 else if (tx_packets > priv->tx_coal_frames[queue]) 4950 set_ic = true; 4951 else if ((tx_q->tx_count_frames % 4952 priv->tx_coal_frames[queue]) < tx_packets) 4953 set_ic = true; 4954 else 4955 set_ic = false; 4956 4957 if (set_ic) { 4958 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4959 tx_q->tx_count_frames = 0; 4960 stmmac_set_tx_ic(priv, desc); 4961 } 4962 4963 /* We've used all descriptors we need for this skb, however, 4964 * advance cur_tx so that it references a fresh descriptor. 4965 * ndo_start_xmit will fill this descriptor the next time it's 4966 * called and stmmac_tx_clean may clean up to this descriptor. 4967 */ 4968 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4969 tx_q->cur_tx = entry; 4970 4971 if (netif_msg_pktdata(priv)) { 4972 netdev_dbg(priv->dev, 4973 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4974 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4975 entry, first_desc, nfrags); 4976 4977 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4978 print_pkt(skb->data, skb->len); 4979 } 4980 4981 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4982 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4983 __func__); 4984 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4985 } 4986 4987 txq_stats = &priv->xstats.txq_stats[queue]; 4988 u64_stats_update_begin(&txq_stats->q_syncp); 4989 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4990 if (set_ic) 4991 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4992 u64_stats_update_end(&txq_stats->q_syncp); 4993 4994 /* Set the OWN bit on the first descriptor now that all descriptors 4995 * for this skb are populated. 4996 */ 4997 stmmac_set_tx_owner(priv, first_desc); 4998 4999 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 5000 5001 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5002 skb_tx_timestamp(skb); 5003 stmmac_flush_tx_descriptors(priv, queue); 5004 stmmac_tx_timer_arm(priv, queue); 5005 5006 return NETDEV_TX_OK; 5007 5008 dma_map_err: 5009 netdev_err(priv->dev, "Tx DMA map failed\n"); 5010 max_sdu_err: 5011 dev_kfree_skb(skb); 5012 priv->xstats.tx_dropped++; 5013 return NETDEV_TX_OK; 5014 } 5015 5016 static netdev_features_t stmmac_features_check(struct sk_buff *skb, 5017 struct net_device *dev, 5018 netdev_features_t features) 5019 { 5020 struct stmmac_priv *priv = netdev_priv(dev); 5021 u16 queue = skb_get_queue_mapping(skb); 5022 5023 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 5024 * queues. In that case, checksum offloading for those queues that don't 5025 * support tx coe needs to fallback to software checksum calculation. 5026 * 5027 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 5028 * also have to be checksummed in software. 5029 * 5030 * Note that disabling hardware checksumming also disables TSO. See 5031 * harmonize_features() in net/core/dev.c 5032 */ 5033 if (priv->plat->tx_queues_cfg[queue].coe_unsupported || 5034 !stmmac_has_ip_ethertype(skb)) 5035 features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 5036 5037 if (skb_is_gso(skb)) { 5038 if (!stmmac_tso_channel_permitted(priv, queue) || 5039 !stmmac_tso_valid_packet(skb)) 5040 features &= ~NETIF_F_GSO_MASK; 5041 5042 /* If we are going to be using hardware TSO, always insert 5043 * VLAN tag to SKB payload for TSO frames. 5044 * 5045 * Never insert VLAN tag by HW, since segments split by 5046 * TSO engine will be un-tagged by mistake. 5047 */ 5048 if (features & NETIF_F_GSO_MASK) 5049 features &= ~(NETIF_F_HW_VLAN_STAG_TX | 5050 NETIF_F_HW_VLAN_CTAG_TX); 5051 } 5052 5053 return vlan_features_check(skb, features); 5054 } 5055 5056 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 5057 { 5058 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 5059 __be16 vlan_proto = veth->h_vlan_proto; 5060 u16 vlanid; 5061 5062 if ((vlan_proto == htons(ETH_P_8021Q) && 5063 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 5064 (vlan_proto == htons(ETH_P_8021AD) && 5065 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 5066 /* pop the vlan tag */ 5067 vlanid = ntohs(veth->h_vlan_TCI); 5068 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 5069 skb_pull(skb, VLAN_HLEN); 5070 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 5071 } 5072 } 5073 5074 /** 5075 * stmmac_rx_refill - refill used skb preallocated buffers 5076 * @priv: driver private structure 5077 * @queue: RX queue index 5078 * Description : this is to reallocate the skb for the reception process 5079 * that is based on zero-copy. 5080 */ 5081 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 5082 { 5083 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5084 int dirty = stmmac_rx_dirty(priv, queue); 5085 unsigned int entry = rx_q->dirty_rx; 5086 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 5087 5088 if (priv->dma_cap.host_dma_width <= 32) 5089 gfp |= GFP_DMA32; 5090 5091 while (dirty-- > 0) { 5092 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5093 struct dma_desc *p; 5094 bool use_rx_wd; 5095 5096 p = stmmac_get_rx_desc(priv, rx_q, entry); 5097 5098 if (!buf->page) { 5099 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5100 if (!buf->page) 5101 break; 5102 } 5103 5104 if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) { 5105 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5106 if (!buf->sec_page) 5107 break; 5108 5109 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 5110 } 5111 5112 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 5113 5114 stmmac_set_desc_addr(priv, p, buf->addr); 5115 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, 5116 stmmac_rx_check_buf2_cap(priv)); 5117 stmmac_refill_desc3(priv, rx_q, p); 5118 5119 rx_q->rx_count_frames++; 5120 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5121 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5122 rx_q->rx_count_frames = 0; 5123 5124 use_rx_wd = !priv->rx_coal_frames[queue]; 5125 use_rx_wd |= rx_q->rx_count_frames > 0; 5126 if (!priv->use_riwt) 5127 use_rx_wd = false; 5128 5129 dma_wmb(); 5130 stmmac_set_rx_owner(priv, p, use_rx_wd); 5131 5132 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5133 } 5134 rx_q->dirty_rx = entry; 5135 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5136 /* Wake up Rx DMA from the suspend state if required */ 5137 stmmac_enable_dma_reception(priv, priv->ioaddr, queue); 5138 } 5139 5140 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 5141 struct dma_desc *p, 5142 int status, unsigned int len) 5143 { 5144 unsigned int plen = 0, hlen = 0; 5145 int coe = priv->hw->rx_csum; 5146 5147 /* Not first descriptor, SPH enabled: buffer1 only carries the 5148 * split header of the first descriptor, so it is zero here. 5149 */ 5150 if (priv->sph_active && len) 5151 return 0; 5152 5153 /* First descriptor, get split header length */ 5154 stmmac_get_rx_header_len(priv, p, &hlen); 5155 if (priv->sph_active && hlen) { 5156 priv->xstats.rx_split_hdr_pkt_n++; 5157 return hlen; 5158 } 5159 5160 /* Not last descriptor and not split header: buffer1 is fully filled */ 5161 if (status & rx_not_ls) 5162 return priv->dma_conf.dma_buf_sz; 5163 5164 plen = stmmac_get_rx_frame_len(priv, p, coe); 5165 5166 /* Last descriptor and not split header: buffer1 holds the remaining 5167 * bytes of the frame, up to dma_buf_sz 5168 */ 5169 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen - len); 5170 } 5171 5172 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 5173 struct dma_desc *p, 5174 int status, unsigned int len) 5175 { 5176 int coe = priv->hw->rx_csum; 5177 unsigned int plen = 0; 5178 5179 if (!stmmac_rx_check_buf2_cap(priv)) 5180 return 0; 5181 5182 /* For GMAC4, when split header is enabled, in some rare cases, the 5183 * hardware does not fill buf2 of the first descriptor with payload. 5184 * Thus we cannot assume buf2 is always fully filled if it is not 5185 * the last descriptor. Otherwise, the length of buf2 of the second 5186 * descriptor will be calculated wrong and cause an oops. 5187 * 5188 * If this is the last descriptor, 'plen' is the length of the 5189 * received packet that was transferred to system memory. 5190 * Otherwise, it is the accumulated number of bytes that have been 5191 * transferred for the current packet. 5192 * 5193 * Thus 'plen - len' always gives the correct length of buf2. 5194 */ 5195 5196 /* Not GMAC4, or non-SPH and not last descriptor */ 5197 if ((priv->plat->core_type != DWMAC_CORE_GMAC4 || !priv->sph_active) && 5198 (status & rx_not_ls)) 5199 return priv->dma_conf.dma_buf_sz; 5200 5201 /* GMAC4 or last descriptor */ 5202 plen = stmmac_get_rx_frame_len(priv, p, coe); 5203 5204 return plen > len ? plen - len : 0; 5205 } 5206 5207 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 5208 struct xdp_frame *xdpf, bool dma_map) 5209 { 5210 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 5211 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 5212 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 5213 unsigned int entry = tx_q->cur_tx; 5214 enum stmmac_txbuf_type buf_type; 5215 struct dma_desc *tx_desc; 5216 dma_addr_t dma_addr; 5217 bool set_ic; 5218 5219 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 5220 return STMMAC_XDP_CONSUMED; 5221 5222 if (priv->est && priv->est->enable && 5223 priv->est->max_sdu[queue] && 5224 xdpf->len > priv->est->max_sdu[queue]) { 5225 priv->xstats.max_sdu_txq_drop[queue]++; 5226 return STMMAC_XDP_CONSUMED; 5227 } 5228 5229 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 5230 if (dma_map) { 5231 dma_addr = dma_map_single(priv->device, xdpf->data, 5232 xdpf->len, DMA_TO_DEVICE); 5233 if (dma_mapping_error(priv->device, dma_addr)) 5234 return STMMAC_XDP_CONSUMED; 5235 5236 buf_type = STMMAC_TXBUF_T_XDP_NDO; 5237 } else { 5238 struct page *page = virt_to_page(xdpf->data); 5239 5240 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 5241 xdpf->headroom; 5242 dma_sync_single_for_device(priv->device, dma_addr, 5243 xdpf->len, DMA_BIDIRECTIONAL); 5244 5245 buf_type = STMMAC_TXBUF_T_XDP_TX; 5246 } 5247 5248 stmmac_set_tx_dma_entry(tx_q, entry, buf_type, dma_addr, xdpf->len, 5249 false); 5250 stmmac_set_tx_dma_last_segment(tx_q, entry); 5251 5252 tx_q->xdpf[entry] = xdpf; 5253 5254 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 5255 5256 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 5257 csum, priv->descriptor_mode, true, true, 5258 xdpf->len); 5259 5260 tx_q->tx_count_frames++; 5261 5262 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 5263 set_ic = true; 5264 else 5265 set_ic = false; 5266 5267 if (set_ic) { 5268 tx_q->tx_count_frames = 0; 5269 stmmac_set_tx_ic(priv, tx_desc); 5270 u64_stats_update_begin(&txq_stats->q_syncp); 5271 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 5272 u64_stats_update_end(&txq_stats->q_syncp); 5273 } 5274 5275 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5276 5277 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 5278 tx_q->cur_tx = entry; 5279 5280 return STMMAC_XDP_TX; 5281 } 5282 5283 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 5284 int cpu) 5285 { 5286 int index = cpu; 5287 5288 if (unlikely(index < 0)) 5289 index = 0; 5290 5291 while (index >= priv->plat->tx_queues_to_use) 5292 index -= priv->plat->tx_queues_to_use; 5293 5294 return index; 5295 } 5296 5297 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 5298 struct xdp_buff *xdp) 5299 { 5300 bool zc = !!(xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL); 5301 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 5302 int cpu = smp_processor_id(); 5303 struct netdev_queue *nq; 5304 int queue; 5305 int res; 5306 5307 if (unlikely(!xdpf)) 5308 return STMMAC_XDP_CONSUMED; 5309 5310 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5311 nq = netdev_get_tx_queue(priv->dev, queue); 5312 5313 __netif_tx_lock(nq, cpu); 5314 /* Avoids TX time-out as we are sharing with slow path */ 5315 txq_trans_cond_update(nq); 5316 5317 /* For zero copy XDP_TX action, dma_map is true */ 5318 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, zc); 5319 if (res == STMMAC_XDP_TX) { 5320 stmmac_flush_tx_descriptors(priv, queue); 5321 } else if (res == STMMAC_XDP_CONSUMED && zc) { 5322 /* xdp has been freed by xdp_convert_buff_to_frame(), 5323 * no need to call xsk_buff_free() again, so return 5324 * STMMAC_XSK_CONSUMED. 5325 */ 5326 res = STMMAC_XSK_CONSUMED; 5327 xdp_return_frame(xdpf); 5328 } 5329 5330 __netif_tx_unlock(nq); 5331 5332 return res; 5333 } 5334 5335 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 5336 struct bpf_prog *prog, 5337 struct xdp_buff *xdp) 5338 { 5339 u32 act; 5340 int res; 5341 5342 act = bpf_prog_run_xdp(prog, xdp); 5343 switch (act) { 5344 case XDP_PASS: 5345 res = STMMAC_XDP_PASS; 5346 break; 5347 case XDP_TX: 5348 res = stmmac_xdp_xmit_back(priv, xdp); 5349 break; 5350 case XDP_REDIRECT: 5351 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 5352 res = STMMAC_XDP_CONSUMED; 5353 else 5354 res = STMMAC_XDP_REDIRECT; 5355 break; 5356 default: 5357 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5358 fallthrough; 5359 case XDP_ABORTED: 5360 trace_xdp_exception(priv->dev, prog, act); 5361 fallthrough; 5362 case XDP_DROP: 5363 res = STMMAC_XDP_CONSUMED; 5364 break; 5365 } 5366 5367 return res; 5368 } 5369 5370 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5371 struct xdp_buff *xdp) 5372 { 5373 struct bpf_prog *prog; 5374 int res; 5375 5376 prog = READ_ONCE(priv->xdp_prog); 5377 if (!prog) { 5378 res = STMMAC_XDP_PASS; 5379 goto out; 5380 } 5381 5382 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5383 out: 5384 return ERR_PTR(-res); 5385 } 5386 5387 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5388 int xdp_status) 5389 { 5390 int cpu = smp_processor_id(); 5391 int queue; 5392 5393 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5394 5395 if (xdp_status & STMMAC_XDP_TX) 5396 stmmac_tx_timer_arm(priv, queue); 5397 5398 if (xdp_status & STMMAC_XDP_REDIRECT) 5399 xdp_do_flush(); 5400 } 5401 5402 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5403 struct xdp_buff *xdp) 5404 { 5405 unsigned int metasize = xdp->data - xdp->data_meta; 5406 unsigned int datasize = xdp->data_end - xdp->data; 5407 struct sk_buff *skb; 5408 5409 skb = napi_alloc_skb(&ch->rxtx_napi, 5410 xdp->data_end - xdp->data_hard_start); 5411 if (unlikely(!skb)) 5412 return NULL; 5413 5414 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5415 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5416 if (metasize) 5417 skb_metadata_set(skb, metasize); 5418 5419 return skb; 5420 } 5421 5422 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5423 struct dma_desc *p, struct dma_desc *np, 5424 struct xdp_buff *xdp) 5425 { 5426 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5427 struct stmmac_channel *ch = &priv->channel[queue]; 5428 unsigned int len = xdp->data_end - xdp->data; 5429 enum pkt_hash_types hash_type; 5430 int coe = priv->hw->rx_csum; 5431 struct sk_buff *skb; 5432 u32 hash; 5433 5434 skb = stmmac_construct_skb_zc(ch, xdp); 5435 if (!skb) { 5436 priv->xstats.rx_dropped++; 5437 return; 5438 } 5439 5440 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5441 if (priv->hw->hw_vlan_en) 5442 /* MAC level stripping. */ 5443 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5444 else 5445 /* Driver level stripping. */ 5446 stmmac_rx_vlan(priv->dev, skb); 5447 skb->protocol = eth_type_trans(skb, priv->dev); 5448 5449 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5450 skb_checksum_none_assert(skb); 5451 else 5452 skb->ip_summed = CHECKSUM_UNNECESSARY; 5453 5454 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5455 skb_set_hash(skb, hash, hash_type); 5456 5457 skb_record_rx_queue(skb, queue); 5458 napi_gro_receive(&ch->rxtx_napi, skb); 5459 5460 u64_stats_update_begin(&rxq_stats->napi_syncp); 5461 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5462 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5463 u64_stats_update_end(&rxq_stats->napi_syncp); 5464 } 5465 5466 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5467 { 5468 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5469 unsigned int entry = rx_q->dirty_rx; 5470 struct dma_desc *rx_desc = NULL; 5471 bool ret = true; 5472 5473 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5474 5475 while (budget-- > 0 && entry != rx_q->cur_rx) { 5476 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5477 dma_addr_t dma_addr; 5478 bool use_rx_wd; 5479 5480 if (!buf->xdp) { 5481 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5482 if (!buf->xdp) { 5483 ret = false; 5484 break; 5485 } 5486 } 5487 5488 rx_desc = stmmac_get_rx_desc(priv, rx_q, entry); 5489 5490 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5491 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5492 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5493 stmmac_refill_desc3(priv, rx_q, rx_desc); 5494 5495 rx_q->rx_count_frames++; 5496 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5497 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5498 rx_q->rx_count_frames = 0; 5499 5500 use_rx_wd = !priv->rx_coal_frames[queue]; 5501 use_rx_wd |= rx_q->rx_count_frames > 0; 5502 if (!priv->use_riwt) 5503 use_rx_wd = false; 5504 5505 dma_wmb(); 5506 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5507 5508 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5509 } 5510 5511 if (rx_desc) { 5512 rx_q->dirty_rx = entry; 5513 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5514 } 5515 5516 return ret; 5517 } 5518 5519 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5520 { 5521 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5522 * to represent incoming packet, whereas cb field in the same structure 5523 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5524 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5525 */ 5526 return (struct stmmac_xdp_buff *)xdp; 5527 } 5528 5529 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5530 { 5531 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5532 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5533 unsigned int count = 0, error = 0, len = 0; 5534 int dirty = stmmac_rx_dirty(priv, queue); 5535 unsigned int next_entry = rx_q->cur_rx; 5536 u32 rx_errors = 0, rx_dropped = 0; 5537 unsigned int desc_size; 5538 struct bpf_prog *prog; 5539 bool failure = false; 5540 int xdp_status = 0; 5541 int status = 0; 5542 5543 if (netif_msg_rx_status(priv)) { 5544 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5545 5546 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5547 desc_size = stmmac_get_rx_desc_size(priv); 5548 5549 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5550 rx_q->dma_rx_phy, desc_size); 5551 } 5552 while (count < limit) { 5553 struct stmmac_rx_buffer *buf; 5554 struct stmmac_xdp_buff *ctx; 5555 unsigned int buf1_len = 0; 5556 struct dma_desc *np, *p; 5557 int entry; 5558 int res; 5559 5560 if (!count && rx_q->state_saved) { 5561 error = rx_q->state.error; 5562 len = rx_q->state.len; 5563 } else { 5564 rx_q->state_saved = false; 5565 error = 0; 5566 len = 0; 5567 } 5568 5569 read_again: 5570 if (count >= limit) 5571 break; 5572 5573 buf1_len = 0; 5574 entry = next_entry; 5575 buf = &rx_q->buf_pool[entry]; 5576 5577 if (dirty >= STMMAC_RX_FILL_BATCH) { 5578 failure = failure || 5579 !stmmac_rx_refill_zc(priv, queue, dirty); 5580 dirty = 0; 5581 } 5582 5583 p = stmmac_get_rx_desc(priv, rx_q, entry); 5584 5585 /* read the status of the incoming frame */ 5586 status = stmmac_rx_status(priv, &priv->xstats, p); 5587 /* check if managed by the DMA otherwise go ahead */ 5588 if (unlikely(status & dma_own)) 5589 break; 5590 5591 /* Prefetch the next RX descriptor */ 5592 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5593 priv->dma_conf.dma_rx_size); 5594 if (unlikely(next_entry == rx_q->dirty_rx)) 5595 break; 5596 5597 rx_q->cur_rx = next_entry; 5598 5599 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5600 5601 prefetch(np); 5602 5603 /* Ensure a valid XSK buffer before proceed */ 5604 if (!buf->xdp) 5605 break; 5606 5607 if (priv->extend_desc) 5608 stmmac_rx_extended_status(priv, &priv->xstats, 5609 rx_q->dma_erx + entry); 5610 if (unlikely(status == discard_frame)) { 5611 xsk_buff_free(buf->xdp); 5612 buf->xdp = NULL; 5613 dirty++; 5614 error = 1; 5615 if (!priv->hwts_rx_en) 5616 rx_errors++; 5617 } 5618 5619 if (unlikely(error && (status & rx_not_ls))) 5620 goto read_again; 5621 if (unlikely(error)) { 5622 count++; 5623 continue; 5624 } 5625 5626 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5627 if (likely(status & rx_not_ls)) { 5628 xsk_buff_free(buf->xdp); 5629 buf->xdp = NULL; 5630 dirty++; 5631 count++; 5632 goto read_again; 5633 } 5634 5635 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5636 ctx->priv = priv; 5637 ctx->desc = p; 5638 ctx->ndesc = np; 5639 5640 /* XDP ZC Frame only support primary buffers for now */ 5641 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5642 len += buf1_len; 5643 5644 /* ACS is disabled; strip manually. */ 5645 if (likely(!(status & rx_not_ls))) { 5646 buf1_len -= ETH_FCS_LEN; 5647 len -= ETH_FCS_LEN; 5648 } 5649 5650 /* RX buffer is good and fit into a XSK pool buffer */ 5651 buf->xdp->data_end = buf->xdp->data + buf1_len; 5652 xsk_buff_dma_sync_for_cpu(buf->xdp); 5653 5654 prog = READ_ONCE(priv->xdp_prog); 5655 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5656 5657 switch (res) { 5658 case STMMAC_XDP_PASS: 5659 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5660 xsk_buff_free(buf->xdp); 5661 break; 5662 case STMMAC_XDP_CONSUMED: 5663 xsk_buff_free(buf->xdp); 5664 fallthrough; 5665 case STMMAC_XSK_CONSUMED: 5666 rx_dropped++; 5667 break; 5668 case STMMAC_XDP_TX: 5669 case STMMAC_XDP_REDIRECT: 5670 xdp_status |= res; 5671 break; 5672 } 5673 5674 buf->xdp = NULL; 5675 dirty++; 5676 count++; 5677 } 5678 5679 if (status & rx_not_ls) { 5680 rx_q->state_saved = true; 5681 rx_q->state.error = error; 5682 rx_q->state.len = len; 5683 } 5684 5685 stmmac_finalize_xdp_rx(priv, xdp_status); 5686 5687 u64_stats_update_begin(&rxq_stats->napi_syncp); 5688 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5689 u64_stats_update_end(&rxq_stats->napi_syncp); 5690 5691 priv->xstats.rx_dropped += rx_dropped; 5692 priv->xstats.rx_errors += rx_errors; 5693 5694 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5695 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5696 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5697 else 5698 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5699 5700 return (int)count; 5701 } 5702 5703 return failure ? limit : (int)count; 5704 } 5705 5706 /** 5707 * stmmac_rx - manage the receive process 5708 * @priv: driver private structure 5709 * @limit: napi bugget 5710 * @queue: RX queue index. 5711 * Description : this the function called by the napi poll method. 5712 * It gets all the frames inside the ring. 5713 */ 5714 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5715 { 5716 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5717 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5718 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5719 struct stmmac_channel *ch = &priv->channel[queue]; 5720 unsigned int count = 0, error = 0, len = 0; 5721 int status = 0, coe = priv->hw->rx_csum; 5722 unsigned int next_entry = rx_q->cur_rx; 5723 enum dma_data_direction dma_dir; 5724 unsigned int desc_size; 5725 struct sk_buff *skb = NULL; 5726 struct stmmac_xdp_buff ctx; 5727 bool fcs_stripped = false; 5728 int xdp_status = 0; 5729 int bufsz; 5730 5731 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5732 bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5733 5734 if (netif_msg_rx_status(priv)) { 5735 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5736 5737 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5738 desc_size = stmmac_get_rx_desc_size(priv); 5739 5740 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5741 rx_q->dma_rx_phy, desc_size); 5742 } 5743 while (count < limit) { 5744 unsigned int buf1_len = 0, buf2_len = 0; 5745 enum pkt_hash_types hash_type; 5746 struct stmmac_rx_buffer *buf; 5747 struct dma_desc *np, *p; 5748 int entry; 5749 u32 hash; 5750 5751 if (!count && rx_q->state_saved) { 5752 skb = rx_q->state.skb; 5753 error = rx_q->state.error; 5754 len = rx_q->state.len; 5755 } else { 5756 rx_q->state_saved = false; 5757 skb = NULL; 5758 error = 0; 5759 len = 0; 5760 } 5761 5762 read_again: 5763 if (count >= limit) 5764 break; 5765 5766 buf1_len = 0; 5767 buf2_len = 0; 5768 entry = next_entry; 5769 buf = &rx_q->buf_pool[entry]; 5770 5771 p = stmmac_get_rx_desc(priv, rx_q, entry); 5772 5773 /* read the status of the incoming frame */ 5774 status = stmmac_rx_status(priv, &priv->xstats, p); 5775 /* check if managed by the DMA otherwise go ahead */ 5776 if (unlikely(status & dma_own)) 5777 break; 5778 5779 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5780 priv->dma_conf.dma_rx_size); 5781 if (unlikely(next_entry == rx_q->dirty_rx)) 5782 break; 5783 5784 rx_q->cur_rx = next_entry; 5785 5786 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5787 5788 prefetch(np); 5789 5790 if (priv->extend_desc) 5791 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5792 if (unlikely(status == discard_frame)) { 5793 page_pool_put_page(rx_q->page_pool, buf->page, 0, true); 5794 buf->page = NULL; 5795 error = 1; 5796 if (!priv->hwts_rx_en) 5797 rx_errors++; 5798 } 5799 5800 if (unlikely(error && (status & rx_not_ls))) 5801 goto read_again; 5802 if (unlikely(error)) { 5803 dev_kfree_skb(skb); 5804 skb = NULL; 5805 count++; 5806 continue; 5807 } 5808 5809 /* Buffer is good. Go on. */ 5810 5811 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5812 len += buf1_len; 5813 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5814 len += buf2_len; 5815 5816 /* ACS is disabled; strip manually. */ 5817 if (likely(!(status & rx_not_ls))) 5818 len -= ETH_FCS_LEN; 5819 5820 if (!skb) { 5821 unsigned int pre_len, sync_len; 5822 5823 /* Each frame starts here: reset the FCS handling */ 5824 fcs_stripped = false; 5825 5826 dma_sync_single_for_cpu(priv->device, buf->addr, 5827 buf1_len, dma_dir); 5828 net_prefetch(page_address(buf->page) + 5829 buf->page_offset); 5830 5831 if (stmmac_xdp_is_enabled(priv) && !buf2_len) { 5832 buf1_len -= ETH_FCS_LEN; 5833 fcs_stripped = true; 5834 } 5835 5836 xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq); 5837 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5838 buf->page_offset, buf1_len, true); 5839 5840 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5841 buf->page_offset; 5842 5843 ctx.priv = priv; 5844 ctx.desc = p; 5845 ctx.ndesc = np; 5846 5847 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5848 /* Due xdp_adjust_tail: DMA sync for_device 5849 * cover max len CPU touch 5850 */ 5851 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5852 buf->page_offset; 5853 sync_len = max(sync_len, pre_len); 5854 5855 /* For Not XDP_PASS verdict */ 5856 if (IS_ERR(skb)) { 5857 unsigned int xdp_res = -PTR_ERR(skb); 5858 5859 if (xdp_res & STMMAC_XDP_CONSUMED) { 5860 page_pool_put_page(rx_q->page_pool, 5861 virt_to_head_page(ctx.xdp.data), 5862 sync_len, true); 5863 buf->page = NULL; 5864 rx_dropped++; 5865 5866 /* Clear skb as it was set as 5867 * status by XDP program. 5868 */ 5869 skb = NULL; 5870 5871 if (unlikely((status & rx_not_ls))) 5872 goto read_again; 5873 5874 count++; 5875 continue; 5876 } else if (xdp_res & (STMMAC_XDP_TX | 5877 STMMAC_XDP_REDIRECT)) { 5878 xdp_status |= xdp_res; 5879 buf->page = NULL; 5880 skb = NULL; 5881 count++; 5882 continue; 5883 } 5884 } 5885 } 5886 5887 if (!skb) { 5888 unsigned int head_pad_len; 5889 5890 /* XDP program may expand or reduce tail */ 5891 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5892 5893 skb = napi_build_skb(page_address(buf->page), 5894 rx_q->napi_skb_frag_size); 5895 if (!skb) { 5896 page_pool_recycle_direct(rx_q->page_pool, 5897 buf->page); 5898 buf->page = NULL; 5899 rx_dropped++; 5900 count++; 5901 goto drain_data; 5902 } 5903 5904 /* XDP program may adjust header */ 5905 head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; 5906 skb_reserve(skb, head_pad_len); 5907 skb_put(skb, buf1_len); 5908 skb_mark_for_recycle(skb); 5909 buf->page = NULL; 5910 } else if (buf1_len) { 5911 dma_sync_single_for_cpu(priv->device, buf->addr, 5912 buf1_len, dma_dir); 5913 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5914 buf->page, buf->page_offset, buf1_len, 5915 priv->dma_conf.dma_buf_sz); 5916 buf->page = NULL; 5917 } 5918 5919 if (buf2_len) { 5920 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5921 buf2_len, dma_dir); 5922 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5923 buf->sec_page, 0, buf2_len, 5924 priv->dma_conf.dma_buf_sz); 5925 buf->sec_page = NULL; 5926 } 5927 5928 drain_data: 5929 if (likely(status & rx_not_ls)) 5930 goto read_again; 5931 if (!skb) 5932 continue; 5933 5934 /* Got entire packet into SKB. Finish it. */ 5935 5936 /* Remove FCS if needed */ 5937 if (!fcs_stripped && pskb_trim(skb, len)) { 5938 dev_kfree_skb_any(skb); 5939 skb = NULL; 5940 rx_dropped++; 5941 count++; 5942 continue; 5943 } 5944 5945 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5946 5947 if (priv->hw->hw_vlan_en) 5948 /* MAC level stripping. */ 5949 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5950 else 5951 /* Driver level stripping. */ 5952 stmmac_rx_vlan(priv->dev, skb); 5953 5954 skb->protocol = eth_type_trans(skb, priv->dev); 5955 5956 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) || 5957 (status & csum_none)) 5958 skb_checksum_none_assert(skb); 5959 else 5960 skb->ip_summed = CHECKSUM_UNNECESSARY; 5961 5962 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5963 skb_set_hash(skb, hash, hash_type); 5964 5965 skb_record_rx_queue(skb, queue); 5966 napi_gro_receive(&ch->rx_napi, skb); 5967 skb = NULL; 5968 5969 rx_packets++; 5970 rx_bytes += len; 5971 count++; 5972 } 5973 5974 if (status & rx_not_ls || skb) { 5975 rx_q->state_saved = true; 5976 rx_q->state.skb = skb; 5977 rx_q->state.error = error; 5978 rx_q->state.len = len; 5979 } 5980 5981 stmmac_finalize_xdp_rx(priv, xdp_status); 5982 5983 stmmac_rx_refill(priv, queue); 5984 5985 u64_stats_update_begin(&rxq_stats->napi_syncp); 5986 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5987 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5988 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5989 u64_stats_update_end(&rxq_stats->napi_syncp); 5990 5991 priv->xstats.rx_dropped += rx_dropped; 5992 priv->xstats.rx_errors += rx_errors; 5993 5994 return count; 5995 } 5996 5997 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5998 { 5999 struct stmmac_channel *ch = 6000 container_of(napi, struct stmmac_channel, rx_napi); 6001 struct stmmac_priv *priv = ch->priv_data; 6002 struct stmmac_rxq_stats *rxq_stats; 6003 u32 chan = ch->index; 6004 int work_done; 6005 6006 rxq_stats = &priv->xstats.rxq_stats[chan]; 6007 u64_stats_update_begin(&rxq_stats->napi_syncp); 6008 u64_stats_inc(&rxq_stats->napi.poll); 6009 u64_stats_update_end(&rxq_stats->napi_syncp); 6010 6011 work_done = stmmac_rx(priv, budget, chan); 6012 if (work_done < budget && napi_complete_done(napi, work_done)) { 6013 unsigned long flags; 6014 6015 spin_lock_irqsave(&ch->lock, flags); 6016 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 6017 spin_unlock_irqrestore(&ch->lock, flags); 6018 } 6019 6020 return work_done; 6021 } 6022 6023 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 6024 { 6025 struct stmmac_channel *ch = 6026 container_of(napi, struct stmmac_channel, tx_napi); 6027 struct stmmac_priv *priv = ch->priv_data; 6028 struct stmmac_txq_stats *txq_stats; 6029 bool pending_packets = false; 6030 u32 chan = ch->index; 6031 int work_done; 6032 6033 txq_stats = &priv->xstats.txq_stats[chan]; 6034 u64_stats_update_begin(&txq_stats->napi_syncp); 6035 u64_stats_inc(&txq_stats->napi.poll); 6036 u64_stats_update_end(&txq_stats->napi_syncp); 6037 6038 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 6039 work_done = min(work_done, budget); 6040 6041 if (work_done < budget && napi_complete_done(napi, work_done)) { 6042 unsigned long flags; 6043 6044 spin_lock_irqsave(&ch->lock, flags); 6045 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 6046 spin_unlock_irqrestore(&ch->lock, flags); 6047 } 6048 6049 /* TX still have packet to handle, check if we need to arm tx timer */ 6050 if (pending_packets) 6051 stmmac_tx_timer_arm(priv, chan); 6052 6053 return work_done; 6054 } 6055 6056 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 6057 { 6058 struct stmmac_channel *ch = 6059 container_of(napi, struct stmmac_channel, rxtx_napi); 6060 struct stmmac_priv *priv = ch->priv_data; 6061 bool tx_pending_packets = false; 6062 int rx_done, tx_done, rxtx_done; 6063 struct stmmac_rxq_stats *rxq_stats; 6064 struct stmmac_txq_stats *txq_stats; 6065 u32 chan = ch->index; 6066 6067 rxq_stats = &priv->xstats.rxq_stats[chan]; 6068 u64_stats_update_begin(&rxq_stats->napi_syncp); 6069 u64_stats_inc(&rxq_stats->napi.poll); 6070 u64_stats_update_end(&rxq_stats->napi_syncp); 6071 6072 txq_stats = &priv->xstats.txq_stats[chan]; 6073 u64_stats_update_begin(&txq_stats->napi_syncp); 6074 u64_stats_inc(&txq_stats->napi.poll); 6075 u64_stats_update_end(&txq_stats->napi_syncp); 6076 6077 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 6078 tx_done = min(tx_done, budget); 6079 6080 rx_done = stmmac_rx_zc(priv, budget, chan); 6081 6082 rxtx_done = max(tx_done, rx_done); 6083 6084 /* If either TX or RX work is not complete, return budget 6085 * and keep pooling 6086 */ 6087 if (rxtx_done >= budget) 6088 return budget; 6089 6090 /* all work done, exit the polling mode */ 6091 if (napi_complete_done(napi, rxtx_done)) { 6092 unsigned long flags; 6093 6094 spin_lock_irqsave(&ch->lock, flags); 6095 /* Both RX and TX work done are complete, 6096 * so enable both RX & TX IRQs. 6097 */ 6098 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6099 spin_unlock_irqrestore(&ch->lock, flags); 6100 } 6101 6102 /* TX still have packet to handle, check if we need to arm tx timer */ 6103 if (tx_pending_packets) 6104 stmmac_tx_timer_arm(priv, chan); 6105 6106 return min(rxtx_done, budget - 1); 6107 } 6108 6109 /** 6110 * stmmac_tx_timeout 6111 * @dev : Pointer to net device structure 6112 * @txqueue: the index of the hanging transmit queue 6113 * Description: this function is called when a packet transmission fails to 6114 * complete within a reasonable time. The driver will mark the error in the 6115 * netdev structure and arrange for the device to be reset to a sane state 6116 * in order to transmit a new packet. 6117 */ 6118 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 6119 { 6120 struct stmmac_priv *priv = netdev_priv(dev); 6121 6122 stmmac_global_err(priv); 6123 } 6124 6125 /** 6126 * stmmac_set_rx_mode - entry point for multicast addressing 6127 * @dev : pointer to the device structure 6128 * Description: 6129 * This function is a driver entry point which gets called by the kernel 6130 * whenever multicast addresses must be enabled/disabled. 6131 * Return value: 6132 * void. 6133 * 6134 * FIXME: This may need RXC to be running, but it may be called with BH 6135 * disabled, which means we can't call phylink_rx_clk_stop*(). 6136 */ 6137 static void stmmac_set_rx_mode(struct net_device *dev) 6138 { 6139 struct stmmac_priv *priv = netdev_priv(dev); 6140 6141 stmmac_set_filter(priv, priv->hw, dev); 6142 } 6143 6144 /** 6145 * stmmac_change_mtu - entry point to change MTU size for the device. 6146 * @dev : device pointer. 6147 * @new_mtu : the new MTU size for the device. 6148 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 6149 * to drive packet transmission. Ethernet has an MTU of 1500 octets 6150 * (ETH_DATA_LEN). This value can be changed with ifconfig. 6151 * Return value: 6152 * 0 on success and an appropriate (-)ve integer as defined in errno.h 6153 * file on failure. 6154 */ 6155 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 6156 { 6157 struct stmmac_priv *priv = netdev_priv(dev); 6158 int txfifosz = priv->plat->tx_fifo_size; 6159 struct stmmac_dma_conf *dma_conf; 6160 const int mtu = new_mtu; 6161 int ret; 6162 6163 if (txfifosz == 0) 6164 txfifosz = priv->dma_cap.tx_fifo_size; 6165 6166 txfifosz /= priv->plat->tx_queues_to_use; 6167 6168 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 6169 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 6170 return -EINVAL; 6171 } 6172 6173 new_mtu = STMMAC_ALIGN(new_mtu); 6174 6175 /* If condition true, FIFO is too small or MTU too large */ 6176 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 6177 return -EINVAL; 6178 6179 if (netif_running(dev)) { 6180 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 6181 /* Try to allocate the new DMA conf with the new mtu */ 6182 dma_conf = stmmac_setup_dma_desc(priv, mtu); 6183 if (IS_ERR(dma_conf)) { 6184 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 6185 mtu); 6186 return PTR_ERR(dma_conf); 6187 } 6188 6189 __stmmac_release(dev); 6190 6191 ret = __stmmac_open(dev, dma_conf); 6192 if (ret) { 6193 free_dma_desc_resources(priv, dma_conf); 6194 kfree(dma_conf); 6195 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 6196 return ret; 6197 } 6198 6199 kfree(dma_conf); 6200 6201 stmmac_set_rx_mode(dev); 6202 } 6203 6204 WRITE_ONCE(dev->mtu, mtu); 6205 netdev_update_features(dev); 6206 6207 return 0; 6208 } 6209 6210 static netdev_features_t stmmac_fix_features(struct net_device *dev, 6211 netdev_features_t features) 6212 { 6213 struct stmmac_priv *priv = netdev_priv(dev); 6214 6215 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 6216 features &= ~NETIF_F_RXCSUM; 6217 6218 if (!priv->plat->tx_coe) 6219 features &= ~NETIF_F_CSUM_MASK; 6220 6221 /* Some GMAC devices have a bugged Jumbo frame support that 6222 * needs to have the Tx COE disabled for oversized frames 6223 * (due to limited buffer sizes). In this case we disable 6224 * the TX csum insertion in the TDES and not use SF. 6225 */ 6226 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 6227 features &= ~NETIF_F_CSUM_MASK; 6228 6229 if (priv->plat->core_type == DWMAC_CORE_XGMAC) { 6230 if (features & NETIF_F_HW_VLAN_CTAG_RX) 6231 features |= NETIF_F_HW_VLAN_STAG_RX; 6232 else 6233 features &= ~NETIF_F_HW_VLAN_STAG_RX; 6234 } 6235 6236 return features; 6237 } 6238 6239 static int stmmac_set_features(struct net_device *netdev, 6240 netdev_features_t features) 6241 { 6242 struct stmmac_priv *priv = netdev_priv(netdev); 6243 6244 /* Keep the COE Type in case of csum is supporting */ 6245 if (features & NETIF_F_RXCSUM) 6246 priv->hw->rx_csum = priv->plat->rx_coe; 6247 else 6248 priv->hw->rx_csum = 0; 6249 /* No check needed because rx_coe has been set before and it will be 6250 * fixed in case of issue. 6251 */ 6252 stmmac_rx_ipc(priv, priv->hw); 6253 6254 if (priv->sph_capable) { 6255 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 6256 u8 chan; 6257 6258 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 6259 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6260 } 6261 6262 if (features & NETIF_F_HW_VLAN_CTAG_RX) 6263 priv->hw->hw_vlan_en = true; 6264 else 6265 priv->hw->hw_vlan_en = false; 6266 6267 phylink_rx_clk_stop_block(priv->phylink); 6268 stmmac_set_hw_vlan_mode(priv, priv->hw); 6269 phylink_rx_clk_stop_unblock(priv->phylink); 6270 6271 return 0; 6272 } 6273 6274 static void stmmac_common_interrupt(struct stmmac_priv *priv) 6275 { 6276 u8 rx_cnt = priv->plat->rx_queues_to_use; 6277 u8 tx_cnt = priv->plat->tx_queues_to_use; 6278 u8 queues_count; 6279 bool xmac; 6280 u8 queue; 6281 6282 xmac = dwmac_is_xmac(priv->plat->core_type); 6283 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 6284 6285 if (priv->irq_wake) 6286 pm_wakeup_event(priv->device, 0); 6287 6288 if (priv->dma_cap.estsel) 6289 stmmac_est_irq_status(priv, priv, priv->dev, 6290 &priv->xstats, tx_cnt); 6291 6292 if (stmmac_fpe_supported(priv)) 6293 stmmac_fpe_irq_status(priv); 6294 6295 /* To handle GMAC own interrupts */ 6296 if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) { 6297 int status = stmmac_host_irq_status(priv, &priv->xstats); 6298 6299 if (unlikely(status)) { 6300 /* For LPI we need to save the tx status */ 6301 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 6302 priv->tx_path_in_lpi_mode = true; 6303 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 6304 priv->tx_path_in_lpi_mode = false; 6305 } 6306 6307 for (queue = 0; queue < queues_count; queue++) 6308 stmmac_host_mtl_irq_status(priv, priv->hw, queue); 6309 6310 stmmac_timestamp_interrupt(priv, priv); 6311 } 6312 } 6313 6314 /** 6315 * stmmac_interrupt - main ISR 6316 * @irq: interrupt number. 6317 * @dev_id: to pass the net device pointer. 6318 * Description: this is the main driver interrupt service routine. 6319 * It can call: 6320 * o DMA service routine (to manage incoming frame reception and transmission 6321 * status) 6322 * o Core interrupts to manage: remote wake-up, management counter, LPI 6323 * interrupts. 6324 */ 6325 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6326 { 6327 struct net_device *dev = (struct net_device *)dev_id; 6328 struct stmmac_priv *priv = netdev_priv(dev); 6329 6330 /* Check if adapter is up */ 6331 if (test_bit(STMMAC_DOWN, &priv->state)) 6332 return IRQ_HANDLED; 6333 6334 /* Check ASP error if it isn't delivered via an individual IRQ */ 6335 if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) 6336 return IRQ_HANDLED; 6337 6338 /* To handle Common interrupts */ 6339 stmmac_common_interrupt(priv); 6340 6341 /* To handle DMA interrupts */ 6342 stmmac_dma_interrupt(priv); 6343 6344 return IRQ_HANDLED; 6345 } 6346 6347 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6348 { 6349 struct net_device *dev = (struct net_device *)dev_id; 6350 struct stmmac_priv *priv = netdev_priv(dev); 6351 6352 /* Check if adapter is up */ 6353 if (test_bit(STMMAC_DOWN, &priv->state)) 6354 return IRQ_HANDLED; 6355 6356 /* To handle Common interrupts */ 6357 stmmac_common_interrupt(priv); 6358 6359 return IRQ_HANDLED; 6360 } 6361 6362 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6363 { 6364 struct net_device *dev = (struct net_device *)dev_id; 6365 struct stmmac_priv *priv = netdev_priv(dev); 6366 6367 /* Check if adapter is up */ 6368 if (test_bit(STMMAC_DOWN, &priv->state)) 6369 return IRQ_HANDLED; 6370 6371 /* Check if a fatal error happened */ 6372 stmmac_safety_feat_interrupt(priv); 6373 6374 return IRQ_HANDLED; 6375 } 6376 6377 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6378 { 6379 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6380 struct stmmac_dma_conf *dma_conf; 6381 int chan = tx_q->queue_index; 6382 struct stmmac_priv *priv; 6383 int status; 6384 6385 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6386 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6387 6388 /* Check if adapter is up */ 6389 if (test_bit(STMMAC_DOWN, &priv->state)) 6390 return IRQ_HANDLED; 6391 6392 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6393 6394 if (unlikely(status & tx_hard_error_bump_tc)) { 6395 /* Try to bump up the dma threshold on this failure */ 6396 stmmac_bump_dma_threshold(priv, chan); 6397 } else if (unlikely(status == tx_hard_error)) { 6398 stmmac_tx_err(priv, chan); 6399 } 6400 6401 return IRQ_HANDLED; 6402 } 6403 6404 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6405 { 6406 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6407 struct stmmac_dma_conf *dma_conf; 6408 int chan = rx_q->queue_index; 6409 struct stmmac_priv *priv; 6410 6411 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6412 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6413 6414 /* Check if adapter is up */ 6415 if (test_bit(STMMAC_DOWN, &priv->state)) 6416 return IRQ_HANDLED; 6417 6418 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6419 6420 return IRQ_HANDLED; 6421 } 6422 6423 /** 6424 * stmmac_ioctl - Entry point for the Ioctl 6425 * @dev: Device pointer. 6426 * @rq: An IOCTL specific structure, that can contain a pointer to 6427 * a proprietary structure used to pass information to the driver. 6428 * @cmd: IOCTL command 6429 * Description: Forward the PHY ioctls to phylink 6430 * Return: Zero on success or negative error code. 6431 */ 6432 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6433 { 6434 struct stmmac_priv *priv = netdev_priv (dev); 6435 6436 if (!netif_running(dev)) 6437 return -EINVAL; 6438 6439 return phylink_mii_ioctl(priv->phylink, rq, cmd); 6440 } 6441 6442 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6443 void *cb_priv) 6444 { 6445 struct stmmac_priv *priv = cb_priv; 6446 int ret = -EOPNOTSUPP; 6447 6448 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6449 return ret; 6450 6451 __stmmac_disable_all_queues(priv); 6452 6453 switch (type) { 6454 case TC_SETUP_CLSU32: 6455 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6456 break; 6457 case TC_SETUP_CLSFLOWER: 6458 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6459 break; 6460 default: 6461 break; 6462 } 6463 6464 stmmac_enable_all_queues(priv); 6465 return ret; 6466 } 6467 6468 static LIST_HEAD(stmmac_block_cb_list); 6469 6470 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6471 void *type_data) 6472 { 6473 struct stmmac_priv *priv = netdev_priv(ndev); 6474 6475 switch (type) { 6476 case TC_QUERY_CAPS: 6477 return stmmac_tc_query_caps(priv, priv, type_data); 6478 case TC_SETUP_QDISC_MQPRIO: 6479 return stmmac_tc_setup_mqprio(priv, priv, type_data); 6480 case TC_SETUP_BLOCK: 6481 return flow_block_cb_setup_simple(type_data, 6482 &stmmac_block_cb_list, 6483 stmmac_setup_tc_block_cb, 6484 priv, priv, true); 6485 case TC_SETUP_QDISC_CBS: 6486 return stmmac_tc_setup_cbs(priv, priv, type_data); 6487 case TC_SETUP_QDISC_TAPRIO: 6488 return stmmac_tc_setup_taprio(priv, priv, type_data); 6489 case TC_SETUP_QDISC_ETF: 6490 return stmmac_tc_setup_etf(priv, priv, type_data); 6491 default: 6492 return -EOPNOTSUPP; 6493 } 6494 } 6495 6496 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6497 struct net_device *sb_dev) 6498 { 6499 int gso = skb_shinfo(skb)->gso_type; 6500 6501 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6502 /* 6503 * There is no way to determine the number of TSO/USO 6504 * capable Queues. Let's use always the Queue 0 6505 * because if TSO/USO is supported then at least this 6506 * one will be capable. 6507 */ 6508 return 0; 6509 } 6510 6511 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6512 } 6513 6514 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6515 { 6516 struct stmmac_priv *priv = netdev_priv(ndev); 6517 int ret = 0; 6518 6519 ret = pm_runtime_resume_and_get(priv->device); 6520 if (ret < 0) 6521 return ret; 6522 6523 ret = eth_mac_addr(ndev, addr); 6524 if (ret) 6525 goto set_mac_error; 6526 6527 phylink_rx_clk_stop_block(priv->phylink); 6528 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6529 phylink_rx_clk_stop_unblock(priv->phylink); 6530 6531 set_mac_error: 6532 pm_runtime_put(priv->device); 6533 6534 return ret; 6535 } 6536 6537 #ifdef CONFIG_DEBUG_FS 6538 static struct dentry *stmmac_fs_dir; 6539 6540 static void sysfs_display_ring(void *head, int size, int extend_desc, 6541 struct seq_file *seq, dma_addr_t dma_phy_addr) 6542 { 6543 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6544 struct dma_desc *p = (struct dma_desc *)head; 6545 unsigned int desc_size; 6546 dma_addr_t dma_addr; 6547 int i; 6548 6549 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6550 for (i = 0; i < size; i++) { 6551 dma_addr = dma_phy_addr + i * desc_size; 6552 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6553 i, &dma_addr, 6554 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6555 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6556 if (extend_desc) 6557 p = &(++ep)->basic; 6558 else 6559 p++; 6560 } 6561 } 6562 6563 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6564 { 6565 struct net_device *dev = seq->private; 6566 struct stmmac_priv *priv = netdev_priv(dev); 6567 u8 rx_count = priv->plat->rx_queues_to_use; 6568 u8 tx_count = priv->plat->tx_queues_to_use; 6569 u8 queue; 6570 6571 if ((dev->flags & IFF_UP) == 0) 6572 return 0; 6573 6574 for (queue = 0; queue < rx_count; queue++) { 6575 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6576 6577 seq_printf(seq, "RX Queue %d:\n", queue); 6578 6579 if (priv->extend_desc) { 6580 seq_printf(seq, "Extended descriptor ring:\n"); 6581 sysfs_display_ring((void *)rx_q->dma_erx, 6582 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6583 } else { 6584 seq_printf(seq, "Descriptor ring:\n"); 6585 sysfs_display_ring((void *)rx_q->dma_rx, 6586 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6587 } 6588 } 6589 6590 for (queue = 0; queue < tx_count; queue++) { 6591 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6592 6593 seq_printf(seq, "TX Queue %d:\n", queue); 6594 6595 if (priv->extend_desc) { 6596 seq_printf(seq, "Extended descriptor ring:\n"); 6597 sysfs_display_ring((void *)tx_q->dma_etx, 6598 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6599 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6600 seq_printf(seq, "Descriptor ring:\n"); 6601 sysfs_display_ring((void *)tx_q->dma_tx, 6602 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6603 } 6604 } 6605 6606 return 0; 6607 } 6608 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6609 6610 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6611 { 6612 static const char * const dwxgmac_timestamp_source[] = { 6613 "None", 6614 "Internal", 6615 "External", 6616 "Both", 6617 }; 6618 static const char * const dwxgmac_safety_feature_desc[] = { 6619 "No", 6620 "All Safety Features with ECC and Parity", 6621 "All Safety Features without ECC or Parity", 6622 "All Safety Features with Parity Only", 6623 "ECC Only", 6624 "UNDEFINED", 6625 "UNDEFINED", 6626 "UNDEFINED", 6627 }; 6628 struct net_device *dev = seq->private; 6629 struct stmmac_priv *priv = netdev_priv(dev); 6630 6631 if (!priv->hw_cap_support) { 6632 seq_printf(seq, "DMA HW features not supported\n"); 6633 return 0; 6634 } 6635 6636 seq_printf(seq, "==============================\n"); 6637 seq_printf(seq, "\tDMA HW features\n"); 6638 seq_printf(seq, "==============================\n"); 6639 6640 seq_printf(seq, "\t10/100 Mbps: %s\n", 6641 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6642 seq_printf(seq, "\t1000 Mbps: %s\n", 6643 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6644 seq_printf(seq, "\tHalf duplex: %s\n", 6645 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6646 if (priv->plat->core_type == DWMAC_CORE_XGMAC) { 6647 seq_printf(seq, 6648 "\tNumber of Additional MAC address registers: %d\n", 6649 priv->dma_cap.multi_addr); 6650 } else { 6651 seq_printf(seq, "\tHash Filter: %s\n", 6652 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6653 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6654 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6655 } 6656 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6657 (priv->dma_cap.pcs) ? "Y" : "N"); 6658 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6659 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6660 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6661 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6662 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6663 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6664 seq_printf(seq, "\tRMON module: %s\n", 6665 (priv->dma_cap.rmon) ? "Y" : "N"); 6666 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6667 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6668 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6669 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6670 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 6671 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6672 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6673 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6674 (priv->dma_cap.eee) ? "Y" : "N"); 6675 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6676 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6677 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6678 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6679 priv->plat->core_type == DWMAC_CORE_XGMAC) { 6680 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6681 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6682 } else { 6683 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6684 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6685 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6686 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6687 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6688 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6689 } 6690 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6691 priv->dma_cap.number_rx_channel); 6692 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6693 priv->dma_cap.number_tx_channel); 6694 seq_printf(seq, "\tNumber of Additional RX queues: %u\n", 6695 priv->dma_cap.number_rx_queues); 6696 seq_printf(seq, "\tNumber of Additional TX queues: %u\n", 6697 priv->dma_cap.number_tx_queues); 6698 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6699 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6700 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6701 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6702 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6703 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6704 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6705 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6706 priv->dma_cap.pps_out_num); 6707 seq_printf(seq, "\tSafety Features: %s\n", 6708 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6709 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6710 priv->dma_cap.frpsel ? "Y" : "N"); 6711 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6712 priv->dma_cap.host_dma_width); 6713 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6714 priv->dma_cap.rssen ? "Y" : "N"); 6715 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6716 priv->dma_cap.vlhash ? "Y" : "N"); 6717 seq_printf(seq, "\tSplit Header: %s\n", 6718 priv->dma_cap.sphen ? "Y" : "N"); 6719 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6720 priv->dma_cap.vlins ? "Y" : "N"); 6721 seq_printf(seq, "\tDouble VLAN: %s\n", 6722 priv->dma_cap.dvlan ? "Y" : "N"); 6723 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6724 priv->dma_cap.l3l4fnum); 6725 seq_printf(seq, "\tARP Offloading: %s\n", 6726 priv->dma_cap.arpoffsel ? "Y" : "N"); 6727 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6728 priv->dma_cap.estsel ? "Y" : "N"); 6729 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6730 priv->dma_cap.fpesel ? "Y" : "N"); 6731 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6732 priv->dma_cap.tbssel ? "Y" : "N"); 6733 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6734 priv->dma_cap.tbs_ch_num); 6735 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6736 priv->dma_cap.sgfsel ? "Y" : "N"); 6737 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6738 BIT(priv->dma_cap.ttsfd) >> 1); 6739 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6740 priv->dma_cap.numtc); 6741 seq_printf(seq, "\tDCB Feature: %s\n", 6742 priv->dma_cap.dcben ? "Y" : "N"); 6743 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6744 priv->dma_cap.advthword ? "Y" : "N"); 6745 seq_printf(seq, "\tPTP Offload: %s\n", 6746 priv->dma_cap.ptoen ? "Y" : "N"); 6747 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6748 priv->dma_cap.osten ? "Y" : "N"); 6749 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6750 priv->dma_cap.pfcen ? "Y" : "N"); 6751 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6752 BIT(priv->dma_cap.frpes) << 6); 6753 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6754 BIT(priv->dma_cap.frpbs) << 6); 6755 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6756 priv->dma_cap.frppipe_num); 6757 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6758 priv->dma_cap.nrvf_num ? 6759 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6760 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6761 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6762 seq_printf(seq, "\tDepth of GCL: %lu\n", 6763 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6764 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6765 priv->dma_cap.cbtisel ? "Y" : "N"); 6766 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6767 priv->dma_cap.aux_snapshot_n); 6768 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6769 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6770 seq_printf(seq, "\tEnhanced DMA: %s\n", 6771 priv->dma_cap.edma ? "Y" : "N"); 6772 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6773 priv->dma_cap.ediffc ? "Y" : "N"); 6774 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6775 priv->dma_cap.vxn ? "Y" : "N"); 6776 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6777 priv->dma_cap.dbgmem ? "Y" : "N"); 6778 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6779 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6780 return 0; 6781 } 6782 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6783 6784 /* Use network device events to rename debugfs file entries. 6785 */ 6786 static int stmmac_device_event(struct notifier_block *unused, 6787 unsigned long event, void *ptr) 6788 { 6789 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6790 struct stmmac_priv *priv = netdev_priv(dev); 6791 6792 if (dev->netdev_ops != &stmmac_netdev_ops) 6793 goto done; 6794 6795 switch (event) { 6796 case NETDEV_CHANGENAME: 6797 debugfs_change_name(priv->dbgfs_dir, "%s", dev->name); 6798 break; 6799 } 6800 done: 6801 return NOTIFY_DONE; 6802 } 6803 6804 static struct notifier_block stmmac_notifier = { 6805 .notifier_call = stmmac_device_event, 6806 }; 6807 6808 static void stmmac_init_fs(struct net_device *dev) 6809 { 6810 struct stmmac_priv *priv = netdev_priv(dev); 6811 6812 rtnl_lock(); 6813 6814 /* Create per netdev entries */ 6815 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6816 6817 /* Entry to report DMA RX/TX rings */ 6818 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6819 &stmmac_rings_status_fops); 6820 6821 /* Entry to report the DMA HW features */ 6822 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6823 &stmmac_dma_cap_fops); 6824 6825 rtnl_unlock(); 6826 } 6827 6828 static void stmmac_exit_fs(struct net_device *dev) 6829 { 6830 struct stmmac_priv *priv = netdev_priv(dev); 6831 6832 debugfs_remove_recursive(priv->dbgfs_dir); 6833 } 6834 #endif /* CONFIG_DEBUG_FS */ 6835 6836 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6837 { 6838 unsigned char *data = (unsigned char *)&vid_le; 6839 unsigned char data_byte = 0; 6840 u32 crc = ~0x0; 6841 u32 temp = 0; 6842 int i, bits; 6843 6844 bits = get_bitmask_order(VLAN_VID_MASK); 6845 for (i = 0; i < bits; i++) { 6846 if ((i % 8) == 0) 6847 data_byte = data[i / 8]; 6848 6849 temp = ((crc & 1) ^ data_byte) & 1; 6850 crc >>= 1; 6851 data_byte >>= 1; 6852 6853 if (temp) 6854 crc ^= 0xedb88320; 6855 } 6856 6857 return crc; 6858 } 6859 6860 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_svlan) 6861 { 6862 u32 crc, hash = 0; 6863 u16 vid = 0; 6864 6865 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6866 __le16 vid_le = cpu_to_le16(vid); 6867 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6868 hash |= (1 << crc); 6869 } 6870 6871 if (!netif_running(priv->dev)) 6872 return 0; 6873 6874 return stmmac_update_vlan_hash(priv, priv->hw, hash, is_svlan); 6875 } 6876 6877 /* FIXME: This may need RXC to be running, but it may be called with BH 6878 * disabled, which means we can't call phylink_rx_clk_stop*(). 6879 */ 6880 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6881 { 6882 struct stmmac_priv *priv = netdev_priv(ndev); 6883 unsigned int num_svlans; 6884 bool is_svlan = false; 6885 int ret; 6886 6887 ret = pm_runtime_resume_and_get(priv->device); 6888 if (ret < 0) 6889 return ret; 6890 6891 if (be16_to_cpu(proto) == ETH_P_8021AD) 6892 is_svlan = true; 6893 6894 set_bit(vid, priv->active_vlans); 6895 num_svlans = priv->num_svlans + is_svlan; 6896 ret = stmmac_vlan_update(priv, num_svlans); 6897 if (ret) { 6898 clear_bit(vid, priv->active_vlans); 6899 goto err_pm_put; 6900 } 6901 6902 if (priv->hw->num_vlan) { 6903 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6904 if (ret) { 6905 clear_bit(vid, priv->active_vlans); 6906 stmmac_vlan_update(priv, priv->num_svlans); 6907 goto err_pm_put; 6908 } 6909 } 6910 6911 priv->num_svlans = num_svlans; 6912 6913 err_pm_put: 6914 pm_runtime_put(priv->device); 6915 6916 return ret; 6917 } 6918 6919 /* FIXME: This may need RXC to be running, but it may be called with BH 6920 * disabled, which means we can't call phylink_rx_clk_stop*(). 6921 */ 6922 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6923 { 6924 struct stmmac_priv *priv = netdev_priv(ndev); 6925 unsigned int num_svlans; 6926 bool is_svlan = false; 6927 int ret; 6928 6929 ret = pm_runtime_resume_and_get(priv->device); 6930 if (ret < 0) 6931 return ret; 6932 6933 if (be16_to_cpu(proto) == ETH_P_8021AD) 6934 is_svlan = true; 6935 6936 clear_bit(vid, priv->active_vlans); 6937 num_svlans = priv->num_svlans - is_svlan; 6938 ret = stmmac_vlan_update(priv, num_svlans); 6939 if (ret) { 6940 set_bit(vid, priv->active_vlans); 6941 goto del_vlan_error; 6942 } 6943 6944 if (priv->hw->num_vlan) { 6945 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6946 if (ret) { 6947 set_bit(vid, priv->active_vlans); 6948 stmmac_vlan_update(priv, priv->num_svlans); 6949 goto del_vlan_error; 6950 } 6951 } 6952 6953 priv->num_svlans = num_svlans; 6954 6955 del_vlan_error: 6956 pm_runtime_put(priv->device); 6957 6958 return ret; 6959 } 6960 6961 static void stmmac_vlan_restore(struct stmmac_priv *priv) 6962 { 6963 if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) 6964 return; 6965 6966 if (priv->hw->num_vlan) 6967 stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); 6968 6969 stmmac_vlan_update(priv, priv->num_svlans); 6970 } 6971 6972 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6973 { 6974 struct stmmac_priv *priv = netdev_priv(dev); 6975 6976 switch (bpf->command) { 6977 case XDP_SETUP_PROG: 6978 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6979 case XDP_SETUP_XSK_POOL: 6980 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6981 bpf->xsk.queue_id); 6982 default: 6983 return -EOPNOTSUPP; 6984 } 6985 } 6986 6987 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6988 struct xdp_frame **frames, u32 flags) 6989 { 6990 struct stmmac_priv *priv = netdev_priv(dev); 6991 int cpu = smp_processor_id(); 6992 struct netdev_queue *nq; 6993 int i, nxmit = 0; 6994 int queue; 6995 6996 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6997 return -ENETDOWN; 6998 6999 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 7000 return -EINVAL; 7001 7002 queue = stmmac_xdp_get_tx_queue(priv, cpu); 7003 nq = netdev_get_tx_queue(priv->dev, queue); 7004 7005 __netif_tx_lock(nq, cpu); 7006 /* Avoids TX time-out as we are sharing with slow path */ 7007 txq_trans_cond_update(nq); 7008 7009 for (i = 0; i < num_frames; i++) { 7010 int res; 7011 7012 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 7013 if (res == STMMAC_XDP_CONSUMED) 7014 break; 7015 7016 nxmit++; 7017 } 7018 7019 if (flags & XDP_XMIT_FLUSH) { 7020 stmmac_flush_tx_descriptors(priv, queue); 7021 stmmac_tx_timer_arm(priv, queue); 7022 } 7023 7024 __netif_tx_unlock(nq); 7025 7026 return nxmit; 7027 } 7028 7029 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 7030 { 7031 struct stmmac_channel *ch = &priv->channel[queue]; 7032 unsigned long flags; 7033 7034 spin_lock_irqsave(&ch->lock, flags); 7035 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7036 spin_unlock_irqrestore(&ch->lock, flags); 7037 7038 stmmac_stop_rx_dma(priv, queue); 7039 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7040 } 7041 7042 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 7043 { 7044 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7045 struct stmmac_channel *ch = &priv->channel[queue]; 7046 unsigned long flags; 7047 int ret; 7048 7049 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7050 if (ret) { 7051 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 7052 return; 7053 } 7054 7055 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 7056 if (ret) { 7057 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7058 netdev_err(priv->dev, "Failed to init RX desc.\n"); 7059 return; 7060 } 7061 7062 stmmac_reset_rx_queue(priv, queue); 7063 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 7064 7065 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7066 rx_q->dma_rx_phy, queue); 7067 7068 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->buf_alloc_num); 7069 7070 stmmac_set_queue_rx_buf_size(priv, rx_q, queue); 7071 7072 stmmac_start_rx_dma(priv, queue); 7073 7074 spin_lock_irqsave(&ch->lock, flags); 7075 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7076 spin_unlock_irqrestore(&ch->lock, flags); 7077 } 7078 7079 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 7080 { 7081 struct stmmac_channel *ch = &priv->channel[queue]; 7082 unsigned long flags; 7083 7084 spin_lock_irqsave(&ch->lock, flags); 7085 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7086 spin_unlock_irqrestore(&ch->lock, flags); 7087 7088 stmmac_stop_tx_dma(priv, queue); 7089 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7090 } 7091 7092 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 7093 { 7094 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7095 struct stmmac_channel *ch = &priv->channel[queue]; 7096 unsigned long flags; 7097 int ret; 7098 7099 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7100 if (ret) { 7101 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 7102 return; 7103 } 7104 7105 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 7106 if (ret) { 7107 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7108 netdev_err(priv->dev, "Failed to init TX desc.\n"); 7109 return; 7110 } 7111 7112 stmmac_reset_tx_queue(priv, queue); 7113 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 7114 7115 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7116 tx_q->dma_tx_phy, queue); 7117 7118 if (tx_q->tbs & STMMAC_TBS_AVAIL) 7119 stmmac_enable_tbs(priv, priv->ioaddr, 1, queue); 7120 7121 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, 0); 7122 7123 stmmac_start_tx_dma(priv, queue); 7124 7125 spin_lock_irqsave(&ch->lock, flags); 7126 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7127 spin_unlock_irqrestore(&ch->lock, flags); 7128 } 7129 7130 void stmmac_xdp_release(struct net_device *dev) 7131 { 7132 struct stmmac_priv *priv = netdev_priv(dev); 7133 u8 chan; 7134 7135 /* Ensure tx function is not running */ 7136 netif_tx_disable(dev); 7137 7138 /* Disable NAPI process */ 7139 stmmac_disable_all_queues(priv); 7140 7141 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7142 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7143 7144 /* Free the IRQ lines */ 7145 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 7146 7147 /* Stop TX/RX DMA channels */ 7148 stmmac_stop_all_dma(priv); 7149 7150 /* Release and free the Rx/Tx resources */ 7151 free_dma_desc_resources(priv, &priv->dma_conf); 7152 7153 /* Disable the MAC Rx/Tx */ 7154 stmmac_mac_set(priv, priv->ioaddr, false); 7155 7156 /* set trans_start so we don't get spurious 7157 * watchdogs during reset 7158 */ 7159 netif_trans_update(dev); 7160 netif_carrier_off(dev); 7161 } 7162 7163 int stmmac_xdp_open(struct net_device *dev) 7164 { 7165 struct stmmac_priv *priv = netdev_priv(dev); 7166 u8 rx_cnt = priv->plat->rx_queues_to_use; 7167 u8 tx_cnt = priv->plat->tx_queues_to_use; 7168 u8 dma_csr_ch = max(rx_cnt, tx_cnt); 7169 struct stmmac_rx_queue *rx_q; 7170 struct stmmac_tx_queue *tx_q; 7171 bool sph_en; 7172 u8 chan; 7173 int ret; 7174 7175 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 7176 if (ret < 0) { 7177 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 7178 __func__); 7179 goto dma_desc_error; 7180 } 7181 7182 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 7183 if (ret < 0) { 7184 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 7185 __func__); 7186 goto init_error; 7187 } 7188 7189 stmmac_reset_queues_param(priv); 7190 7191 /* DMA CSR Channel configuration */ 7192 for (chan = 0; chan < dma_csr_ch; chan++) { 7193 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 7194 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 7195 } 7196 7197 /* Adjust Split header */ 7198 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 7199 7200 /* DMA RX Channel Configuration */ 7201 for (chan = 0; chan < rx_cnt; chan++) { 7202 rx_q = &priv->dma_conf.rx_queue[chan]; 7203 7204 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7205 rx_q->dma_rx_phy, chan); 7206 7207 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 7208 rx_q->buf_alloc_num); 7209 7210 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 7211 7212 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 7213 } 7214 7215 /* DMA TX Channel Configuration */ 7216 for (chan = 0; chan < tx_cnt; chan++) { 7217 tx_q = &priv->dma_conf.tx_queue[chan]; 7218 7219 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7220 tx_q->dma_tx_phy, chan); 7221 7222 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 7223 7224 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 7225 } 7226 7227 /* Enable the MAC Rx/Tx */ 7228 stmmac_mac_set(priv, priv->ioaddr, true); 7229 7230 /* Start Rx & Tx DMA Channels */ 7231 stmmac_start_all_dma(priv); 7232 7233 ret = stmmac_request_irq(dev); 7234 if (ret) 7235 goto irq_error; 7236 7237 /* Enable NAPI process*/ 7238 stmmac_enable_all_queues(priv); 7239 netif_carrier_on(dev); 7240 netif_tx_start_all_queues(dev); 7241 stmmac_enable_all_dma_irq(priv); 7242 7243 return 0; 7244 7245 irq_error: 7246 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7247 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7248 7249 init_error: 7250 free_dma_desc_resources(priv, &priv->dma_conf); 7251 dma_desc_error: 7252 return ret; 7253 } 7254 7255 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 7256 { 7257 struct stmmac_priv *priv = netdev_priv(dev); 7258 struct stmmac_rx_queue *rx_q; 7259 struct stmmac_tx_queue *tx_q; 7260 struct stmmac_channel *ch; 7261 7262 if (test_bit(STMMAC_DOWN, &priv->state) || 7263 !netif_carrier_ok(priv->dev)) 7264 return -ENETDOWN; 7265 7266 if (!stmmac_xdp_is_enabled(priv)) 7267 return -EINVAL; 7268 7269 if (queue >= priv->plat->rx_queues_to_use || 7270 queue >= priv->plat->tx_queues_to_use) 7271 return -EINVAL; 7272 7273 rx_q = &priv->dma_conf.rx_queue[queue]; 7274 tx_q = &priv->dma_conf.tx_queue[queue]; 7275 ch = &priv->channel[queue]; 7276 7277 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 7278 return -EINVAL; 7279 7280 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 7281 /* EQoS does not have per-DMA channel SW interrupt, 7282 * so we schedule RX Napi straight-away. 7283 */ 7284 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 7285 __napi_schedule(&ch->rxtx_napi); 7286 } 7287 7288 return 0; 7289 } 7290 7291 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7292 { 7293 struct stmmac_priv *priv = netdev_priv(dev); 7294 u8 tx_cnt = priv->plat->tx_queues_to_use; 7295 u8 rx_cnt = priv->plat->rx_queues_to_use; 7296 unsigned int start; 7297 u8 q; 7298 7299 for (q = 0; q < tx_cnt; q++) { 7300 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7301 u64 tx_packets; 7302 u64 tx_bytes; 7303 7304 do { 7305 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 7306 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 7307 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 7308 do { 7309 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 7310 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 7311 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 7312 7313 stats->tx_packets += tx_packets; 7314 stats->tx_bytes += tx_bytes; 7315 } 7316 7317 for (q = 0; q < rx_cnt; q++) { 7318 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7319 u64 rx_packets; 7320 u64 rx_bytes; 7321 7322 do { 7323 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 7324 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 7325 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 7326 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 7327 7328 stats->rx_packets += rx_packets; 7329 stats->rx_bytes += rx_bytes; 7330 } 7331 7332 stats->rx_dropped = priv->xstats.rx_dropped; 7333 stats->rx_errors = priv->xstats.rx_errors; 7334 stats->tx_dropped = priv->xstats.tx_dropped; 7335 stats->tx_errors = priv->xstats.tx_errors; 7336 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7337 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7338 stats->rx_length_errors = priv->xstats.rx_length; 7339 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7340 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7341 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7342 } 7343 7344 static const struct net_device_ops stmmac_netdev_ops = { 7345 .ndo_open = stmmac_open, 7346 .ndo_start_xmit = stmmac_xmit, 7347 .ndo_features_check = stmmac_features_check, 7348 .ndo_stop = stmmac_release, 7349 .ndo_change_mtu = stmmac_change_mtu, 7350 .ndo_fix_features = stmmac_fix_features, 7351 .ndo_set_features = stmmac_set_features, 7352 .ndo_set_rx_mode = stmmac_set_rx_mode, 7353 .ndo_tx_timeout = stmmac_tx_timeout, 7354 .ndo_eth_ioctl = stmmac_ioctl, 7355 .ndo_get_stats64 = stmmac_get_stats64, 7356 .ndo_setup_tc = stmmac_setup_tc, 7357 .ndo_select_queue = stmmac_select_queue, 7358 .ndo_set_mac_address = stmmac_set_mac_address, 7359 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7360 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7361 .ndo_bpf = stmmac_bpf, 7362 .ndo_xdp_xmit = stmmac_xdp_xmit, 7363 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7364 .ndo_hwtstamp_get = stmmac_hwtstamp_get, 7365 .ndo_hwtstamp_set = stmmac_hwtstamp_set, 7366 }; 7367 7368 static void stmmac_reset_subtask(struct stmmac_priv *priv) 7369 { 7370 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7371 return; 7372 if (test_bit(STMMAC_DOWN, &priv->state)) 7373 return; 7374 7375 netdev_err(priv->dev, "Reset adapter.\n"); 7376 7377 rtnl_lock(); 7378 netif_trans_update(priv->dev); 7379 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7380 usleep_range(1000, 2000); 7381 7382 set_bit(STMMAC_DOWN, &priv->state); 7383 dev_close(priv->dev); 7384 dev_open(priv->dev, NULL); 7385 clear_bit(STMMAC_DOWN, &priv->state); 7386 clear_bit(STMMAC_RESETING, &priv->state); 7387 rtnl_unlock(); 7388 } 7389 7390 static void stmmac_service_task(struct work_struct *work) 7391 { 7392 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7393 service_task); 7394 7395 stmmac_reset_subtask(priv); 7396 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7397 } 7398 7399 static void stmmac_print_actphyif(struct stmmac_priv *priv) 7400 { 7401 const char **phyif_table; 7402 const char *actphyif_str; 7403 size_t phyif_table_size; 7404 7405 switch (priv->plat->core_type) { 7406 case DWMAC_CORE_MAC100: 7407 return; 7408 7409 case DWMAC_CORE_GMAC: 7410 case DWMAC_CORE_GMAC4: 7411 phyif_table = stmmac_dwmac_actphyif; 7412 phyif_table_size = ARRAY_SIZE(stmmac_dwmac_actphyif); 7413 break; 7414 7415 case DWMAC_CORE_XGMAC: 7416 phyif_table = stmmac_dwxgmac_phyif; 7417 phyif_table_size = ARRAY_SIZE(stmmac_dwxgmac_phyif); 7418 break; 7419 } 7420 7421 if (priv->dma_cap.actphyif < phyif_table_size) 7422 actphyif_str = phyif_table[priv->dma_cap.actphyif]; 7423 else 7424 actphyif_str = NULL; 7425 7426 if (!actphyif_str) 7427 actphyif_str = "unknown"; 7428 7429 dev_info(priv->device, "Active PHY interface: %s (%u)\n", 7430 actphyif_str, priv->dma_cap.actphyif); 7431 } 7432 7433 /** 7434 * stmmac_hw_init - Init the MAC device 7435 * @priv: driver private structure 7436 * Description: this function is to configure the MAC device according to 7437 * some platform parameters or the HW capability register. It prepares the 7438 * driver to use either ring or chain modes and to setup either enhanced or 7439 * normal descriptors. 7440 */ 7441 static int stmmac_hw_init(struct stmmac_priv *priv) 7442 { 7443 int ret; 7444 7445 /* dwmac-sun8i only work in chain mode */ 7446 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7447 chain_mode = 1; 7448 priv->chain_mode = !!chain_mode; 7449 7450 /* Initialize HW Interface */ 7451 ret = stmmac_hwif_init(priv); 7452 if (ret) 7453 return ret; 7454 7455 /* Get the HW capability (new GMAC newer than 3.50a) */ 7456 priv->hw_cap_support = stmmac_get_hw_features(priv); 7457 if (priv->hw_cap_support) { 7458 dev_info(priv->device, "DMA HW capability register supported\n"); 7459 7460 /* We can override some gmac/dma configuration fields: e.g. 7461 * enh_desc, tx_coe (e.g. that are passed through the 7462 * platform) with the values from the HW capability 7463 * register (if supported). 7464 */ 7465 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7466 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7467 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7468 if (priv->dma_cap.hash_tb_sz) { 7469 priv->hw->multicast_filter_bins = 7470 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7471 priv->hw->mcast_bits_log2 = 7472 ilog2(priv->hw->multicast_filter_bins); 7473 } 7474 7475 /* TXCOE doesn't work in thresh DMA mode */ 7476 if (priv->plat->force_thresh_dma_mode) 7477 priv->plat->tx_coe = false; 7478 else 7479 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7480 7481 /* In case of GMAC4 rx_coe is from HW cap register. */ 7482 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7483 7484 if (priv->dma_cap.rx_coe_type2) 7485 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7486 else if (priv->dma_cap.rx_coe_type1) 7487 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7488 7489 stmmac_print_actphyif(priv); 7490 } else { 7491 dev_info(priv->device, "No HW DMA feature register supported\n"); 7492 } 7493 7494 if (priv->plat->rx_coe) { 7495 priv->hw->rx_csum = priv->plat->rx_coe; 7496 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7497 if (priv->synopsys_id < DWMAC_CORE_4_00) 7498 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7499 } 7500 if (priv->plat->tx_coe) 7501 dev_info(priv->device, "TX Checksum insertion supported\n"); 7502 7503 if (priv->plat->pmt) { 7504 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7505 device_set_wakeup_capable(priv->device, 1); 7506 devm_pm_set_wake_irq(priv->device, priv->wol_irq); 7507 } 7508 7509 if (priv->dma_cap.number_rx_queues && 7510 priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) { 7511 dev_warn(priv->device, 7512 "Number of Rx queues (%u) exceeds dma capability\n", 7513 priv->plat->rx_queues_to_use); 7514 priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues; 7515 } 7516 if (priv->dma_cap.number_tx_queues && 7517 priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) { 7518 dev_warn(priv->device, 7519 "Number of Tx queues (%u) exceeds dma capability\n", 7520 priv->plat->tx_queues_to_use); 7521 priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues; 7522 } 7523 7524 if (priv->dma_cap.rx_fifo_size && 7525 priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) { 7526 dev_warn(priv->device, 7527 "Rx FIFO size (%u) exceeds dma capability\n", 7528 priv->plat->rx_fifo_size); 7529 priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size; 7530 } 7531 if (priv->dma_cap.tx_fifo_size && 7532 priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) { 7533 dev_warn(priv->device, 7534 "Tx FIFO size (%u) exceeds dma capability\n", 7535 priv->plat->tx_fifo_size); 7536 priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size; 7537 } 7538 7539 priv->hw->vlan_fail_q_en = 7540 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7541 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7542 7543 /* Run HW quirks, if any */ 7544 if (priv->hwif_quirks) { 7545 ret = priv->hwif_quirks(priv); 7546 if (ret) 7547 return ret; 7548 } 7549 7550 /* Set alternate descriptor size (which tells the hardware that 7551 * descriptors are 8 32-bit words) when using extended descriptors 7552 * with ring mode. Only applicable for pre-v4.0 cores. Platform glue 7553 * is not expected to change this. 7554 */ 7555 priv->plat->dma_cfg->atds = priv->extend_desc && 7556 priv->descriptor_mode == STMMAC_RING_MODE; 7557 7558 /* Rx Watchdog is available in the COREs newer than the 3.40. 7559 * In some case, for example on bugged HW this feature 7560 * has to be disable and this can be done by passing the 7561 * riwt_off field from the platform. 7562 */ 7563 if ((priv->synopsys_id >= DWMAC_CORE_3_50 || 7564 priv->plat->core_type == DWMAC_CORE_XGMAC) && 7565 !priv->plat->riwt_off) { 7566 priv->use_riwt = 1; 7567 dev_info(priv->device, 7568 "Enable RX Mitigation via HW Watchdog Timer\n"); 7569 } 7570 7571 /* Unimplemented PCS init (as indicated by stmmac_do_callback() 7572 * perversely returning -EINVAL) is non-fatal. 7573 */ 7574 ret = stmmac_mac_pcs_init(priv); 7575 if (ret != -EINVAL) 7576 return ret; 7577 7578 return 0; 7579 } 7580 7581 static void stmmac_napi_add(struct net_device *dev) 7582 { 7583 struct stmmac_priv *priv = netdev_priv(dev); 7584 u8 queue, maxq; 7585 7586 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7587 7588 for (queue = 0; queue < maxq; queue++) { 7589 struct stmmac_channel *ch = &priv->channel[queue]; 7590 7591 ch->priv_data = priv; 7592 ch->index = queue; 7593 spin_lock_init(&ch->lock); 7594 7595 if (queue < priv->plat->rx_queues_to_use) { 7596 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7597 } 7598 if (queue < priv->plat->tx_queues_to_use) { 7599 netif_napi_add_tx(dev, &ch->tx_napi, 7600 stmmac_napi_poll_tx); 7601 } 7602 if (queue < priv->plat->rx_queues_to_use && 7603 queue < priv->plat->tx_queues_to_use) { 7604 netif_napi_add(dev, &ch->rxtx_napi, 7605 stmmac_napi_poll_rxtx); 7606 } 7607 } 7608 } 7609 7610 static void stmmac_napi_del(struct net_device *dev) 7611 { 7612 struct stmmac_priv *priv = netdev_priv(dev); 7613 u8 queue, maxq; 7614 7615 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7616 7617 for (queue = 0; queue < maxq; queue++) { 7618 struct stmmac_channel *ch = &priv->channel[queue]; 7619 7620 if (queue < priv->plat->rx_queues_to_use) 7621 netif_napi_del(&ch->rx_napi); 7622 if (queue < priv->plat->tx_queues_to_use) 7623 netif_napi_del(&ch->tx_napi); 7624 if (queue < priv->plat->rx_queues_to_use && 7625 queue < priv->plat->tx_queues_to_use) { 7626 netif_napi_del(&ch->rxtx_napi); 7627 } 7628 } 7629 } 7630 7631 int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt) 7632 { 7633 struct stmmac_priv *priv = netdev_priv(dev); 7634 int ret = 0, i; 7635 7636 if (netif_running(dev)) 7637 stmmac_release(dev); 7638 7639 stmmac_napi_del(dev); 7640 7641 priv->plat->rx_queues_to_use = rx_cnt; 7642 priv->plat->tx_queues_to_use = tx_cnt; 7643 if (!netif_is_rxfh_configured(dev)) 7644 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7645 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7646 rx_cnt); 7647 7648 stmmac_napi_add(dev); 7649 7650 if (netif_running(dev)) 7651 ret = stmmac_open(dev); 7652 7653 return ret; 7654 } 7655 7656 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7657 { 7658 struct stmmac_priv *priv = netdev_priv(dev); 7659 int ret = 0; 7660 7661 if (netif_running(dev)) 7662 stmmac_release(dev); 7663 7664 priv->dma_conf.dma_rx_size = rx_size; 7665 priv->dma_conf.dma_tx_size = tx_size; 7666 7667 if (netif_running(dev)) 7668 ret = stmmac_open(dev); 7669 7670 return ret; 7671 } 7672 7673 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7674 { 7675 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7676 struct dma_desc *desc_contains_ts = ctx->desc; 7677 struct stmmac_priv *priv = ctx->priv; 7678 struct dma_desc *ndesc = ctx->ndesc; 7679 struct dma_desc *desc = ctx->desc; 7680 u64 ns = 0; 7681 7682 if (!priv->hwts_rx_en) 7683 return -ENODATA; 7684 7685 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7686 if (dwmac_is_xmac(priv->plat->core_type)) 7687 desc_contains_ts = ndesc; 7688 7689 /* Check if timestamp is available */ 7690 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7691 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7692 ns -= priv->plat->cdc_error_adj; 7693 *timestamp = ns_to_ktime(ns); 7694 return 0; 7695 } 7696 7697 return -ENODATA; 7698 } 7699 7700 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7701 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7702 }; 7703 7704 static int stmmac_dl_ts_coarse_set(struct devlink *dl, u32 id, 7705 struct devlink_param_gset_ctx *ctx, 7706 struct netlink_ext_ack *extack) 7707 { 7708 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7709 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7710 7711 priv->tsfupdt_coarse = ctx->val.vbool; 7712 7713 if (priv->tsfupdt_coarse) 7714 priv->systime_flags &= ~PTP_TCR_TSCFUPDT; 7715 else 7716 priv->systime_flags |= PTP_TCR_TSCFUPDT; 7717 7718 /* In Coarse mode, we can use a smaller subsecond increment, let's 7719 * reconfigure the systime, subsecond increment and addend. 7720 */ 7721 stmmac_update_subsecond_increment(priv); 7722 7723 return 0; 7724 } 7725 7726 static int stmmac_dl_ts_coarse_get(struct devlink *dl, u32 id, 7727 struct devlink_param_gset_ctx *ctx, 7728 struct netlink_ext_ack *extack) 7729 { 7730 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7731 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7732 7733 ctx->val.vbool = priv->tsfupdt_coarse; 7734 7735 return 0; 7736 } 7737 7738 static const struct devlink_param stmmac_devlink_params[] = { 7739 DEVLINK_PARAM_DRIVER(STMMAC_DEVLINK_PARAM_ID_TS_COARSE, "phc_coarse_adj", 7740 DEVLINK_PARAM_TYPE_BOOL, 7741 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 7742 stmmac_dl_ts_coarse_get, 7743 stmmac_dl_ts_coarse_set, NULL), 7744 }; 7745 7746 /* None of the generic devlink parameters are implemented */ 7747 static const struct devlink_ops stmmac_devlink_ops = {}; 7748 7749 static int stmmac_register_devlink(struct stmmac_priv *priv) 7750 { 7751 struct stmmac_devlink_priv *dl_priv; 7752 int ret; 7753 7754 /* For now, what is exposed over devlink is only relevant when 7755 * timestamping is available and we have a valid ptp clock rate 7756 */ 7757 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp) || 7758 !priv->plat->clk_ptp_rate) 7759 return 0; 7760 7761 priv->devlink = devlink_alloc(&stmmac_devlink_ops, sizeof(*dl_priv), 7762 priv->device); 7763 if (!priv->devlink) 7764 return -ENOMEM; 7765 7766 dl_priv = devlink_priv(priv->devlink); 7767 dl_priv->stmmac_priv = priv; 7768 7769 ret = devlink_params_register(priv->devlink, stmmac_devlink_params, 7770 ARRAY_SIZE(stmmac_devlink_params)); 7771 if (ret) 7772 goto dl_free; 7773 7774 devlink_register(priv->devlink); 7775 return 0; 7776 7777 dl_free: 7778 devlink_free(priv->devlink); 7779 7780 return ret; 7781 } 7782 7783 static void stmmac_unregister_devlink(struct stmmac_priv *priv) 7784 { 7785 if (!priv->devlink) 7786 return; 7787 7788 devlink_unregister(priv->devlink); 7789 devlink_params_unregister(priv->devlink, stmmac_devlink_params, 7790 ARRAY_SIZE(stmmac_devlink_params)); 7791 devlink_free(priv->devlink); 7792 } 7793 7794 struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev) 7795 { 7796 struct plat_stmmacenet_data *plat_dat; 7797 int i; 7798 7799 plat_dat = devm_kzalloc(dev, sizeof(*plat_dat), GFP_KERNEL); 7800 if (!plat_dat) 7801 return NULL; 7802 7803 plat_dat->dma_cfg = &plat_dat->__dma_cfg; 7804 7805 /* Set the defaults: 7806 * - phy autodetection 7807 * - determine GMII_Address CR field from CSR clock 7808 * - allow MTU up to JUMBO_LEN 7809 * - hash table size 7810 * - one unicast filter entry 7811 */ 7812 plat_dat->phy_addr = -1; 7813 plat_dat->clk_csr = -1; 7814 plat_dat->maxmtu = JUMBO_LEN; 7815 plat_dat->multicast_filter_bins = HASH_TABLE_SIZE; 7816 plat_dat->unicast_filter_entries = 1; 7817 7818 /* Set the mtl defaults */ 7819 plat_dat->tx_queues_to_use = 1; 7820 plat_dat->rx_queues_to_use = 1; 7821 7822 /* Setup the default RX queue channel map */ 7823 for (i = 0; i < ARRAY_SIZE(plat_dat->rx_queues_cfg); i++) 7824 plat_dat->rx_queues_cfg[i].chan = i; 7825 7826 return plat_dat; 7827 } 7828 EXPORT_SYMBOL_GPL(stmmac_plat_dat_alloc); 7829 7830 static int __stmmac_dvr_probe(struct device *device, 7831 struct plat_stmmacenet_data *plat_dat, 7832 struct stmmac_resources *res) 7833 { 7834 struct net_device *ndev = NULL; 7835 struct stmmac_priv *priv; 7836 int i, ret = 0; 7837 u8 rxq; 7838 7839 if (!plat_dat->dma_cfg || !plat_dat->dma_cfg->pbl) { 7840 dev_err(device, "invalid DMA configuration\n"); 7841 return -EINVAL; 7842 } 7843 7844 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7845 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7846 if (!ndev) 7847 return -ENOMEM; 7848 7849 SET_NETDEV_DEV(ndev, device); 7850 7851 priv = netdev_priv(ndev); 7852 priv->device = device; 7853 priv->dev = ndev; 7854 7855 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7856 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7857 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7858 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7859 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7860 } 7861 7862 priv->xstats.pcpu_stats = 7863 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7864 if (!priv->xstats.pcpu_stats) 7865 return -ENOMEM; 7866 7867 stmmac_set_ethtool_ops(ndev); 7868 priv->pause_time = pause; 7869 priv->plat = plat_dat; 7870 priv->ioaddr = res->addr; 7871 priv->dev->base_addr = (unsigned long)res->addr; 7872 priv->plat->dma_cfg->multi_msi_en = 7873 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7874 7875 priv->dev->irq = res->irq; 7876 priv->wol_irq = res->wol_irq; 7877 priv->sfty_irq = res->sfty_irq; 7878 7879 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) { 7880 ret = stmmac_msi_init(priv, res); 7881 if (ret) 7882 return ret; 7883 } 7884 7885 if (!is_zero_ether_addr(res->mac)) 7886 eth_hw_addr_set(priv->dev, res->mac); 7887 7888 dev_set_drvdata(device, priv->dev); 7889 7890 /* Verify driver arguments */ 7891 stmmac_verify_args(); 7892 7893 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7894 if (!priv->af_xdp_zc_qps) 7895 return -ENOMEM; 7896 7897 /* Allocate workqueue */ 7898 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7899 if (!priv->wq) { 7900 dev_err(priv->device, "failed to create workqueue\n"); 7901 ret = -ENOMEM; 7902 goto error_wq_init; 7903 } 7904 7905 INIT_WORK(&priv->service_task, stmmac_service_task); 7906 7907 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 7908 7909 /* Override with kernel parameters if supplied XXX CRS XXX 7910 * this needs to have multiple instances 7911 */ 7912 if ((phyaddr >= 0) && (phyaddr <= 31)) 7913 priv->plat->phy_addr = phyaddr; 7914 7915 if (priv->plat->stmmac_rst) { 7916 ret = reset_control_assert(priv->plat->stmmac_rst); 7917 reset_control_deassert(priv->plat->stmmac_rst); 7918 /* Some reset controllers have only reset callback instead of 7919 * assert + deassert callbacks pair. 7920 */ 7921 if (ret == -ENOTSUPP) 7922 reset_control_reset(priv->plat->stmmac_rst); 7923 } 7924 7925 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7926 if (ret == -ENOTSUPP) 7927 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7928 ERR_PTR(ret)); 7929 7930 /* Wait a bit for the reset to take effect */ 7931 udelay(10); 7932 7933 /* Init MAC and get the capabilities */ 7934 ret = stmmac_hw_init(priv); 7935 if (ret) 7936 goto error_hw_init; 7937 7938 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7939 */ 7940 if (priv->synopsys_id < DWMAC_CORE_5_20) 7941 priv->plat->dma_cfg->dche = false; 7942 7943 stmmac_check_ether_addr(priv); 7944 7945 ndev->netdev_ops = &stmmac_netdev_ops; 7946 7947 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7948 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7949 7950 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7951 NETIF_F_RXCSUM; 7952 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7953 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7954 7955 ret = stmmac_tc_init(priv, priv); 7956 if (!ret) { 7957 ndev->hw_features |= NETIF_F_HW_TC; 7958 } 7959 7960 stmmac_set_gso_features(ndev); 7961 7962 if (priv->dma_cap.sphen && 7963 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7964 ndev->hw_features |= NETIF_F_GRO; 7965 priv->sph_capable = true; 7966 priv->sph_active = priv->sph_capable; 7967 dev_info(priv->device, "SPH feature enabled\n"); 7968 } 7969 7970 /* Ideally our host DMA address width is the same as for the 7971 * device. However, it may differ and then we have to use our 7972 * host DMA width for allocation and the device DMA width for 7973 * register handling. 7974 */ 7975 if (priv->plat->host_dma_width) 7976 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7977 else 7978 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7979 7980 if (priv->dma_cap.host_dma_width) { 7981 ret = dma_set_mask_and_coherent(device, 7982 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7983 if (!ret) { 7984 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7985 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7986 7987 /* 7988 * If more than 32 bits can be addressed, make sure to 7989 * enable enhanced addressing mode. 7990 */ 7991 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7992 priv->plat->dma_cfg->eame = true; 7993 } else { 7994 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7995 if (ret) { 7996 dev_err(priv->device, "Failed to set DMA Mask\n"); 7997 goto error_hw_init; 7998 } 7999 8000 priv->dma_cap.host_dma_width = 32; 8001 } 8002 } 8003 8004 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 8005 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 8006 #ifdef STMMAC_VLAN_TAG_USED 8007 /* Both mac100 and gmac support receive VLAN tag detection */ 8008 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; 8009 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 8010 ndev->features |= NETIF_F_HW_VLAN_STAG_RX; 8011 8012 if (dwmac_is_xmac(priv->plat->core_type)) { 8013 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 8014 priv->hw->hw_vlan_en = true; 8015 } 8016 if (priv->dma_cap.vlhash) { 8017 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 8018 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 8019 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 8020 } 8021 if (priv->dma_cap.vlins) 8022 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 8023 #endif 8024 priv->msg_enable = netif_msg_init(debug, default_msg_level); 8025 8026 priv->xstats.threshold = tc; 8027 8028 /* Initialize RSS */ 8029 rxq = priv->plat->rx_queues_to_use; 8030 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 8031 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 8032 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 8033 8034 if (priv->dma_cap.rssen && priv->plat->rss_en) 8035 ndev->features |= NETIF_F_RXHASH; 8036 8037 ndev->vlan_features |= ndev->features; 8038 8039 /* MTU range: 46 - hw-specific max */ 8040 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 8041 8042 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 8043 ndev->max_mtu = XGMAC_JUMBO_LEN; 8044 else if (priv->plat->enh_desc || priv->synopsys_id >= DWMAC_CORE_4_00) 8045 ndev->max_mtu = JUMBO_LEN; 8046 else 8047 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 8048 8049 /* Warn if the platform's maxmtu is smaller than the minimum MTU, 8050 * otherwise clamp the maximum MTU above to the platform's maxmtu. 8051 */ 8052 if (priv->plat->maxmtu < ndev->min_mtu) 8053 dev_warn(priv->device, 8054 "%s: warning: maxmtu having invalid value (%d)\n", 8055 __func__, priv->plat->maxmtu); 8056 else if (priv->plat->maxmtu < ndev->max_mtu) 8057 ndev->max_mtu = priv->plat->maxmtu; 8058 8059 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 8060 8061 /* Setup channels NAPI */ 8062 stmmac_napi_add(ndev); 8063 8064 mutex_init(&priv->lock); 8065 rwlock_init(&priv->ptp_lock); 8066 8067 stmmac_fpe_init(priv); 8068 8069 stmmac_check_pcs_mode(priv); 8070 8071 pm_runtime_get_noresume(device); 8072 pm_runtime_set_active(device); 8073 if (!pm_runtime_enabled(device)) 8074 pm_runtime_enable(device); 8075 8076 ret = stmmac_mdio_register(ndev); 8077 if (ret < 0) { 8078 dev_err_probe(priv->device, ret, 8079 "MDIO bus (id: %d) registration failed\n", 8080 priv->plat->bus_id); 8081 goto error_mdio_register; 8082 } 8083 8084 ret = stmmac_pcs_setup(ndev); 8085 if (ret) 8086 goto error_pcs_setup; 8087 8088 ret = stmmac_phylink_setup(priv); 8089 if (ret) { 8090 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 8091 goto error_phy_setup; 8092 } 8093 8094 ret = stmmac_register_devlink(priv); 8095 if (ret) 8096 goto error_devlink_setup; 8097 8098 ret = register_netdev(ndev); 8099 if (ret) { 8100 dev_err(priv->device, "%s: ERROR %i registering the device\n", 8101 __func__, ret); 8102 goto error_netdev_register; 8103 } 8104 8105 #ifdef CONFIG_DEBUG_FS 8106 stmmac_init_fs(ndev); 8107 #endif 8108 8109 if (priv->plat->dump_debug_regs) 8110 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 8111 8112 /* Let pm_runtime_put() disable the clocks. 8113 * If CONFIG_PM is not enabled, the clocks will stay powered. 8114 */ 8115 pm_runtime_put(device); 8116 8117 return ret; 8118 8119 error_netdev_register: 8120 stmmac_unregister_devlink(priv); 8121 error_devlink_setup: 8122 phylink_destroy(priv->phylink); 8123 error_phy_setup: 8124 stmmac_pcs_clean(ndev); 8125 error_pcs_setup: 8126 stmmac_mdio_unregister(ndev); 8127 error_mdio_register: 8128 stmmac_napi_del(ndev); 8129 error_hw_init: 8130 destroy_workqueue(priv->wq); 8131 error_wq_init: 8132 bitmap_free(priv->af_xdp_zc_qps); 8133 8134 return ret; 8135 } 8136 8137 /** 8138 * stmmac_dvr_probe 8139 * @dev: device pointer 8140 * @plat_dat: platform data pointer 8141 * @res: stmmac resource pointer 8142 * Description: this is the main probe function used to 8143 * call the alloc_etherdev, allocate the priv structure. 8144 * Return: 8145 * returns 0 on success, otherwise errno. 8146 */ 8147 int stmmac_dvr_probe(struct device *dev, struct plat_stmmacenet_data *plat_dat, 8148 struct stmmac_resources *res) 8149 { 8150 int ret; 8151 8152 if (plat_dat->init) { 8153 ret = plat_dat->init(dev, plat_dat->bsp_priv); 8154 if (ret) 8155 return ret; 8156 } 8157 8158 ret = __stmmac_dvr_probe(dev, plat_dat, res); 8159 if (ret && plat_dat->exit) 8160 plat_dat->exit(dev, plat_dat->bsp_priv); 8161 8162 return ret; 8163 } 8164 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 8165 8166 /** 8167 * stmmac_dvr_remove 8168 * @dev: device pointer 8169 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 8170 * changes the link status, releases the DMA descriptor rings. 8171 */ 8172 void stmmac_dvr_remove(struct device *dev) 8173 { 8174 struct net_device *ndev = dev_get_drvdata(dev); 8175 struct stmmac_priv *priv = netdev_priv(ndev); 8176 8177 netdev_info(priv->dev, "%s: removing driver", __func__); 8178 8179 pm_runtime_get_sync(dev); 8180 8181 unregister_netdev(ndev); 8182 8183 #ifdef CONFIG_DEBUG_FS 8184 stmmac_exit_fs(ndev); 8185 #endif 8186 stmmac_unregister_devlink(priv); 8187 8188 phylink_destroy(priv->phylink); 8189 if (priv->plat->stmmac_rst) 8190 reset_control_assert(priv->plat->stmmac_rst); 8191 reset_control_assert(priv->plat->stmmac_ahb_rst); 8192 8193 stmmac_pcs_clean(ndev); 8194 stmmac_mdio_unregister(ndev); 8195 8196 destroy_workqueue(priv->wq); 8197 mutex_destroy(&priv->lock); 8198 bitmap_free(priv->af_xdp_zc_qps); 8199 8200 pm_runtime_disable(dev); 8201 pm_runtime_put_noidle(dev); 8202 8203 if (priv->plat->exit) 8204 priv->plat->exit(dev, priv->plat->bsp_priv); 8205 } 8206 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 8207 8208 /** 8209 * stmmac_suspend - suspend callback 8210 * @dev: device pointer 8211 * Description: this is the function to suspend the device and it is called 8212 * by the platform driver to stop the network queue, release the resources, 8213 * program the PMT register (for WoL), clean and release driver resources. 8214 */ 8215 int stmmac_suspend(struct device *dev) 8216 { 8217 struct net_device *ndev = dev_get_drvdata(dev); 8218 struct stmmac_priv *priv = netdev_priv(ndev); 8219 u8 chan; 8220 8221 if (!ndev || !netif_running(ndev)) 8222 goto suspend_bsp; 8223 8224 mutex_lock(&priv->lock); 8225 8226 netif_device_detach(ndev); 8227 8228 stmmac_disable_all_queues(priv); 8229 8230 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 8231 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 8232 8233 if (priv->eee_sw_timer_en) { 8234 priv->tx_path_in_lpi_mode = false; 8235 timer_delete_sync(&priv->eee_ctrl_timer); 8236 } 8237 8238 /* Stop TX/RX DMA */ 8239 stmmac_stop_all_dma(priv); 8240 8241 stmmac_legacy_serdes_power_down(priv); 8242 8243 /* Enable Power down mode by programming the PMT regs */ 8244 if (priv->wolopts) { 8245 stmmac_pmt(priv, priv->hw, priv->wolopts); 8246 priv->irq_wake = 1; 8247 } else { 8248 stmmac_mac_set(priv, priv->ioaddr, false); 8249 pinctrl_pm_select_sleep_state(priv->device); 8250 } 8251 8252 mutex_unlock(&priv->lock); 8253 8254 rtnl_lock(); 8255 phylink_suspend(priv->phylink, !!priv->wolopts); 8256 rtnl_unlock(); 8257 8258 if (stmmac_fpe_supported(priv)) 8259 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 8260 8261 suspend_bsp: 8262 if (priv->plat->suspend) 8263 return priv->plat->suspend(dev, priv->plat->bsp_priv); 8264 8265 return 0; 8266 } 8267 EXPORT_SYMBOL_GPL(stmmac_suspend); 8268 8269 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 8270 { 8271 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 8272 8273 rx_q->cur_rx = 0; 8274 rx_q->dirty_rx = 0; 8275 } 8276 8277 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 8278 { 8279 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 8280 8281 tx_q->cur_tx = 0; 8282 tx_q->dirty_tx = 0; 8283 tx_q->mss = 0; 8284 8285 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 8286 } 8287 8288 /** 8289 * stmmac_reset_queues_param - reset queue parameters 8290 * @priv: device pointer 8291 */ 8292 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 8293 { 8294 u8 rx_cnt = priv->plat->rx_queues_to_use; 8295 u8 tx_cnt = priv->plat->tx_queues_to_use; 8296 u8 queue; 8297 8298 for (queue = 0; queue < rx_cnt; queue++) 8299 stmmac_reset_rx_queue(priv, queue); 8300 8301 for (queue = 0; queue < tx_cnt; queue++) 8302 stmmac_reset_tx_queue(priv, queue); 8303 } 8304 8305 /** 8306 * stmmac_resume - resume callback 8307 * @dev: device pointer 8308 * Description: when resume this function is invoked to setup the DMA and CORE 8309 * in a usable state. 8310 */ 8311 int stmmac_resume(struct device *dev) 8312 { 8313 struct net_device *ndev = dev_get_drvdata(dev); 8314 struct stmmac_priv *priv = netdev_priv(ndev); 8315 int ret; 8316 8317 if (priv->plat->resume) { 8318 ret = priv->plat->resume(dev, priv->plat->bsp_priv); 8319 if (ret) 8320 return ret; 8321 } 8322 8323 if (!netif_running(ndev)) 8324 return 0; 8325 8326 /* Power Down bit, into the PM register, is cleared 8327 * automatically as soon as a magic packet or a Wake-up frame 8328 * is received. Anyway, it's better to manually clear 8329 * this bit because it can generate problems while resuming 8330 * from another devices (e.g. serial console). 8331 */ 8332 if (priv->wolopts) { 8333 mutex_lock(&priv->lock); 8334 stmmac_pmt(priv, priv->hw, 0); 8335 mutex_unlock(&priv->lock); 8336 priv->irq_wake = 0; 8337 } else { 8338 pinctrl_pm_select_default_state(priv->device); 8339 /* reset the phy so that it's ready */ 8340 if (priv->mii) 8341 stmmac_mdio_reset(priv->mii); 8342 } 8343 8344 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 8345 ret = stmmac_legacy_serdes_power_up(priv); 8346 if (ret < 0) 8347 return ret; 8348 } 8349 8350 rtnl_lock(); 8351 8352 /* Prepare the PHY to resume, ensuring that its clocks which are 8353 * necessary for the MAC DMA reset to complete are running 8354 */ 8355 phylink_prepare_resume(priv->phylink); 8356 8357 mutex_lock(&priv->lock); 8358 8359 stmmac_reset_queues_param(priv); 8360 8361 stmmac_free_tx_skbufs(priv); 8362 stmmac_clear_descriptors(priv, &priv->dma_conf); 8363 8364 ret = stmmac_hw_setup(ndev); 8365 if (ret < 0) { 8366 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 8367 stmmac_legacy_serdes_power_down(priv); 8368 mutex_unlock(&priv->lock); 8369 rtnl_unlock(); 8370 return ret; 8371 } 8372 8373 stmmac_init_timestamping(priv); 8374 8375 stmmac_init_coalesce(priv); 8376 phylink_rx_clk_stop_block(priv->phylink); 8377 stmmac_set_rx_mode(ndev); 8378 phylink_rx_clk_stop_unblock(priv->phylink); 8379 8380 stmmac_vlan_restore(priv); 8381 8382 stmmac_enable_all_queues(priv); 8383 stmmac_enable_all_dma_irq(priv); 8384 8385 mutex_unlock(&priv->lock); 8386 8387 /* phylink_resume() must be called after the hardware has been 8388 * initialised because it may bring the link up immediately in a 8389 * workqueue thread, which will race with initialisation. 8390 */ 8391 phylink_resume(priv->phylink); 8392 rtnl_unlock(); 8393 8394 netif_device_attach(ndev); 8395 8396 return 0; 8397 } 8398 EXPORT_SYMBOL_GPL(stmmac_resume); 8399 8400 /* This is not the same as EXPORT_GPL_SIMPLE_DEV_PM_OPS() when CONFIG_PM=n */ 8401 DEFINE_SIMPLE_DEV_PM_OPS(stmmac_simple_pm_ops, stmmac_suspend, stmmac_resume); 8402 EXPORT_SYMBOL_GPL(stmmac_simple_pm_ops); 8403 8404 #ifndef MODULE 8405 static int __init stmmac_cmdline_opt(char *str) 8406 { 8407 char *opt; 8408 8409 if (!str || !*str) 8410 return 1; 8411 while ((opt = strsep(&str, ",")) != NULL) { 8412 if (!strncmp(opt, "debug:", 6)) { 8413 if (kstrtoint(opt + 6, 0, &debug)) 8414 goto err; 8415 } else if (!strncmp(opt, "phyaddr:", 8)) { 8416 if (kstrtoint(opt + 8, 0, &phyaddr)) 8417 goto err; 8418 } else if (!strncmp(opt, "tc:", 3)) { 8419 if (kstrtoint(opt + 3, 0, &tc)) 8420 goto err; 8421 } else if (!strncmp(opt, "watchdog:", 9)) { 8422 if (kstrtoint(opt + 9, 0, &watchdog)) 8423 goto err; 8424 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 8425 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 8426 goto err; 8427 } else if (!strncmp(opt, "pause:", 6)) { 8428 if (kstrtoint(opt + 6, 0, &pause)) 8429 goto err; 8430 } else if (!strncmp(opt, "eee_timer:", 10)) { 8431 if (kstrtoint(opt + 10, 0, &eee_timer)) 8432 goto err; 8433 } else if (!strncmp(opt, "chain_mode:", 11)) { 8434 if (kstrtoint(opt + 11, 0, &chain_mode)) 8435 goto err; 8436 } 8437 } 8438 return 1; 8439 8440 err: 8441 pr_err("%s: ERROR broken module parameter conversion", __func__); 8442 return 1; 8443 } 8444 8445 __setup("stmmaceth=", stmmac_cmdline_opt); 8446 #endif /* MODULE */ 8447 8448 static int __init stmmac_init(void) 8449 { 8450 #ifdef CONFIG_DEBUG_FS 8451 /* Create debugfs main directory if it doesn't exist yet */ 8452 if (!stmmac_fs_dir) 8453 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 8454 register_netdevice_notifier(&stmmac_notifier); 8455 #endif 8456 8457 return 0; 8458 } 8459 8460 static void __exit stmmac_exit(void) 8461 { 8462 #ifdef CONFIG_DEBUG_FS 8463 unregister_netdevice_notifier(&stmmac_notifier); 8464 debugfs_remove_recursive(stmmac_fs_dir); 8465 #endif 8466 } 8467 8468 module_init(stmmac_init) 8469 module_exit(stmmac_exit) 8470 8471 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 8472 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 8473 MODULE_LICENSE("GPL"); 8474