1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/circ_buf.h> 18 #include <linux/clk.h> 19 #include <linux/kernel.h> 20 #include <linux/interrupt.h> 21 #include <linux/ip.h> 22 #include <linux/tcp.h> 23 #include <linux/skbuff.h> 24 #include <linux/ethtool.h> 25 #include <linux/if_ether.h> 26 #include <linux/crc32.h> 27 #include <linux/mii.h> 28 #include <linux/if.h> 29 #include <linux/if_vlan.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/slab.h> 32 #include <linux/pm_runtime.h> 33 #include <linux/pm_wakeirq.h> 34 #include <linux/prefetch.h> 35 #include <linux/pinctrl/consumer.h> 36 #ifdef CONFIG_DEBUG_FS 37 #include <linux/debugfs.h> 38 #include <linux/seq_file.h> 39 #endif /* CONFIG_DEBUG_FS */ 40 #include <linux/net_tstamp.h> 41 #include <linux/phylink.h> 42 #include <linux/udp.h> 43 #include <linux/bpf_trace.h> 44 #include <net/devlink.h> 45 #include <net/page_pool/helpers.h> 46 #include <net/pkt_cls.h> 47 #include <net/xdp_sock_drv.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac_fpe.h" 50 #include "stmmac.h" 51 #include "stmmac_pcs.h" 52 #include "stmmac_xdp.h" 53 #include <linux/reset.h> 54 #include <linux/of_mdio.h> 55 #include "dwmac1000.h" 56 #include "dwxgmac2.h" 57 #include "hwif.h" 58 59 /* As long as the interface is active, we keep the timestamping counter enabled 60 * with fine resolution and binary rollover. This avoid non-monotonic behavior 61 * (clock jumps) when changing timestamping settings at runtime. 62 */ 63 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCTRLSSR) 64 65 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 66 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 67 68 /* Module parameters */ 69 #define TX_TIMEO 5000 70 static int watchdog = TX_TIMEO; 71 module_param(watchdog, int, 0644); 72 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 73 74 static int debug = -1; 75 module_param(debug, int, 0644); 76 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 77 78 static int phyaddr = -1; 79 module_param(phyaddr, int, 0444); 80 MODULE_PARM_DESC(phyaddr, "Physical device address"); 81 82 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 83 84 /* Limit to make sure XDP TX and slow path can coexist */ 85 #define STMMAC_XSK_TX_BUDGET_MAX 256 86 #define STMMAC_TX_XSK_AVAIL 16 87 #define STMMAC_RX_FILL_BATCH 16 88 89 #define STMMAC_XDP_PASS 0 90 #define STMMAC_XDP_CONSUMED BIT(0) 91 #define STMMAC_XDP_TX BIT(1) 92 #define STMMAC_XDP_REDIRECT BIT(2) 93 #define STMMAC_XSK_CONSUMED BIT(3) 94 95 static int flow_ctrl = 0xdead; 96 module_param(flow_ctrl, int, 0644); 97 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)"); 98 99 static int pause = PAUSE_TIME; 100 module_param(pause, int, 0644); 101 MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)"); 102 103 #define TC_DEFAULT 64 104 static int tc = TC_DEFAULT; 105 module_param(tc, int, 0644); 106 MODULE_PARM_DESC(tc, "DMA threshold control value"); 107 108 /* This is unused */ 109 #define DEFAULT_BUFSIZE 1536 110 static int buf_sz = DEFAULT_BUFSIZE; 111 module_param(buf_sz, int, 0644); 112 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 113 114 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 115 NETIF_MSG_LINK | NETIF_MSG_IFUP | 116 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 117 118 #define STMMAC_DEFAULT_LPI_TIMER 1000 119 static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 120 module_param(eee_timer, uint, 0644); 121 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 122 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 123 124 /* By default the driver will use the ring mode to manage tx and rx descriptors, 125 * but allow user to force to use the chain instead of the ring 126 */ 127 static unsigned int chain_mode; 128 module_param(chain_mode, int, 0444); 129 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 130 131 static const char *stmmac_dwmac_actphyif[8] = { 132 [PHY_INTF_SEL_GMII_MII] = "GMII/MII", 133 [PHY_INTF_SEL_RGMII] = "RGMII", 134 [PHY_INTF_SEL_SGMII] = "SGMII", 135 [PHY_INTF_SEL_TBI] = "TBI", 136 [PHY_INTF_SEL_RMII] = "RMII", 137 [PHY_INTF_SEL_RTBI] = "RTBI", 138 [PHY_INTF_SEL_SMII] = "SMII", 139 [PHY_INTF_SEL_REVMII] = "REVMII", 140 }; 141 142 static const char *stmmac_dwxgmac_phyif[4] = { 143 [PHY_INTF_GMII] = "GMII", 144 [PHY_INTF_RGMII] = "RGMII", 145 }; 146 147 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 148 /* For MSI interrupts handling */ 149 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 150 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 151 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 152 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 153 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 154 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 155 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 156 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 157 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 158 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 159 u32 rxmode, u32 chan); 160 static void stmmac_vlan_restore(struct stmmac_priv *priv); 161 162 #ifdef CONFIG_DEBUG_FS 163 static const struct net_device_ops stmmac_netdev_ops; 164 static void stmmac_init_fs(struct net_device *dev); 165 static void stmmac_exit_fs(struct net_device *dev); 166 #endif 167 168 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 169 170 struct stmmac_devlink_priv { 171 struct stmmac_priv *stmmac_priv; 172 }; 173 174 enum stmmac_dl_param_id { 175 STMMAC_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, 176 STMMAC_DEVLINK_PARAM_ID_TS_COARSE, 177 }; 178 179 /** 180 * stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock 181 * @bsp_priv: BSP private data structure (unused) 182 * @clk_tx_i: the transmit clock 183 * @interface: the selected interface mode 184 * @speed: the speed that the MAC will be operating at 185 * 186 * Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps, 187 * 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least 188 * MII, GMII, RGMII and RMII interface modes. Platforms can hook this into 189 * the plat_data->set_clk_tx_rate method directly, call it via their own 190 * implementation, or implement their own method should they have more 191 * complex requirements. It is intended to only be used in this method. 192 * 193 * plat_data->clk_tx_i must be filled in. 194 */ 195 int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, 196 phy_interface_t interface, int speed) 197 { 198 long rate = rgmii_clock(speed); 199 200 /* Silently ignore unsupported speeds as rgmii_clock() only 201 * supports 10, 100 and 1000Mbps. We do not want to spit 202 * errors for 2500 and higher speeds here. 203 */ 204 if (rate < 0) 205 return 0; 206 207 return clk_set_rate(clk_tx_i, rate); 208 } 209 EXPORT_SYMBOL_GPL(stmmac_set_clk_tx_rate); 210 211 /** 212 * stmmac_axi_blen_to_mask() - convert a burst length array to reg value 213 * @regval: pointer to a u32 for the resulting register value 214 * @blen: pointer to an array of u32 containing the burst length values in bytes 215 * @len: the number of entries in the @blen array 216 */ 217 void stmmac_axi_blen_to_mask(u32 *regval, const u32 *blen, size_t len) 218 { 219 size_t i; 220 u32 val; 221 222 for (val = i = 0; i < len; i++) { 223 u32 burst = blen[i]; 224 225 /* Burst values of zero must be skipped. */ 226 if (!burst) 227 continue; 228 229 /* The valid range for the burst length is 4 to 256 inclusive, 230 * and it must be a power of two. 231 */ 232 if (burst < 4 || burst > 256 || !is_power_of_2(burst)) { 233 pr_err("stmmac: invalid burst length %u at index %zu\n", 234 burst, i); 235 continue; 236 } 237 238 /* Since burst is a power of two, and the register field starts 239 * with burst = 4, shift right by two bits so bit 0 of the field 240 * corresponds with the minimum value. 241 */ 242 val |= burst >> 2; 243 } 244 245 *regval = FIELD_PREP(DMA_AXI_BLEN_MASK, val); 246 } 247 EXPORT_SYMBOL_GPL(stmmac_axi_blen_to_mask); 248 249 /** 250 * stmmac_verify_args - verify the driver parameters. 251 * Description: it checks the driver parameters and set a default in case of 252 * errors. 253 */ 254 static void stmmac_verify_args(void) 255 { 256 if (unlikely(watchdog < 0)) 257 watchdog = TX_TIMEO; 258 if (unlikely((pause < 0) || (pause > 0xffff))) 259 pause = PAUSE_TIME; 260 261 if (flow_ctrl != 0xdead) 262 pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n"); 263 } 264 265 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 266 { 267 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 268 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 269 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 270 u8 queue; 271 272 for (queue = 0; queue < maxq; queue++) { 273 struct stmmac_channel *ch = &priv->channel[queue]; 274 275 if (stmmac_xdp_is_enabled(priv) && 276 test_bit(queue, priv->af_xdp_zc_qps)) { 277 napi_disable(&ch->rxtx_napi); 278 continue; 279 } 280 281 if (queue < rx_queues_cnt) 282 napi_disable(&ch->rx_napi); 283 if (queue < tx_queues_cnt) 284 napi_disable(&ch->tx_napi); 285 } 286 } 287 288 /** 289 * stmmac_disable_all_queues - Disable all queues 290 * @priv: driver private structure 291 */ 292 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 293 { 294 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 295 struct stmmac_rx_queue *rx_q; 296 u8 queue; 297 298 /* synchronize_rcu() needed for pending XDP buffers to drain */ 299 for (queue = 0; queue < rx_queues_cnt; queue++) { 300 rx_q = &priv->dma_conf.rx_queue[queue]; 301 if (rx_q->xsk_pool) { 302 synchronize_rcu(); 303 break; 304 } 305 } 306 307 __stmmac_disable_all_queues(priv); 308 } 309 310 /** 311 * stmmac_enable_all_queues - Enable all queues 312 * @priv: driver private structure 313 */ 314 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 315 { 316 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 317 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 318 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 319 u8 queue; 320 321 for (queue = 0; queue < maxq; queue++) { 322 struct stmmac_channel *ch = &priv->channel[queue]; 323 324 if (stmmac_xdp_is_enabled(priv) && 325 test_bit(queue, priv->af_xdp_zc_qps)) { 326 napi_enable(&ch->rxtx_napi); 327 continue; 328 } 329 330 if (queue < rx_queues_cnt) 331 napi_enable(&ch->rx_napi); 332 if (queue < tx_queues_cnt) 333 napi_enable(&ch->tx_napi); 334 } 335 } 336 337 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 338 { 339 if (!test_bit(STMMAC_DOWN, &priv->state) && 340 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 341 queue_work(priv->wq, &priv->service_task); 342 } 343 344 static void stmmac_global_err(struct stmmac_priv *priv) 345 { 346 netif_carrier_off(priv->dev); 347 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 348 stmmac_service_event_schedule(priv); 349 } 350 351 static void print_pkt(unsigned char *buf, int len) 352 { 353 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 354 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 355 } 356 357 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 358 { 359 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 360 361 return CIRC_SPACE(tx_q->cur_tx, tx_q->dirty_tx, 362 priv->dma_conf.dma_tx_size); 363 } 364 365 static size_t stmmac_get_tx_desc_size(struct stmmac_priv *priv, 366 struct stmmac_tx_queue *tx_q) 367 { 368 if (priv->extend_desc) 369 return sizeof(struct dma_extended_desc); 370 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 371 return sizeof(struct dma_edesc); 372 else 373 return sizeof(struct dma_desc); 374 } 375 376 static struct dma_desc *stmmac_get_tx_desc(struct stmmac_priv *priv, 377 struct stmmac_tx_queue *tx_q, 378 unsigned int index) 379 { 380 if (priv->extend_desc) 381 return &tx_q->dma_etx[index].basic; 382 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 383 return &tx_q->dma_entx[index].basic; 384 else 385 return &tx_q->dma_tx[index]; 386 } 387 388 static void stmmac_set_queue_tx_tail_ptr(struct stmmac_priv *priv, 389 struct stmmac_tx_queue *tx_q, 390 unsigned int chan, unsigned int index) 391 { 392 size_t desc_size; 393 u32 tx_tail_addr; 394 395 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 396 397 tx_tail_addr = tx_q->dma_tx_phy + index * desc_size; 398 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_tail_addr, chan); 399 } 400 401 static size_t stmmac_get_rx_desc_size(struct stmmac_priv *priv) 402 { 403 if (priv->extend_desc) 404 return sizeof(struct dma_extended_desc); 405 else 406 return sizeof(struct dma_desc); 407 } 408 409 static struct dma_desc *stmmac_get_rx_desc(struct stmmac_priv *priv, 410 struct stmmac_rx_queue *rx_q, 411 unsigned int index) 412 { 413 if (priv->extend_desc) 414 return &rx_q->dma_erx[index].basic; 415 else 416 return &rx_q->dma_rx[index]; 417 } 418 419 static void stmmac_set_queue_rx_tail_ptr(struct stmmac_priv *priv, 420 struct stmmac_rx_queue *rx_q, 421 unsigned int chan, unsigned int index) 422 { 423 /* This only needs to deal with normal descriptors as enhanced 424 * descriptiors are only supported with dwmac1000 (<v4.0) which 425 * does not implement .set_rx_tail_ptr 426 */ 427 u32 rx_tail_addr = rx_q->dma_rx_phy + index * sizeof(struct dma_desc); 428 429 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_tail_addr, chan); 430 } 431 432 static void stmmac_set_queue_rx_buf_size(struct stmmac_priv *priv, 433 struct stmmac_rx_queue *rx_q, 434 unsigned int chan) 435 { 436 u32 buf_size; 437 438 if (rx_q->xsk_pool && rx_q->buf_alloc_num) 439 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 440 else 441 buf_size = priv->dma_conf.dma_buf_sz; 442 443 stmmac_set_dma_bfsize(priv, priv->ioaddr, buf_size, chan); 444 } 445 446 /** 447 * stmmac_rx_dirty - Get RX queue dirty 448 * @priv: driver private structure 449 * @queue: RX queue index 450 */ 451 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 452 { 453 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 454 455 return CIRC_CNT(rx_q->cur_rx, rx_q->dirty_rx, 456 priv->dma_conf.dma_rx_size); 457 } 458 459 static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) 460 { 461 u8 tx_cnt = priv->plat->tx_queues_to_use; 462 u8 queue; 463 464 /* check if all TX queues have the work finished */ 465 for (queue = 0; queue < tx_cnt; queue++) { 466 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 467 468 if (tx_q->dirty_tx != tx_q->cur_tx) 469 return true; /* still unfinished work */ 470 } 471 472 return false; 473 } 474 475 static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv) 476 { 477 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 478 } 479 480 /** 481 * stmmac_try_to_start_sw_lpi - check and enter in LPI mode 482 * @priv: driver private structure 483 * Description: this function is to verify and enter in LPI mode in case of 484 * EEE. 485 */ 486 static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv) 487 { 488 if (stmmac_eee_tx_busy(priv)) { 489 stmmac_restart_sw_lpi_timer(priv); 490 return; 491 } 492 493 /* Check and enter in LPI mode */ 494 if (!priv->tx_path_in_lpi_mode) 495 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED, 496 priv->tx_lpi_clk_stop, 0); 497 } 498 499 /** 500 * stmmac_stop_sw_lpi - stop transmitting LPI 501 * @priv: driver private structure 502 * Description: When using software-controlled LPI, stop transmitting LPI state. 503 */ 504 static void stmmac_stop_sw_lpi(struct stmmac_priv *priv) 505 { 506 timer_delete_sync(&priv->eee_ctrl_timer); 507 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 508 priv->tx_path_in_lpi_mode = false; 509 } 510 511 /** 512 * stmmac_eee_ctrl_timer - EEE TX SW timer. 513 * @t: timer_list struct containing private info 514 * Description: 515 * if there is no data transfer and if we are not in LPI state, 516 * then MAC Transmitter can be moved to LPI state. 517 */ 518 static void stmmac_eee_ctrl_timer(struct timer_list *t) 519 { 520 struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer); 521 522 stmmac_try_to_start_sw_lpi(priv); 523 } 524 525 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 526 * @priv: driver private structure 527 * @p : descriptor pointer 528 * @skb : the socket buffer 529 * Description : 530 * This function will read timestamp from the descriptor & pass it to stack. 531 * and also perform some sanity checks. 532 */ 533 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 534 struct dma_desc *p, struct sk_buff *skb) 535 { 536 struct skb_shared_hwtstamps shhwtstamp; 537 bool found = false; 538 u64 ns = 0; 539 540 if (!priv->hwts_tx_en) 541 return; 542 543 /* exit if skb doesn't support hw tstamp */ 544 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 545 return; 546 547 /* check tx tstamp status */ 548 if (stmmac_get_tx_timestamp_status(priv, p)) { 549 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 550 found = true; 551 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 552 found = true; 553 } 554 555 if (found) { 556 ns -= priv->plat->cdc_error_adj; 557 558 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 559 shhwtstamp.hwtstamp = ns_to_ktime(ns); 560 561 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 562 /* pass tstamp to stack */ 563 skb_tstamp_tx(skb, &shhwtstamp); 564 } 565 } 566 567 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 568 * @priv: driver private structure 569 * @p : descriptor pointer 570 * @np : next descriptor pointer 571 * @skb : the socket buffer 572 * Description : 573 * This function will read received packet's timestamp from the descriptor 574 * and pass it to stack. It also perform some sanity checks. 575 */ 576 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 577 struct dma_desc *np, struct sk_buff *skb) 578 { 579 struct skb_shared_hwtstamps *shhwtstamp = NULL; 580 struct dma_desc *desc = p; 581 u64 ns = 0; 582 583 if (!priv->hwts_rx_en) 584 return; 585 /* For GMAC4, the valid timestamp is from CTX next desc. */ 586 if (dwmac_is_xmac(priv->plat->core_type)) 587 desc = np; 588 589 /* Check if timestamp is available */ 590 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 591 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 592 593 ns -= priv->plat->cdc_error_adj; 594 595 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 596 shhwtstamp = skb_hwtstamps(skb); 597 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 598 shhwtstamp->hwtstamp = ns_to_ktime(ns); 599 } else { 600 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 601 } 602 } 603 604 static void stmmac_update_subsecond_increment(struct stmmac_priv *priv) 605 { 606 bool xmac = dwmac_is_xmac(priv->plat->core_type); 607 u32 sec_inc = 0; 608 u64 temp = 0; 609 610 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 611 612 /* program Sub Second Increment reg */ 613 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 614 priv->plat->clk_ptp_rate, 615 xmac, &sec_inc); 616 temp = div_u64(1000000000ULL, sec_inc); 617 618 /* Store sub second increment for later use */ 619 priv->sub_second_inc = sec_inc; 620 621 /* calculate default added value: 622 * formula is : 623 * addend = (2^32)/freq_div_ratio; 624 * where, freq_div_ratio = 1e9ns/sec_inc 625 */ 626 temp = (u64)(temp << 32); 627 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 628 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 629 } 630 631 /** 632 * stmmac_hwtstamp_set - control hardware timestamping. 633 * @dev: device pointer. 634 * @config: the timestamping configuration. 635 * @extack: netlink extended ack structure for error reporting. 636 * Description: 637 * This function configures the MAC to enable/disable both outgoing(TX) 638 * and incoming(RX) packets time stamping based on user input. 639 * Return Value: 640 * 0 on success and an appropriate -ve integer on failure. 641 */ 642 static int stmmac_hwtstamp_set(struct net_device *dev, 643 struct kernel_hwtstamp_config *config, 644 struct netlink_ext_ack *extack) 645 { 646 struct stmmac_priv *priv = netdev_priv(dev); 647 u32 ptp_v2 = 0; 648 u32 tstamp_all = 0; 649 u32 ptp_over_ipv4_udp = 0; 650 u32 ptp_over_ipv6_udp = 0; 651 u32 ptp_over_ethernet = 0; 652 u32 snap_type_sel = 0; 653 u32 ts_master_en = 0; 654 u32 ts_event_en = 0; 655 656 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 657 NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping"); 658 priv->hwts_tx_en = 0; 659 priv->hwts_rx_en = 0; 660 661 return -EOPNOTSUPP; 662 } 663 664 if (!netif_running(dev)) { 665 NL_SET_ERR_MSG_MOD(extack, 666 "Cannot change timestamping configuration while down"); 667 return -ENODEV; 668 } 669 670 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 671 __func__, config->flags, config->tx_type, config->rx_filter); 672 673 if (config->tx_type != HWTSTAMP_TX_OFF && 674 config->tx_type != HWTSTAMP_TX_ON) 675 return -ERANGE; 676 677 if (priv->adv_ts) { 678 switch (config->rx_filter) { 679 case HWTSTAMP_FILTER_NONE: 680 /* time stamp no incoming packet at all */ 681 config->rx_filter = HWTSTAMP_FILTER_NONE; 682 break; 683 684 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 685 /* PTP v1, UDP, any kind of event packet */ 686 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 687 /* 'xmac' hardware can support Sync, Pdelay_Req and 688 * Pdelay_resp by setting bit14 and bits17/16 to 01 689 * This leaves Delay_Req timestamps out. 690 * Enable all events *and* general purpose message 691 * timestamping 692 */ 693 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 694 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 695 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 696 break; 697 698 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 699 /* PTP v1, UDP, Sync packet */ 700 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 701 /* take time stamp for SYNC messages only */ 702 ts_event_en = PTP_TCR_TSEVNTENA; 703 704 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 705 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 706 break; 707 708 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 709 /* PTP v1, UDP, Delay_req packet */ 710 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 711 /* take time stamp for Delay_Req messages only */ 712 ts_master_en = PTP_TCR_TSMSTRENA; 713 ts_event_en = PTP_TCR_TSEVNTENA; 714 715 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 716 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 717 break; 718 719 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 720 /* PTP v2, UDP, any kind of event packet */ 721 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 722 ptp_v2 = PTP_TCR_TSVER2ENA; 723 /* take time stamp for all event messages */ 724 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 725 726 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 727 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 728 break; 729 730 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 731 /* PTP v2, UDP, Sync packet */ 732 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 733 ptp_v2 = PTP_TCR_TSVER2ENA; 734 /* take time stamp for SYNC messages only */ 735 ts_event_en = PTP_TCR_TSEVNTENA; 736 737 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 738 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 739 break; 740 741 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 742 /* PTP v2, UDP, Delay_req packet */ 743 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 744 ptp_v2 = PTP_TCR_TSVER2ENA; 745 /* take time stamp for Delay_Req messages only */ 746 ts_master_en = PTP_TCR_TSMSTRENA; 747 ts_event_en = PTP_TCR_TSEVNTENA; 748 749 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 750 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 751 break; 752 753 case HWTSTAMP_FILTER_PTP_V2_EVENT: 754 /* PTP v2/802.AS1 any layer, any kind of event packet */ 755 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 756 ptp_v2 = PTP_TCR_TSVER2ENA; 757 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 758 if (priv->synopsys_id < DWMAC_CORE_4_10) 759 ts_event_en = PTP_TCR_TSEVNTENA; 760 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 761 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 762 ptp_over_ethernet = PTP_TCR_TSIPENA; 763 break; 764 765 case HWTSTAMP_FILTER_PTP_V2_SYNC: 766 /* PTP v2/802.AS1, any layer, Sync packet */ 767 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 768 ptp_v2 = PTP_TCR_TSVER2ENA; 769 /* take time stamp for SYNC messages only */ 770 ts_event_en = PTP_TCR_TSEVNTENA; 771 772 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 773 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 774 ptp_over_ethernet = PTP_TCR_TSIPENA; 775 break; 776 777 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 778 /* PTP v2/802.AS1, any layer, Delay_req packet */ 779 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 780 ptp_v2 = PTP_TCR_TSVER2ENA; 781 /* take time stamp for Delay_Req messages only */ 782 ts_master_en = PTP_TCR_TSMSTRENA; 783 ts_event_en = PTP_TCR_TSEVNTENA; 784 785 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 786 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 787 ptp_over_ethernet = PTP_TCR_TSIPENA; 788 break; 789 790 case HWTSTAMP_FILTER_NTP_ALL: 791 case HWTSTAMP_FILTER_ALL: 792 /* time stamp any incoming packet */ 793 config->rx_filter = HWTSTAMP_FILTER_ALL; 794 tstamp_all = PTP_TCR_TSENALL; 795 break; 796 797 default: 798 return -ERANGE; 799 } 800 } else { 801 switch (config->rx_filter) { 802 case HWTSTAMP_FILTER_NONE: 803 config->rx_filter = HWTSTAMP_FILTER_NONE; 804 break; 805 default: 806 /* PTP v1, UDP, any kind of event packet */ 807 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 808 break; 809 } 810 } 811 priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE; 812 priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON; 813 814 priv->systime_flags = STMMAC_HWTS_ACTIVE; 815 if (!priv->tsfupdt_coarse) 816 priv->systime_flags |= PTP_TCR_TSCFUPDT; 817 818 if (priv->hwts_tx_en || priv->hwts_rx_en) { 819 priv->systime_flags |= tstamp_all | ptp_v2 | 820 ptp_over_ethernet | ptp_over_ipv6_udp | 821 ptp_over_ipv4_udp | ts_event_en | 822 ts_master_en | snap_type_sel; 823 } 824 825 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 826 827 priv->tstamp_config = *config; 828 829 return 0; 830 } 831 832 /** 833 * stmmac_hwtstamp_get - read hardware timestamping. 834 * @dev: device pointer. 835 * @config: the timestamping configuration. 836 * Description: 837 * This function obtain the current hardware timestamping settings 838 * as requested. 839 */ 840 static int stmmac_hwtstamp_get(struct net_device *dev, 841 struct kernel_hwtstamp_config *config) 842 { 843 struct stmmac_priv *priv = netdev_priv(dev); 844 845 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 846 return -EOPNOTSUPP; 847 848 *config = priv->tstamp_config; 849 850 return 0; 851 } 852 853 /** 854 * stmmac_init_tstamp_counter - init hardware timestamping counter 855 * @priv: driver private structure 856 * @systime_flags: timestamping flags 857 * Description: 858 * Initialize hardware counter for packet timestamping. 859 * This is valid as long as the interface is open and not suspended. 860 * Will be rerun after resuming from suspend, case in which the timestamping 861 * flags updated by stmmac_hwtstamp_set() also need to be restored. 862 */ 863 static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, 864 u32 systime_flags) 865 { 866 struct timespec64 now; 867 868 if (!priv->plat->clk_ptp_rate) { 869 netdev_err(priv->dev, "Invalid PTP clock rate"); 870 return -EINVAL; 871 } 872 873 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 874 priv->systime_flags = systime_flags; 875 876 stmmac_update_subsecond_increment(priv); 877 878 /* initialize system time */ 879 ktime_get_real_ts64(&now); 880 881 /* lower 32 bits of tv_sec are safe until y2106 */ 882 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 883 884 return 0; 885 } 886 887 /** 888 * stmmac_init_timestamping - initialise timestamping 889 * @priv: driver private structure 890 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 891 * This is done by looking at the HW cap. register. 892 * This function also registers the ptp driver. 893 */ 894 static int stmmac_init_timestamping(struct stmmac_priv *priv) 895 { 896 bool xmac = dwmac_is_xmac(priv->plat->core_type); 897 int ret; 898 899 if (priv->plat->ptp_clk_freq_config) 900 priv->plat->ptp_clk_freq_config(priv); 901 902 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) { 903 netdev_info(priv->dev, "PTP not supported by HW\n"); 904 return -EOPNOTSUPP; 905 } 906 907 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE | 908 PTP_TCR_TSCFUPDT); 909 if (ret) { 910 netdev_warn(priv->dev, "PTP init failed\n"); 911 return ret; 912 } 913 914 priv->adv_ts = 0; 915 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 916 if (xmac && priv->dma_cap.atime_stamp) 917 priv->adv_ts = 1; 918 /* Dwmac 3.x core with extend_desc can support adv_ts */ 919 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 920 priv->adv_ts = 1; 921 922 if (priv->dma_cap.time_stamp) 923 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 924 925 if (priv->adv_ts) 926 netdev_info(priv->dev, 927 "IEEE 1588-2008 Advanced Timestamp supported\n"); 928 929 memset(&priv->tstamp_config, 0, sizeof(priv->tstamp_config)); 930 priv->hwts_tx_en = 0; 931 priv->hwts_rx_en = 0; 932 933 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 934 stmmac_hwtstamp_correct_latency(priv, priv); 935 936 return 0; 937 } 938 939 static void stmmac_setup_ptp(struct stmmac_priv *priv) 940 { 941 int ret; 942 943 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 944 if (ret < 0) 945 netdev_warn(priv->dev, 946 "failed to enable PTP reference clock: %pe\n", 947 ERR_PTR(ret)); 948 949 if (stmmac_init_timestamping(priv) == 0) 950 stmmac_ptp_register(priv); 951 } 952 953 static void stmmac_release_ptp(struct stmmac_priv *priv) 954 { 955 stmmac_ptp_unregister(priv); 956 clk_disable_unprepare(priv->plat->clk_ptp_ref); 957 } 958 959 static void stmmac_legacy_serdes_power_down(struct stmmac_priv *priv) 960 { 961 if (priv->plat->serdes_powerdown && priv->legacy_serdes_is_powered) 962 priv->plat->serdes_powerdown(priv->dev, priv->plat->bsp_priv); 963 964 priv->legacy_serdes_is_powered = false; 965 } 966 967 static int stmmac_legacy_serdes_power_up(struct stmmac_priv *priv) 968 { 969 int ret; 970 971 if (!priv->plat->serdes_powerup) 972 return 0; 973 974 ret = priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 975 if (ret < 0) 976 netdev_err(priv->dev, "SerDes powerup failed\n"); 977 else 978 priv->legacy_serdes_is_powered = true; 979 980 return ret; 981 } 982 983 /** 984 * stmmac_mac_flow_ctrl - Configure flow control in all queues 985 * @priv: driver private structure 986 * @duplex: duplex passed to the next function 987 * @flow_ctrl: desired flow control modes 988 * Description: It is used for configuring the flow control in all queues 989 */ 990 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex, 991 unsigned int flow_ctrl) 992 { 993 u8 tx_cnt = priv->plat->tx_queues_to_use; 994 995 stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time, 996 tx_cnt); 997 } 998 999 static unsigned long stmmac_mac_get_caps(struct phylink_config *config, 1000 phy_interface_t interface) 1001 { 1002 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1003 1004 /* Refresh the MAC-specific capabilities */ 1005 stmmac_mac_update_caps(priv); 1006 1007 if (priv->hw_cap_support && !priv->dma_cap.half_duplex) 1008 priv->hw->link.caps &= ~(MAC_1000HD | MAC_100HD | MAC_10HD); 1009 1010 config->mac_capabilities = priv->hw->link.caps; 1011 1012 if (priv->plat->max_speed) 1013 phylink_limit_mac_speed(config, priv->plat->max_speed); 1014 1015 return config->mac_capabilities; 1016 } 1017 1018 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 1019 phy_interface_t interface) 1020 { 1021 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1022 struct phylink_pcs *pcs; 1023 1024 if (priv->plat->select_pcs) { 1025 pcs = priv->plat->select_pcs(priv, interface); 1026 if (!IS_ERR(pcs)) 1027 return pcs; 1028 } 1029 1030 if (priv->integrated_pcs && 1031 test_bit(interface, priv->integrated_pcs->pcs.supported_interfaces)) 1032 return &priv->integrated_pcs->pcs; 1033 1034 return NULL; 1035 } 1036 1037 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 1038 const struct phylink_link_state *state) 1039 { 1040 /* Nothing to do, xpcs_config() handles everything */ 1041 } 1042 1043 static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode, 1044 phy_interface_t interface) 1045 { 1046 struct net_device *ndev = to_net_dev(config->dev); 1047 struct stmmac_priv *priv = netdev_priv(ndev); 1048 1049 if (priv->plat->mac_finish) 1050 priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, 1051 interface); 1052 1053 return 0; 1054 } 1055 1056 static void stmmac_mac_link_down(struct phylink_config *config, 1057 unsigned int mode, phy_interface_t interface) 1058 { 1059 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1060 1061 stmmac_mac_set(priv, priv->ioaddr, false); 1062 if (priv->dma_cap.eee) 1063 stmmac_set_eee_pls(priv, priv->hw, false); 1064 1065 if (stmmac_fpe_supported(priv)) 1066 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, false); 1067 } 1068 1069 static void stmmac_mac_link_up(struct phylink_config *config, 1070 struct phy_device *phy, 1071 unsigned int mode, phy_interface_t interface, 1072 int speed, int duplex, 1073 bool tx_pause, bool rx_pause) 1074 { 1075 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1076 unsigned int flow_ctrl; 1077 u32 old_ctrl, ctrl; 1078 int ret; 1079 1080 if (priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) 1081 stmmac_legacy_serdes_power_up(priv); 1082 1083 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1084 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1085 1086 if (interface == PHY_INTERFACE_MODE_USXGMII) { 1087 switch (speed) { 1088 case SPEED_10000: 1089 ctrl |= priv->hw->link.xgmii.speed10000; 1090 break; 1091 case SPEED_5000: 1092 ctrl |= priv->hw->link.xgmii.speed5000; 1093 break; 1094 case SPEED_2500: 1095 ctrl |= priv->hw->link.xgmii.speed2500; 1096 break; 1097 default: 1098 return; 1099 } 1100 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1101 switch (speed) { 1102 case SPEED_100000: 1103 ctrl |= priv->hw->link.xlgmii.speed100000; 1104 break; 1105 case SPEED_50000: 1106 ctrl |= priv->hw->link.xlgmii.speed50000; 1107 break; 1108 case SPEED_40000: 1109 ctrl |= priv->hw->link.xlgmii.speed40000; 1110 break; 1111 case SPEED_25000: 1112 ctrl |= priv->hw->link.xlgmii.speed25000; 1113 break; 1114 case SPEED_10000: 1115 ctrl |= priv->hw->link.xgmii.speed10000; 1116 break; 1117 case SPEED_2500: 1118 ctrl |= priv->hw->link.speed2500; 1119 break; 1120 case SPEED_1000: 1121 ctrl |= priv->hw->link.speed1000; 1122 break; 1123 default: 1124 return; 1125 } 1126 } else { 1127 switch (speed) { 1128 case SPEED_2500: 1129 ctrl |= priv->hw->link.speed2500; 1130 break; 1131 case SPEED_1000: 1132 ctrl |= priv->hw->link.speed1000; 1133 break; 1134 case SPEED_100: 1135 ctrl |= priv->hw->link.speed100; 1136 break; 1137 case SPEED_10: 1138 ctrl |= priv->hw->link.speed10; 1139 break; 1140 default: 1141 return; 1142 } 1143 } 1144 1145 if (priv->plat->fix_mac_speed) 1146 priv->plat->fix_mac_speed(priv->plat->bsp_priv, interface, 1147 speed, mode); 1148 1149 if (!duplex) 1150 ctrl &= ~priv->hw->link.duplex; 1151 else 1152 ctrl |= priv->hw->link.duplex; 1153 1154 /* Flow Control operation */ 1155 if (rx_pause && tx_pause) 1156 flow_ctrl = FLOW_AUTO; 1157 else if (rx_pause && !tx_pause) 1158 flow_ctrl = FLOW_RX; 1159 else if (!rx_pause && tx_pause) 1160 flow_ctrl = FLOW_TX; 1161 else 1162 flow_ctrl = FLOW_OFF; 1163 1164 stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl); 1165 1166 if (ctrl != old_ctrl) 1167 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1168 1169 if (priv->plat->set_clk_tx_rate) { 1170 ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv, 1171 priv->plat->clk_tx_i, 1172 interface, speed); 1173 if (ret < 0) 1174 netdev_err(priv->dev, 1175 "failed to configure %s transmit clock for %dMbps: %pe\n", 1176 phy_modes(interface), speed, ERR_PTR(ret)); 1177 } 1178 1179 stmmac_mac_set(priv, priv->ioaddr, true); 1180 if (priv->dma_cap.eee) 1181 stmmac_set_eee_pls(priv, priv->hw, true); 1182 1183 if (stmmac_fpe_supported(priv)) 1184 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, true); 1185 1186 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1187 stmmac_hwtstamp_correct_latency(priv, priv); 1188 } 1189 1190 static void stmmac_mac_disable_tx_lpi(struct phylink_config *config) 1191 { 1192 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1193 1194 priv->eee_active = false; 1195 1196 mutex_lock(&priv->lock); 1197 1198 priv->eee_enabled = false; 1199 1200 netdev_dbg(priv->dev, "disable EEE\n"); 1201 priv->eee_sw_timer_en = false; 1202 timer_delete_sync(&priv->eee_ctrl_timer); 1203 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 1204 priv->tx_path_in_lpi_mode = false; 1205 1206 stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS); 1207 mutex_unlock(&priv->lock); 1208 } 1209 1210 static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 1211 bool tx_clk_stop) 1212 { 1213 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1214 int ret; 1215 1216 priv->tx_lpi_timer = timer; 1217 priv->eee_active = true; 1218 1219 mutex_lock(&priv->lock); 1220 1221 priv->eee_enabled = true; 1222 1223 /* Update the transmit clock stop according to PHY capability if 1224 * the platform allows 1225 */ 1226 if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP) 1227 priv->tx_lpi_clk_stop = tx_clk_stop; 1228 1229 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 1230 STMMAC_DEFAULT_TWT_LS); 1231 1232 /* Try to configure the hardware timer. */ 1233 ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER, 1234 priv->tx_lpi_clk_stop, priv->tx_lpi_timer); 1235 1236 if (ret) { 1237 /* Hardware timer mode not supported, or value out of range. 1238 * Fall back to using software LPI mode 1239 */ 1240 priv->eee_sw_timer_en = true; 1241 stmmac_restart_sw_lpi_timer(priv); 1242 } 1243 1244 mutex_unlock(&priv->lock); 1245 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 1246 1247 return 0; 1248 } 1249 1250 static int stmmac_mac_wol_set(struct phylink_config *config, u32 wolopts, 1251 const u8 *sopass) 1252 { 1253 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1254 1255 device_set_wakeup_enable(priv->device, !!wolopts); 1256 1257 mutex_lock(&priv->lock); 1258 priv->wolopts = wolopts; 1259 mutex_unlock(&priv->lock); 1260 1261 return 0; 1262 } 1263 1264 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1265 .mac_get_caps = stmmac_mac_get_caps, 1266 .mac_select_pcs = stmmac_mac_select_pcs, 1267 .mac_config = stmmac_mac_config, 1268 .mac_finish = stmmac_mac_finish, 1269 .mac_link_down = stmmac_mac_link_down, 1270 .mac_link_up = stmmac_mac_link_up, 1271 .mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi, 1272 .mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi, 1273 .mac_wol_set = stmmac_mac_wol_set, 1274 }; 1275 1276 /** 1277 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1278 * @priv: driver private structure 1279 * Description: this is to verify if the HW supports the PCS. 1280 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1281 * configured for the TBI, RTBI, or SGMII PHY interface. 1282 */ 1283 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1284 { 1285 int interface = priv->plat->phy_interface; 1286 int speed = priv->plat->mac_port_sel_speed; 1287 1288 if (priv->dma_cap.pcs && interface == PHY_INTERFACE_MODE_SGMII) { 1289 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1290 1291 switch (speed) { 1292 case SPEED_10: 1293 case SPEED_100: 1294 case SPEED_1000: 1295 priv->hw->reverse_sgmii_enable = true; 1296 break; 1297 1298 default: 1299 dev_warn(priv->device, "invalid port speed\n"); 1300 fallthrough; 1301 case 0: 1302 priv->hw->reverse_sgmii_enable = false; 1303 break; 1304 } 1305 } 1306 } 1307 1308 /** 1309 * stmmac_init_phy - PHY initialization 1310 * @dev: net device structure 1311 * Description: it initializes the driver's PHY state, and attaches the PHY 1312 * to the mac driver. 1313 * Return value: 1314 * 0 on success 1315 */ 1316 static int stmmac_init_phy(struct net_device *dev) 1317 { 1318 struct stmmac_priv *priv = netdev_priv(dev); 1319 int mode = priv->plat->phy_interface; 1320 struct fwnode_handle *phy_fwnode; 1321 struct fwnode_handle *fwnode; 1322 struct ethtool_keee eee; 1323 u32 dev_flags = 0; 1324 int ret; 1325 1326 if (!phylink_expects_phy(priv->phylink)) 1327 return 0; 1328 1329 if (priv->hw->xpcs && 1330 xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73) 1331 return 0; 1332 1333 fwnode = dev_fwnode(priv->device); 1334 if (fwnode) 1335 phy_fwnode = fwnode_get_phy_node(fwnode); 1336 else 1337 phy_fwnode = NULL; 1338 1339 if (priv->plat->flags & STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD) 1340 dev_flags |= PHY_F_KEEP_PREAMBLE_BEFORE_SFD; 1341 1342 /* Some DT bindings do not set-up the PHY handle. Let's try to 1343 * manually parse it 1344 */ 1345 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1346 int addr = priv->plat->phy_addr; 1347 struct phy_device *phydev; 1348 1349 if (addr < 0) { 1350 netdev_err(priv->dev, "no phy found\n"); 1351 return -ENODEV; 1352 } 1353 1354 phydev = mdiobus_get_phy(priv->mii, addr); 1355 if (!phydev) { 1356 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1357 return -ENODEV; 1358 } 1359 1360 phydev->dev_flags |= dev_flags; 1361 1362 ret = phylink_connect_phy(priv->phylink, phydev); 1363 } else { 1364 fwnode_handle_put(phy_fwnode); 1365 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, dev_flags); 1366 } 1367 1368 if (ret) { 1369 netdev_err(priv->dev, "cannot attach to PHY (error: %pe)\n", 1370 ERR_PTR(ret)); 1371 return ret; 1372 } 1373 1374 /* Configure phylib's copy of the LPI timer. Normally, 1375 * phylink_config.lpi_timer_default would do this, but there is a 1376 * chance that userspace could change the eee_timer setting via sysfs 1377 * before the first open. Thus, preserve existing behaviour. 1378 */ 1379 if (!phylink_ethtool_get_eee(priv->phylink, &eee)) { 1380 eee.tx_lpi_timer = priv->tx_lpi_timer; 1381 phylink_ethtool_set_eee(priv->phylink, &eee); 1382 } 1383 1384 return 0; 1385 } 1386 1387 static int stmmac_phylink_setup(struct stmmac_priv *priv) 1388 { 1389 struct phylink_config *config; 1390 struct phylink_pcs *pcs; 1391 struct phylink *phylink; 1392 1393 config = &priv->phylink_config; 1394 1395 config->dev = &priv->dev->dev; 1396 config->type = PHYLINK_NETDEV; 1397 config->mac_managed_pm = true; 1398 1399 /* Stmmac always requires an RX clock for hardware initialization */ 1400 config->mac_requires_rxc = true; 1401 1402 /* Disable EEE RX clock stop to ensure VLAN register access works 1403 * correctly. 1404 */ 1405 if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI) && 1406 !(priv->dev->features & NETIF_F_VLAN_FEATURES)) 1407 config->eee_rx_clk_stop_enable = true; 1408 1409 /* Set the default transmit clock stop bit based on the platform glue */ 1410 priv->tx_lpi_clk_stop = priv->plat->flags & 1411 STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; 1412 1413 /* Get the PHY interface modes (at the PHY end of the link) that 1414 * are supported by the platform. 1415 */ 1416 if (priv->plat->get_interfaces) 1417 priv->plat->get_interfaces(priv, priv->plat->bsp_priv, 1418 config->supported_interfaces); 1419 1420 config->default_an_inband = priv->plat->default_an_inband; 1421 1422 /* Set the platform/firmware specified interface mode if the 1423 * supported interfaces have not already been provided using 1424 * phy_interface as a last resort. 1425 */ 1426 if (phy_interface_empty(config->supported_interfaces)) 1427 __set_bit(priv->plat->phy_interface, 1428 config->supported_interfaces); 1429 1430 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1431 if (priv->hw->xpcs) 1432 pcs = xpcs_to_phylink_pcs(priv->hw->xpcs); 1433 else 1434 pcs = priv->hw->phylink_pcs; 1435 1436 if (pcs) 1437 phy_interface_or(config->supported_interfaces, 1438 config->supported_interfaces, 1439 pcs->supported_interfaces); 1440 1441 /* Some platforms, e.g. iMX8MP, wire lpi_intr_o to the same interrupt 1442 * used for stmmac's main interrupts, which leads to interrupt storms. 1443 * STMMAC_FLAG_EEE_DISABLE allows EEE to be disabled on such platforms. 1444 */ 1445 if (priv->dma_cap.eee && 1446 !(priv->plat->flags & STMMAC_FLAG_EEE_DISABLE)) { 1447 /* The GMAC 3.74a databook states that EEE is only supported 1448 * in MII, GMII, and RGMII interfaces. 1449 */ 1450 __set_bit(PHY_INTERFACE_MODE_MII, config->lpi_interfaces); 1451 __set_bit(PHY_INTERFACE_MODE_GMII, config->lpi_interfaces); 1452 phy_interface_set_rgmii(config->lpi_interfaces); 1453 1454 /* If we have a non-integrated PCS, assume that it is connected 1455 * to the GMAC using GMII or another EEE compatible interface, 1456 * and thus all PCS-supported interfaces support LPI. 1457 */ 1458 if (pcs) 1459 phy_interface_or(config->lpi_interfaces, 1460 config->lpi_interfaces, 1461 pcs->supported_interfaces); 1462 1463 /* All full duplex speeds above 100Mbps are supported */ 1464 config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD; 1465 config->lpi_timer_default = eee_timer * 1000; 1466 config->eee_enabled_default = true; 1467 } 1468 1469 config->wol_phy_speed_ctrl = true; 1470 if (priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL) { 1471 config->wol_phy_legacy = true; 1472 } else { 1473 if (priv->dma_cap.pmt_remote_wake_up) 1474 config->wol_mac_support |= WAKE_UCAST; 1475 if (priv->dma_cap.pmt_magic_frame) 1476 config->wol_mac_support |= WAKE_MAGIC; 1477 } 1478 1479 phylink = phylink_create(config, dev_fwnode(priv->device), 1480 priv->plat->phy_interface, 1481 &stmmac_phylink_mac_ops); 1482 if (IS_ERR(phylink)) 1483 return PTR_ERR(phylink); 1484 1485 priv->phylink = phylink; 1486 return 0; 1487 } 1488 1489 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1490 struct stmmac_dma_conf *dma_conf) 1491 { 1492 u8 rx_cnt = priv->plat->rx_queues_to_use; 1493 unsigned int desc_size; 1494 void *head_rx; 1495 u8 queue; 1496 1497 /* Display RX rings */ 1498 for (queue = 0; queue < rx_cnt; queue++) { 1499 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1500 1501 pr_info("\tRX Queue %u rings\n", queue); 1502 1503 head_rx = stmmac_get_rx_desc(priv, rx_q, 0); 1504 desc_size = stmmac_get_rx_desc_size(priv); 1505 1506 /* Display RX ring */ 1507 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1508 rx_q->dma_rx_phy, desc_size); 1509 } 1510 } 1511 1512 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1513 struct stmmac_dma_conf *dma_conf) 1514 { 1515 u8 tx_cnt = priv->plat->tx_queues_to_use; 1516 unsigned int desc_size; 1517 void *head_tx; 1518 u8 queue; 1519 1520 /* Display TX rings */ 1521 for (queue = 0; queue < tx_cnt; queue++) { 1522 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1523 1524 pr_info("\tTX Queue %d rings\n", queue); 1525 1526 head_tx = stmmac_get_tx_desc(priv, tx_q, 0); 1527 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 1528 1529 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1530 tx_q->dma_tx_phy, desc_size); 1531 } 1532 } 1533 1534 static void stmmac_display_rings(struct stmmac_priv *priv, 1535 struct stmmac_dma_conf *dma_conf) 1536 { 1537 /* Display RX ring */ 1538 stmmac_display_rx_rings(priv, dma_conf); 1539 1540 /* Display TX ring */ 1541 stmmac_display_tx_rings(priv, dma_conf); 1542 } 1543 1544 static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) 1545 { 1546 if (stmmac_xdp_is_enabled(priv)) 1547 return XDP_PACKET_HEADROOM; 1548 1549 return NET_SKB_PAD; 1550 } 1551 1552 static int stmmac_set_bfsize(int mtu) 1553 { 1554 int ret; 1555 1556 if (mtu >= BUF_SIZE_8KiB) 1557 ret = BUF_SIZE_16KiB; 1558 else if (mtu >= BUF_SIZE_4KiB) 1559 ret = BUF_SIZE_8KiB; 1560 else if (mtu >= BUF_SIZE_2KiB) 1561 ret = BUF_SIZE_4KiB; 1562 else if (mtu > DEFAULT_BUFSIZE) 1563 ret = BUF_SIZE_2KiB; 1564 else 1565 ret = DEFAULT_BUFSIZE; 1566 1567 return ret; 1568 } 1569 1570 /** 1571 * stmmac_clear_rx_descriptors - clear RX descriptors 1572 * @priv: driver private structure 1573 * @dma_conf: structure to take the dma data 1574 * @queue: RX queue index 1575 * Description: this function is called to clear the RX descriptors 1576 * in case of both basic and extended descriptors are used. 1577 */ 1578 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1579 struct stmmac_dma_conf *dma_conf, 1580 u32 queue) 1581 { 1582 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1583 struct dma_desc *desc; 1584 int i; 1585 1586 /* Clear the RX descriptors */ 1587 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1588 desc = stmmac_get_rx_desc(priv, rx_q, i); 1589 1590 stmmac_init_rx_desc(priv, desc, priv->use_riwt, 1591 priv->descriptor_mode, 1592 (i == dma_conf->dma_rx_size - 1), 1593 dma_conf->dma_buf_sz); 1594 } 1595 } 1596 1597 /** 1598 * stmmac_clear_tx_descriptors - clear tx descriptors 1599 * @priv: driver private structure 1600 * @dma_conf: structure to take the dma data 1601 * @queue: TX queue index. 1602 * Description: this function is called to clear the TX descriptors 1603 * in case of both basic and extended descriptors are used. 1604 */ 1605 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1606 struct stmmac_dma_conf *dma_conf, 1607 u32 queue) 1608 { 1609 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1610 int i; 1611 1612 /* Clear the TX descriptors */ 1613 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1614 int last = (i == (dma_conf->dma_tx_size - 1)); 1615 struct dma_desc *p; 1616 1617 p = stmmac_get_tx_desc(priv, tx_q, i); 1618 stmmac_init_tx_desc(priv, p, priv->descriptor_mode, last); 1619 } 1620 } 1621 1622 /** 1623 * stmmac_clear_descriptors - clear descriptors 1624 * @priv: driver private structure 1625 * @dma_conf: structure to take the dma data 1626 * Description: this function is called to clear the TX and RX descriptors 1627 * in case of both basic and extended descriptors are used. 1628 */ 1629 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1630 struct stmmac_dma_conf *dma_conf) 1631 { 1632 u8 rx_queue_cnt = priv->plat->rx_queues_to_use; 1633 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 1634 u8 queue; 1635 1636 /* Clear the RX descriptors */ 1637 for (queue = 0; queue < rx_queue_cnt; queue++) 1638 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1639 1640 /* Clear the TX descriptors */ 1641 for (queue = 0; queue < tx_queue_cnt; queue++) 1642 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1643 } 1644 1645 /** 1646 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1647 * @priv: driver private structure 1648 * @dma_conf: structure to take the dma data 1649 * @p: descriptor pointer 1650 * @i: descriptor index 1651 * @flags: gfp flag 1652 * @queue: RX queue index 1653 * Description: this function is called to allocate a receive buffer, perform 1654 * the DMA mapping and init the descriptor. 1655 */ 1656 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1657 struct stmmac_dma_conf *dma_conf, 1658 struct dma_desc *p, 1659 int i, gfp_t flags, u32 queue) 1660 { 1661 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1662 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1663 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1664 1665 if (priv->dma_cap.host_dma_width <= 32) 1666 gfp |= GFP_DMA32; 1667 1668 if (!buf->page) { 1669 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1670 if (!buf->page) 1671 return -ENOMEM; 1672 buf->page_offset = stmmac_rx_offset(priv); 1673 } 1674 1675 if (priv->sph_active && !buf->sec_page) { 1676 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1677 if (!buf->sec_page) 1678 return -ENOMEM; 1679 1680 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1681 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1682 } else { 1683 buf->sec_page = NULL; 1684 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1685 } 1686 1687 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1688 1689 stmmac_set_desc_addr(priv, p, buf->addr); 1690 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1691 stmmac_init_desc3(priv, p); 1692 1693 return 0; 1694 } 1695 1696 /** 1697 * stmmac_free_rx_buffer - free RX dma buffers 1698 * @priv: private structure 1699 * @rx_q: RX queue 1700 * @i: buffer index. 1701 */ 1702 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1703 struct stmmac_rx_queue *rx_q, 1704 int i) 1705 { 1706 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1707 1708 if (buf->page) 1709 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1710 buf->page = NULL; 1711 1712 if (buf->sec_page) 1713 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1714 buf->sec_page = NULL; 1715 } 1716 1717 /** 1718 * stmmac_free_tx_buffer - free RX dma buffers 1719 * @priv: private structure 1720 * @dma_conf: structure to take the dma data 1721 * @queue: RX queue index 1722 * @i: buffer index. 1723 */ 1724 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1725 struct stmmac_dma_conf *dma_conf, 1726 u32 queue, int i) 1727 { 1728 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1729 1730 if (tx_q->tx_skbuff_dma[i].buf && 1731 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1732 if (tx_q->tx_skbuff_dma[i].map_as_page) 1733 dma_unmap_page(priv->device, 1734 tx_q->tx_skbuff_dma[i].buf, 1735 tx_q->tx_skbuff_dma[i].len, 1736 DMA_TO_DEVICE); 1737 else 1738 dma_unmap_single(priv->device, 1739 tx_q->tx_skbuff_dma[i].buf, 1740 tx_q->tx_skbuff_dma[i].len, 1741 DMA_TO_DEVICE); 1742 } 1743 1744 if (tx_q->xdpf[i] && 1745 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1746 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1747 xdp_return_frame(tx_q->xdpf[i]); 1748 tx_q->xdpf[i] = NULL; 1749 } 1750 1751 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1752 tx_q->xsk_frames_done++; 1753 1754 if (tx_q->tx_skbuff[i] && 1755 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1756 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1757 tx_q->tx_skbuff[i] = NULL; 1758 } 1759 1760 tx_q->tx_skbuff_dma[i].buf = 0; 1761 tx_q->tx_skbuff_dma[i].map_as_page = false; 1762 } 1763 1764 /** 1765 * dma_free_rx_skbufs - free RX dma buffers 1766 * @priv: private structure 1767 * @dma_conf: structure to take the dma data 1768 * @queue: RX queue index 1769 */ 1770 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1771 struct stmmac_dma_conf *dma_conf, 1772 u32 queue) 1773 { 1774 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1775 int i; 1776 1777 for (i = 0; i < dma_conf->dma_rx_size; i++) 1778 stmmac_free_rx_buffer(priv, rx_q, i); 1779 } 1780 1781 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1782 struct stmmac_dma_conf *dma_conf, 1783 u32 queue, gfp_t flags) 1784 { 1785 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1786 int i; 1787 1788 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1789 struct dma_desc *p; 1790 int ret; 1791 1792 p = stmmac_get_rx_desc(priv, rx_q, i); 1793 1794 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1795 queue); 1796 if (ret) 1797 return ret; 1798 1799 rx_q->buf_alloc_num++; 1800 } 1801 1802 return 0; 1803 } 1804 1805 /** 1806 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1807 * @priv: private structure 1808 * @dma_conf: structure to take the dma data 1809 * @queue: RX queue index 1810 */ 1811 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1812 struct stmmac_dma_conf *dma_conf, 1813 u32 queue) 1814 { 1815 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1816 int i; 1817 1818 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1819 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1820 1821 if (!buf->xdp) 1822 continue; 1823 1824 xsk_buff_free(buf->xdp); 1825 buf->xdp = NULL; 1826 } 1827 } 1828 1829 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1830 struct stmmac_dma_conf *dma_conf, 1831 u32 queue) 1832 { 1833 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1834 int i; 1835 1836 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1837 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1838 * use this macro to make sure no size violations. 1839 */ 1840 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1841 1842 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1843 struct stmmac_rx_buffer *buf; 1844 dma_addr_t dma_addr; 1845 struct dma_desc *p; 1846 1847 p = stmmac_get_rx_desc(priv, rx_q, i); 1848 1849 buf = &rx_q->buf_pool[i]; 1850 1851 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1852 if (!buf->xdp) 1853 return -ENOMEM; 1854 1855 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1856 stmmac_set_desc_addr(priv, p, dma_addr); 1857 rx_q->buf_alloc_num++; 1858 } 1859 1860 return 0; 1861 } 1862 1863 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1864 { 1865 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1866 return NULL; 1867 1868 return xsk_get_pool_from_qid(priv->dev, queue); 1869 } 1870 1871 /** 1872 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1873 * @priv: driver private structure 1874 * @dma_conf: structure to take the dma data 1875 * @queue: RX queue index 1876 * @flags: gfp flag. 1877 * Description: this function initializes the DMA RX descriptors 1878 * and allocates the socket buffers. It supports the chained and ring 1879 * modes. 1880 */ 1881 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1882 struct stmmac_dma_conf *dma_conf, 1883 u32 queue, gfp_t flags) 1884 { 1885 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1886 void *des; 1887 int ret; 1888 1889 netif_dbg(priv, probe, priv->dev, 1890 "(%s) dma_rx_phy=0x%08x\n", __func__, 1891 (u32)rx_q->dma_rx_phy); 1892 1893 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1894 1895 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1896 1897 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1898 1899 if (rx_q->xsk_pool) { 1900 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1901 MEM_TYPE_XSK_BUFF_POOL, 1902 NULL)); 1903 netdev_info(priv->dev, 1904 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1905 queue); 1906 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1907 } else { 1908 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1909 MEM_TYPE_PAGE_POOL, 1910 rx_q->page_pool)); 1911 netdev_info(priv->dev, 1912 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1913 queue); 1914 } 1915 1916 if (rx_q->xsk_pool) { 1917 /* RX XDP ZC buffer pool may not be populated, e.g. 1918 * xdpsock TX-only. 1919 */ 1920 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1921 } else { 1922 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1923 if (ret < 0) 1924 return -ENOMEM; 1925 } 1926 1927 /* Setup the chained descriptor addresses */ 1928 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 1929 if (priv->extend_desc) 1930 des = rx_q->dma_erx; 1931 else 1932 des = rx_q->dma_rx; 1933 1934 stmmac_mode_init(priv, des, rx_q->dma_rx_phy, 1935 dma_conf->dma_rx_size, priv->extend_desc); 1936 } 1937 1938 return 0; 1939 } 1940 1941 static int init_dma_rx_desc_rings(struct net_device *dev, 1942 struct stmmac_dma_conf *dma_conf, 1943 gfp_t flags) 1944 { 1945 struct stmmac_priv *priv = netdev_priv(dev); 1946 u8 rx_count = priv->plat->rx_queues_to_use; 1947 int queue; 1948 int ret; 1949 1950 /* RX INITIALIZATION */ 1951 netif_dbg(priv, probe, priv->dev, 1952 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1953 1954 for (queue = 0; queue < rx_count; queue++) { 1955 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1956 if (ret) 1957 goto err_init_rx_buffers; 1958 } 1959 1960 return 0; 1961 1962 err_init_rx_buffers: 1963 while (queue >= 0) { 1964 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1965 1966 if (rx_q->xsk_pool) 1967 dma_free_rx_xskbufs(priv, dma_conf, queue); 1968 else 1969 dma_free_rx_skbufs(priv, dma_conf, queue); 1970 1971 rx_q->buf_alloc_num = 0; 1972 rx_q->xsk_pool = NULL; 1973 1974 queue--; 1975 } 1976 1977 return ret; 1978 } 1979 1980 static void stmmac_set_tx_dma_entry(struct stmmac_tx_queue *tx_q, 1981 unsigned int entry, 1982 enum stmmac_txbuf_type type, 1983 dma_addr_t addr, size_t len, 1984 bool map_as_page) 1985 { 1986 tx_q->tx_skbuff_dma[entry].buf = addr; 1987 tx_q->tx_skbuff_dma[entry].len = len; 1988 tx_q->tx_skbuff_dma[entry].buf_type = type; 1989 tx_q->tx_skbuff_dma[entry].map_as_page = map_as_page; 1990 tx_q->tx_skbuff_dma[entry].last_segment = false; 1991 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1992 } 1993 1994 static void stmmac_set_tx_skb_dma_entry(struct stmmac_tx_queue *tx_q, 1995 unsigned int entry, dma_addr_t addr, 1996 size_t len, bool map_as_page) 1997 { 1998 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_SKB, addr, len, 1999 map_as_page); 2000 } 2001 2002 static void stmmac_set_tx_dma_last_segment(struct stmmac_tx_queue *tx_q, 2003 unsigned int entry) 2004 { 2005 tx_q->tx_skbuff_dma[entry].last_segment = true; 2006 } 2007 2008 /** 2009 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 2010 * @priv: driver private structure 2011 * @dma_conf: structure to take the dma data 2012 * @queue: TX queue index 2013 * Description: this function initializes the DMA TX descriptors 2014 * and allocates the socket buffers. It supports the chained and ring 2015 * modes. 2016 */ 2017 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 2018 struct stmmac_dma_conf *dma_conf, 2019 u32 queue) 2020 { 2021 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2022 int i; 2023 2024 netif_dbg(priv, probe, priv->dev, 2025 "(%s) dma_tx_phy=0x%08x\n", __func__, 2026 (u32)tx_q->dma_tx_phy); 2027 2028 /* Setup the chained descriptor addresses */ 2029 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 2030 if (priv->extend_desc) 2031 stmmac_mode_init(priv, tx_q->dma_etx, 2032 tx_q->dma_tx_phy, 2033 dma_conf->dma_tx_size, 1); 2034 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 2035 stmmac_mode_init(priv, tx_q->dma_tx, 2036 tx_q->dma_tx_phy, 2037 dma_conf->dma_tx_size, 0); 2038 } 2039 2040 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 2041 2042 for (i = 0; i < dma_conf->dma_tx_size; i++) { 2043 struct dma_desc *p; 2044 2045 p = stmmac_get_tx_desc(priv, tx_q, i); 2046 stmmac_clear_desc(priv, p); 2047 stmmac_set_tx_skb_dma_entry(tx_q, i, 0, 0, false); 2048 2049 tx_q->tx_skbuff[i] = NULL; 2050 } 2051 2052 return 0; 2053 } 2054 2055 static int init_dma_tx_desc_rings(struct net_device *dev, 2056 struct stmmac_dma_conf *dma_conf) 2057 { 2058 struct stmmac_priv *priv = netdev_priv(dev); 2059 u8 tx_queue_cnt; 2060 u8 queue; 2061 2062 tx_queue_cnt = priv->plat->tx_queues_to_use; 2063 2064 for (queue = 0; queue < tx_queue_cnt; queue++) 2065 __init_dma_tx_desc_rings(priv, dma_conf, queue); 2066 2067 return 0; 2068 } 2069 2070 /** 2071 * init_dma_desc_rings - init the RX/TX descriptor rings 2072 * @dev: net device structure 2073 * @dma_conf: structure to take the dma data 2074 * @flags: gfp flag. 2075 * Description: this function initializes the DMA RX/TX descriptors 2076 * and allocates the socket buffers. It supports the chained and ring 2077 * modes. 2078 */ 2079 static int init_dma_desc_rings(struct net_device *dev, 2080 struct stmmac_dma_conf *dma_conf, 2081 gfp_t flags) 2082 { 2083 struct stmmac_priv *priv = netdev_priv(dev); 2084 int ret; 2085 2086 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 2087 if (ret) 2088 return ret; 2089 2090 ret = init_dma_tx_desc_rings(dev, dma_conf); 2091 2092 stmmac_clear_descriptors(priv, dma_conf); 2093 2094 if (netif_msg_hw(priv)) 2095 stmmac_display_rings(priv, dma_conf); 2096 2097 return ret; 2098 } 2099 2100 /** 2101 * dma_free_tx_skbufs - free TX dma buffers 2102 * @priv: private structure 2103 * @dma_conf: structure to take the dma data 2104 * @queue: TX queue index 2105 */ 2106 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 2107 struct stmmac_dma_conf *dma_conf, 2108 u32 queue) 2109 { 2110 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2111 int i; 2112 2113 tx_q->xsk_frames_done = 0; 2114 2115 for (i = 0; i < dma_conf->dma_tx_size; i++) 2116 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 2117 2118 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 2119 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2120 tx_q->xsk_frames_done = 0; 2121 tx_q->xsk_pool = NULL; 2122 } 2123 } 2124 2125 /** 2126 * stmmac_free_tx_skbufs - free TX skb buffers 2127 * @priv: private structure 2128 */ 2129 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 2130 { 2131 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 2132 u8 queue; 2133 2134 for (queue = 0; queue < tx_queue_cnt; queue++) 2135 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 2136 } 2137 2138 /** 2139 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 2140 * @priv: private structure 2141 * @dma_conf: structure to take the dma data 2142 * @queue: RX queue index 2143 */ 2144 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 2145 struct stmmac_dma_conf *dma_conf, 2146 u32 queue) 2147 { 2148 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2149 size_t size; 2150 void *addr; 2151 2152 /* Release the DMA RX socket buffers */ 2153 if (rx_q->xsk_pool) 2154 dma_free_rx_xskbufs(priv, dma_conf, queue); 2155 else 2156 dma_free_rx_skbufs(priv, dma_conf, queue); 2157 2158 rx_q->buf_alloc_num = 0; 2159 rx_q->xsk_pool = NULL; 2160 2161 /* Free DMA regions of consistent memory previously allocated */ 2162 if (priv->extend_desc) 2163 addr = rx_q->dma_erx; 2164 else 2165 addr = rx_q->dma_rx; 2166 2167 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2168 2169 dma_free_coherent(priv->device, size, addr, rx_q->dma_rx_phy); 2170 2171 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 2172 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 2173 2174 kfree(rx_q->buf_pool); 2175 if (rx_q->page_pool) 2176 page_pool_destroy(rx_q->page_pool); 2177 } 2178 2179 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 2180 struct stmmac_dma_conf *dma_conf) 2181 { 2182 u8 rx_count = priv->plat->rx_queues_to_use; 2183 u8 queue; 2184 2185 /* Free RX queue resources */ 2186 for (queue = 0; queue < rx_count; queue++) 2187 __free_dma_rx_desc_resources(priv, dma_conf, queue); 2188 } 2189 2190 /** 2191 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 2192 * @priv: private structure 2193 * @dma_conf: structure to take the dma data 2194 * @queue: TX queue index 2195 */ 2196 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 2197 struct stmmac_dma_conf *dma_conf, 2198 u32 queue) 2199 { 2200 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2201 size_t size; 2202 void *addr; 2203 2204 /* Release the DMA TX socket buffers */ 2205 dma_free_tx_skbufs(priv, dma_conf, queue); 2206 2207 if (priv->extend_desc) { 2208 addr = tx_q->dma_etx; 2209 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 2210 addr = tx_q->dma_entx; 2211 } else { 2212 addr = tx_q->dma_tx; 2213 } 2214 2215 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2216 2217 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 2218 2219 kfree(tx_q->tx_skbuff_dma); 2220 kfree(tx_q->tx_skbuff); 2221 } 2222 2223 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2224 struct stmmac_dma_conf *dma_conf) 2225 { 2226 u8 tx_count = priv->plat->tx_queues_to_use; 2227 u8 queue; 2228 2229 /* Free TX queue resources */ 2230 for (queue = 0; queue < tx_count; queue++) 2231 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2232 } 2233 2234 /** 2235 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2236 * @priv: private structure 2237 * @dma_conf: structure to take the dma data 2238 * @queue: RX queue index 2239 * Description: according to which descriptor can be used (extend or basic) 2240 * this function allocates the resources for TX and RX paths. In case of 2241 * reception, for example, it pre-allocated the RX socket buffer in order to 2242 * allow zero-copy mechanism. 2243 */ 2244 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2245 struct stmmac_dma_conf *dma_conf, 2246 u32 queue) 2247 { 2248 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2249 struct stmmac_channel *ch = &priv->channel[queue]; 2250 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2251 struct page_pool_params pp_params = { 0 }; 2252 unsigned int dma_buf_sz_pad, num_pages; 2253 unsigned int napi_id; 2254 size_t size; 2255 void *addr; 2256 int ret; 2257 2258 dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + 2259 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2260 num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE); 2261 2262 rx_q->queue_index = queue; 2263 rx_q->priv_data = priv; 2264 rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE; 2265 2266 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2267 pp_params.pool_size = dma_conf->dma_rx_size; 2268 pp_params.order = order_base_2(num_pages); 2269 pp_params.nid = dev_to_node(priv->device); 2270 pp_params.dev = priv->device; 2271 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2272 pp_params.offset = stmmac_rx_offset(priv); 2273 pp_params.max_len = dma_conf->dma_buf_sz; 2274 2275 if (priv->sph_active) { 2276 pp_params.offset = 0; 2277 pp_params.max_len += stmmac_rx_offset(priv); 2278 } 2279 2280 rx_q->page_pool = page_pool_create(&pp_params); 2281 if (IS_ERR(rx_q->page_pool)) { 2282 ret = PTR_ERR(rx_q->page_pool); 2283 rx_q->page_pool = NULL; 2284 return ret; 2285 } 2286 2287 rx_q->buf_pool = kzalloc_objs(*rx_q->buf_pool, dma_conf->dma_rx_size); 2288 if (!rx_q->buf_pool) 2289 return -ENOMEM; 2290 2291 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2292 2293 addr = dma_alloc_coherent(priv->device, size, &rx_q->dma_rx_phy, 2294 GFP_KERNEL); 2295 if (!addr) 2296 return -ENOMEM; 2297 2298 if (priv->extend_desc) 2299 rx_q->dma_erx = addr; 2300 else 2301 rx_q->dma_rx = addr; 2302 2303 if (stmmac_xdp_is_enabled(priv) && 2304 test_bit(queue, priv->af_xdp_zc_qps)) 2305 napi_id = ch->rxtx_napi.napi_id; 2306 else 2307 napi_id = ch->rx_napi.napi_id; 2308 2309 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, queue, napi_id); 2310 if (ret) { 2311 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2312 return -EINVAL; 2313 } 2314 2315 return 0; 2316 } 2317 2318 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2319 struct stmmac_dma_conf *dma_conf) 2320 { 2321 u8 rx_count = priv->plat->rx_queues_to_use; 2322 u8 queue; 2323 int ret; 2324 2325 /* RX queues buffers and DMA */ 2326 for (queue = 0; queue < rx_count; queue++) { 2327 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2328 if (ret) 2329 goto err_dma; 2330 } 2331 2332 return 0; 2333 2334 err_dma: 2335 free_dma_rx_desc_resources(priv, dma_conf); 2336 2337 return ret; 2338 } 2339 2340 /** 2341 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2342 * @priv: private structure 2343 * @dma_conf: structure to take the dma data 2344 * @queue: TX queue index 2345 * Description: according to which descriptor can be used (extend or basic) 2346 * this function allocates the resources for TX and RX paths. In case of 2347 * reception, for example, it pre-allocated the RX socket buffer in order to 2348 * allow zero-copy mechanism. 2349 */ 2350 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2351 struct stmmac_dma_conf *dma_conf, 2352 u32 queue) 2353 { 2354 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2355 size_t size; 2356 void *addr; 2357 2358 tx_q->queue_index = queue; 2359 tx_q->priv_data = priv; 2360 2361 tx_q->tx_skbuff_dma = kzalloc_objs(*tx_q->tx_skbuff_dma, 2362 dma_conf->dma_tx_size); 2363 if (!tx_q->tx_skbuff_dma) 2364 return -ENOMEM; 2365 2366 tx_q->tx_skbuff = kzalloc_objs(struct sk_buff *, dma_conf->dma_tx_size); 2367 if (!tx_q->tx_skbuff) 2368 return -ENOMEM; 2369 2370 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2371 2372 addr = dma_alloc_coherent(priv->device, size, 2373 &tx_q->dma_tx_phy, GFP_KERNEL); 2374 if (!addr) 2375 return -ENOMEM; 2376 2377 if (priv->extend_desc) 2378 tx_q->dma_etx = addr; 2379 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2380 tx_q->dma_entx = addr; 2381 else 2382 tx_q->dma_tx = addr; 2383 2384 return 0; 2385 } 2386 2387 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2388 struct stmmac_dma_conf *dma_conf) 2389 { 2390 u8 tx_count = priv->plat->tx_queues_to_use; 2391 u8 queue; 2392 int ret; 2393 2394 /* TX queues buffers and DMA */ 2395 for (queue = 0; queue < tx_count; queue++) { 2396 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2397 if (ret) 2398 goto err_dma; 2399 } 2400 2401 return 0; 2402 2403 err_dma: 2404 free_dma_tx_desc_resources(priv, dma_conf); 2405 return ret; 2406 } 2407 2408 /** 2409 * alloc_dma_desc_resources - alloc TX/RX resources. 2410 * @priv: private structure 2411 * @dma_conf: structure to take the dma data 2412 * Description: according to which descriptor can be used (extend or basic) 2413 * this function allocates the resources for TX and RX paths. In case of 2414 * reception, for example, it pre-allocated the RX socket buffer in order to 2415 * allow zero-copy mechanism. 2416 */ 2417 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2418 struct stmmac_dma_conf *dma_conf) 2419 { 2420 /* RX Allocation */ 2421 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2422 2423 if (ret) 2424 return ret; 2425 2426 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2427 2428 return ret; 2429 } 2430 2431 /** 2432 * free_dma_desc_resources - free dma desc resources 2433 * @priv: private structure 2434 * @dma_conf: structure to take the dma data 2435 */ 2436 static void free_dma_desc_resources(struct stmmac_priv *priv, 2437 struct stmmac_dma_conf *dma_conf) 2438 { 2439 /* Release the DMA TX socket buffers */ 2440 free_dma_tx_desc_resources(priv, dma_conf); 2441 2442 /* Release the DMA RX socket buffers later 2443 * to ensure all pending XDP_TX buffers are returned. 2444 */ 2445 free_dma_rx_desc_resources(priv, dma_conf); 2446 } 2447 2448 /** 2449 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2450 * @priv: driver private structure 2451 * Description: It is used for enabling the rx queues in the MAC 2452 */ 2453 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2454 { 2455 u8 rx_queues_count = priv->plat->rx_queues_to_use; 2456 u8 queue; 2457 u8 mode; 2458 2459 for (queue = 0; queue < rx_queues_count; queue++) { 2460 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2461 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2462 } 2463 } 2464 2465 /** 2466 * stmmac_start_rx_dma - start RX DMA channel 2467 * @priv: driver private structure 2468 * @chan: RX channel index 2469 * Description: 2470 * This starts a RX DMA channel 2471 */ 2472 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2473 { 2474 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2475 stmmac_start_rx(priv, priv->ioaddr, chan); 2476 } 2477 2478 /** 2479 * stmmac_start_tx_dma - start TX DMA channel 2480 * @priv: driver private structure 2481 * @chan: TX channel index 2482 * Description: 2483 * This starts a TX DMA channel 2484 */ 2485 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2486 { 2487 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2488 stmmac_start_tx(priv, priv->ioaddr, chan); 2489 } 2490 2491 /** 2492 * stmmac_stop_rx_dma - stop RX DMA channel 2493 * @priv: driver private structure 2494 * @chan: RX channel index 2495 * Description: 2496 * This stops a RX DMA channel 2497 */ 2498 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2499 { 2500 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2501 stmmac_stop_rx(priv, priv->ioaddr, chan); 2502 } 2503 2504 /** 2505 * stmmac_stop_tx_dma - stop TX DMA channel 2506 * @priv: driver private structure 2507 * @chan: TX channel index 2508 * Description: 2509 * This stops a TX DMA channel 2510 */ 2511 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2512 { 2513 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2514 stmmac_stop_tx(priv, priv->ioaddr, chan); 2515 } 2516 2517 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2518 { 2519 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2520 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2521 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2522 u8 chan; 2523 2524 for (chan = 0; chan < dma_csr_ch; chan++) { 2525 struct stmmac_channel *ch = &priv->channel[chan]; 2526 unsigned long flags; 2527 2528 spin_lock_irqsave(&ch->lock, flags); 2529 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2530 spin_unlock_irqrestore(&ch->lock, flags); 2531 } 2532 } 2533 2534 /** 2535 * stmmac_start_all_dma - start all RX and TX DMA channels 2536 * @priv: driver private structure 2537 * Description: 2538 * This starts all the RX and TX DMA channels 2539 */ 2540 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2541 { 2542 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2543 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2544 u8 chan; 2545 2546 for (chan = 0; chan < rx_channels_count; chan++) 2547 stmmac_start_rx_dma(priv, chan); 2548 2549 for (chan = 0; chan < tx_channels_count; chan++) 2550 stmmac_start_tx_dma(priv, chan); 2551 } 2552 2553 /** 2554 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2555 * @priv: driver private structure 2556 * Description: 2557 * This stops the RX and TX DMA channels 2558 */ 2559 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2560 { 2561 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2562 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2563 u8 chan; 2564 2565 for (chan = 0; chan < rx_channels_count; chan++) 2566 stmmac_stop_rx_dma(priv, chan); 2567 2568 for (chan = 0; chan < tx_channels_count; chan++) 2569 stmmac_stop_tx_dma(priv, chan); 2570 } 2571 2572 /** 2573 * stmmac_dma_operation_mode - HW DMA operation mode 2574 * @priv: driver private structure 2575 * Description: it is used for configuring the DMA operation mode register in 2576 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2577 */ 2578 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2579 { 2580 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2581 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2582 int rxfifosz = priv->plat->rx_fifo_size; 2583 int txfifosz = priv->plat->tx_fifo_size; 2584 u32 txmode = 0; 2585 u32 rxmode = 0; 2586 u8 qmode = 0; 2587 u8 chan; 2588 2589 if (rxfifosz == 0) 2590 rxfifosz = priv->dma_cap.rx_fifo_size; 2591 if (txfifosz == 0) 2592 txfifosz = priv->dma_cap.tx_fifo_size; 2593 2594 /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ 2595 if (dwmac_is_xmac(priv->plat->core_type)) { 2596 rxfifosz /= rx_channels_count; 2597 txfifosz /= tx_channels_count; 2598 } 2599 2600 if (priv->plat->force_thresh_dma_mode) { 2601 txmode = tc; 2602 rxmode = tc; 2603 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2604 /* 2605 * In case of GMAC, SF mode can be enabled 2606 * to perform the TX COE in HW. This depends on: 2607 * 1) TX COE if actually supported 2608 * 2) There is no bugged Jumbo frame support 2609 * that needs to not insert csum in the TDES. 2610 */ 2611 txmode = SF_DMA_MODE; 2612 rxmode = SF_DMA_MODE; 2613 priv->xstats.threshold = SF_DMA_MODE; 2614 } else { 2615 txmode = tc; 2616 rxmode = SF_DMA_MODE; 2617 } 2618 2619 /* configure all channels */ 2620 for (chan = 0; chan < rx_channels_count; chan++) { 2621 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2622 2623 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2624 2625 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2626 rxfifosz, qmode); 2627 2628 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 2629 } 2630 2631 for (chan = 0; chan < tx_channels_count; chan++) { 2632 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2633 2634 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2635 txfifosz, qmode); 2636 } 2637 } 2638 2639 static void stmmac_xsk_request_timestamp(void *_priv) 2640 { 2641 struct stmmac_metadata_request *meta_req = _priv; 2642 2643 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2644 *meta_req->set_ic = true; 2645 } 2646 2647 static u64 stmmac_xsk_fill_timestamp(void *_priv) 2648 { 2649 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2650 struct stmmac_priv *priv = tx_compl->priv; 2651 struct dma_desc *desc = tx_compl->desc; 2652 bool found = false; 2653 u64 ns = 0; 2654 2655 if (!priv->hwts_tx_en) 2656 return 0; 2657 2658 /* check tx tstamp status */ 2659 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2660 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2661 found = true; 2662 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2663 found = true; 2664 } 2665 2666 if (found) { 2667 ns -= priv->plat->cdc_error_adj; 2668 return ns_to_ktime(ns); 2669 } 2670 2671 return 0; 2672 } 2673 2674 static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv) 2675 { 2676 struct timespec64 ts = ns_to_timespec64(launch_time); 2677 struct stmmac_metadata_request *meta_req = _priv; 2678 2679 if (meta_req->tbs & STMMAC_TBS_EN) 2680 stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec, 2681 ts.tv_nsec); 2682 } 2683 2684 static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2685 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2686 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2687 .tmo_request_launch_time = stmmac_xsk_request_launch_time, 2688 }; 2689 2690 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2691 { 2692 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2693 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2694 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2695 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 2696 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2697 unsigned int entry = tx_q->cur_tx; 2698 struct dma_desc *tx_desc = NULL; 2699 struct xdp_desc xdp_desc; 2700 bool work_done = true; 2701 u32 tx_set_ic_bit = 0; 2702 2703 /* Avoids TX time-out as we are sharing with slow path */ 2704 txq_trans_cond_update(nq); 2705 2706 budget = min(budget, stmmac_tx_avail(priv, queue)); 2707 2708 for (; budget > 0; budget--) { 2709 struct stmmac_metadata_request meta_req; 2710 struct xsk_tx_metadata *meta = NULL; 2711 dma_addr_t dma_addr; 2712 bool set_ic; 2713 2714 /* We are sharing with slow path and stop XSK TX desc submission when 2715 * available TX ring is less than threshold. 2716 */ 2717 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2718 !netif_carrier_ok(priv->dev)) { 2719 work_done = false; 2720 break; 2721 } 2722 2723 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2724 break; 2725 2726 if (priv->est && priv->est->enable && 2727 priv->est->max_sdu[queue] && 2728 xdp_desc.len > priv->est->max_sdu[queue]) { 2729 priv->xstats.max_sdu_txq_drop[queue]++; 2730 continue; 2731 } 2732 2733 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 2734 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2735 meta = xsk_buff_get_metadata(pool, xdp_desc.addr); 2736 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2737 2738 /* To return XDP buffer to XSK pool, we simple call 2739 * xsk_tx_completed(), so we don't need to fill up 2740 * 'buf' and 'xdpf'. 2741 */ 2742 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_XSK_TX, 2743 0, xdp_desc.len, false); 2744 stmmac_set_tx_dma_last_segment(tx_q, entry); 2745 2746 tx_q->xdpf[entry] = NULL; 2747 2748 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2749 2750 tx_q->tx_count_frames++; 2751 2752 if (!priv->tx_coal_frames[queue]) 2753 set_ic = false; 2754 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2755 set_ic = true; 2756 else 2757 set_ic = false; 2758 2759 meta_req.priv = priv; 2760 meta_req.tx_desc = tx_desc; 2761 meta_req.set_ic = &set_ic; 2762 meta_req.tbs = tx_q->tbs; 2763 meta_req.edesc = &tx_q->dma_entx[entry]; 2764 xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops, 2765 &meta_req); 2766 if (set_ic) { 2767 tx_q->tx_count_frames = 0; 2768 stmmac_set_tx_ic(priv, tx_desc); 2769 tx_set_ic_bit++; 2770 } 2771 2772 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2773 csum, priv->descriptor_mode, true, true, 2774 xdp_desc.len); 2775 2776 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 2777 2778 xsk_tx_metadata_to_compl(meta, 2779 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2780 2781 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2782 entry = tx_q->cur_tx; 2783 } 2784 u64_stats_update_begin(&txq_stats->napi_syncp); 2785 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2786 u64_stats_update_end(&txq_stats->napi_syncp); 2787 2788 if (tx_desc) { 2789 stmmac_flush_tx_descriptors(priv, queue); 2790 xsk_tx_release(pool); 2791 } 2792 2793 /* Return true if all of the 3 conditions are met 2794 * a) TX Budget is still available 2795 * b) work_done = true when XSK TX desc peek is empty (no more 2796 * pending XSK TX for transmission) 2797 */ 2798 return !!budget && work_done; 2799 } 2800 2801 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2802 { 2803 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2804 tc += 64; 2805 2806 if (priv->plat->force_thresh_dma_mode) 2807 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2808 else 2809 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2810 chan); 2811 2812 priv->xstats.threshold = tc; 2813 } 2814 } 2815 2816 /** 2817 * stmmac_tx_clean - to manage the transmission completion 2818 * @priv: driver private structure 2819 * @budget: napi budget limiting this functions packet handling 2820 * @queue: TX queue index 2821 * @pending_packets: signal to arm the TX coal timer 2822 * Description: it reclaims the transmit resources after transmission completes. 2823 * If some packets still needs to be handled, due to TX coalesce, set 2824 * pending_packets to true to make NAPI arm the TX coal timer. 2825 */ 2826 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2827 bool *pending_packets) 2828 { 2829 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2830 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2831 unsigned int bytes_compl = 0, pkts_compl = 0; 2832 unsigned int entry, xmits = 0, count = 0; 2833 u32 tx_packets = 0, tx_errors = 0; 2834 2835 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2836 2837 tx_q->xsk_frames_done = 0; 2838 2839 entry = tx_q->dirty_tx; 2840 2841 /* Try to clean all TX complete frame in 1 shot */ 2842 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2843 struct xdp_frame *xdpf; 2844 struct sk_buff *skb; 2845 struct dma_desc *p; 2846 int status; 2847 2848 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2849 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2850 xdpf = tx_q->xdpf[entry]; 2851 skb = NULL; 2852 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2853 xdpf = NULL; 2854 skb = tx_q->tx_skbuff[entry]; 2855 } else { 2856 xdpf = NULL; 2857 skb = NULL; 2858 } 2859 2860 p = stmmac_get_tx_desc(priv, tx_q, entry); 2861 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2862 /* Check if the descriptor is owned by the DMA */ 2863 if (unlikely(status & tx_dma_own)) 2864 break; 2865 2866 count++; 2867 2868 /* Make sure descriptor fields are read after reading 2869 * the own bit. 2870 */ 2871 dma_rmb(); 2872 2873 /* Just consider the last segment and ...*/ 2874 if (likely(!(status & tx_not_ls))) { 2875 /* ... verify the status error condition */ 2876 if (unlikely(status & tx_err)) { 2877 tx_errors++; 2878 if (unlikely(status & tx_err_bump_tc)) 2879 stmmac_bump_dma_threshold(priv, queue); 2880 } else { 2881 tx_packets++; 2882 } 2883 if (skb) { 2884 stmmac_get_tx_hwtstamp(priv, p, skb); 2885 } else if (tx_q->xsk_pool && 2886 xp_tx_metadata_enabled(tx_q->xsk_pool)) { 2887 struct stmmac_xsk_tx_complete tx_compl = { 2888 .priv = priv, 2889 .desc = p, 2890 }; 2891 2892 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2893 &stmmac_xsk_tx_metadata_ops, 2894 &tx_compl); 2895 } 2896 } 2897 2898 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2899 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2900 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2901 dma_unmap_page(priv->device, 2902 tx_q->tx_skbuff_dma[entry].buf, 2903 tx_q->tx_skbuff_dma[entry].len, 2904 DMA_TO_DEVICE); 2905 else 2906 dma_unmap_single(priv->device, 2907 tx_q->tx_skbuff_dma[entry].buf, 2908 tx_q->tx_skbuff_dma[entry].len, 2909 DMA_TO_DEVICE); 2910 tx_q->tx_skbuff_dma[entry].buf = 0; 2911 tx_q->tx_skbuff_dma[entry].len = 0; 2912 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2913 } 2914 2915 /* This looks at tx_q->tx_skbuff_dma[tx_q->dirty_tx].is_jumbo 2916 * and tx_q->tx_skbuff_dma[tx_q->dirty_tx].last_segment 2917 */ 2918 stmmac_clean_desc3(priv, tx_q, p); 2919 2920 tx_q->tx_skbuff_dma[entry].last_segment = false; 2921 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2922 2923 if (xdpf && 2924 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2925 xdp_return_frame_rx_napi(xdpf); 2926 tx_q->xdpf[entry] = NULL; 2927 } 2928 2929 if (xdpf && 2930 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2931 xdp_return_frame(xdpf); 2932 tx_q->xdpf[entry] = NULL; 2933 } 2934 2935 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2936 tx_q->xsk_frames_done++; 2937 2938 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2939 if (likely(skb)) { 2940 pkts_compl++; 2941 bytes_compl += skb->len; 2942 dev_consume_skb_any(skb); 2943 tx_q->tx_skbuff[entry] = NULL; 2944 } 2945 } 2946 2947 stmmac_release_tx_desc(priv, p, priv->descriptor_mode); 2948 2949 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 2950 } 2951 tx_q->dirty_tx = entry; 2952 2953 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2954 pkts_compl, bytes_compl); 2955 2956 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2957 queue))) && 2958 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2959 2960 netif_dbg(priv, tx_done, priv->dev, 2961 "%s: restart transmit\n", __func__); 2962 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2963 } 2964 2965 if (tx_q->xsk_pool) { 2966 bool work_done; 2967 2968 if (tx_q->xsk_frames_done) 2969 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2970 2971 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2972 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2973 2974 /* For XSK TX, we try to send as many as possible. 2975 * If XSK work done (XSK TX desc empty and budget still 2976 * available), return "budget - 1" to reenable TX IRQ. 2977 * Else, return "budget" to make NAPI continue polling. 2978 */ 2979 work_done = stmmac_xdp_xmit_zc(priv, queue, 2980 STMMAC_XSK_TX_BUDGET_MAX); 2981 if (work_done) 2982 xmits = budget - 1; 2983 else 2984 xmits = budget; 2985 } 2986 2987 if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode) 2988 stmmac_restart_sw_lpi_timer(priv); 2989 2990 /* We still have pending packets, let's call for a new scheduling */ 2991 if (tx_q->dirty_tx != tx_q->cur_tx) 2992 *pending_packets = true; 2993 2994 u64_stats_update_begin(&txq_stats->napi_syncp); 2995 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2996 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2997 u64_stats_inc(&txq_stats->napi.tx_clean); 2998 u64_stats_update_end(&txq_stats->napi_syncp); 2999 3000 priv->xstats.tx_errors += tx_errors; 3001 3002 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 3003 3004 /* Combine decisions from TX clean and XSK TX */ 3005 return max(count, xmits); 3006 } 3007 3008 /** 3009 * stmmac_tx_err - to manage the tx error 3010 * @priv: driver private structure 3011 * @chan: channel index 3012 * Description: it cleans the descriptors and restarts the transmission 3013 * in case of transmission errors. 3014 */ 3015 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 3016 { 3017 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3018 3019 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 3020 3021 stmmac_stop_tx_dma(priv, chan); 3022 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 3023 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 3024 stmmac_reset_tx_queue(priv, chan); 3025 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3026 tx_q->dma_tx_phy, chan); 3027 stmmac_start_tx_dma(priv, chan); 3028 3029 priv->xstats.tx_errors++; 3030 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 3031 } 3032 3033 /** 3034 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 3035 * @priv: driver private structure 3036 * @txmode: TX operating mode 3037 * @rxmode: RX operating mode 3038 * @chan: channel index 3039 * Description: it is used for configuring of the DMA operation mode in 3040 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 3041 * mode. 3042 */ 3043 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 3044 u32 rxmode, u32 chan) 3045 { 3046 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 3047 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 3048 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3049 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3050 int rxfifosz = priv->plat->rx_fifo_size; 3051 int txfifosz = priv->plat->tx_fifo_size; 3052 3053 if (rxfifosz == 0) 3054 rxfifosz = priv->dma_cap.rx_fifo_size; 3055 if (txfifosz == 0) 3056 txfifosz = priv->dma_cap.tx_fifo_size; 3057 3058 /* Adjust for real per queue fifo size */ 3059 rxfifosz /= rx_channels_count; 3060 txfifosz /= tx_channels_count; 3061 3062 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 3063 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 3064 } 3065 3066 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 3067 { 3068 int ret; 3069 3070 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 3071 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 3072 if (ret && (ret != -EINVAL)) { 3073 stmmac_global_err(priv); 3074 return true; 3075 } 3076 3077 return false; 3078 } 3079 3080 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 3081 { 3082 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 3083 &priv->xstats, chan, dir); 3084 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 3085 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3086 struct stmmac_channel *ch = &priv->channel[chan]; 3087 struct napi_struct *rx_napi; 3088 struct napi_struct *tx_napi; 3089 unsigned long flags; 3090 3091 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 3092 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3093 3094 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 3095 if (napi_schedule_prep(rx_napi)) { 3096 spin_lock_irqsave(&ch->lock, flags); 3097 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 3098 spin_unlock_irqrestore(&ch->lock, flags); 3099 __napi_schedule(rx_napi); 3100 } 3101 } 3102 3103 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 3104 if (napi_schedule_prep(tx_napi)) { 3105 spin_lock_irqsave(&ch->lock, flags); 3106 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 3107 spin_unlock_irqrestore(&ch->lock, flags); 3108 __napi_schedule(tx_napi); 3109 } 3110 } 3111 3112 return status; 3113 } 3114 3115 /** 3116 * stmmac_dma_interrupt - DMA ISR 3117 * @priv: driver private structure 3118 * Description: this is the DMA ISR. It is called by the main ISR. 3119 * It calls the dwmac dma routine and schedule poll method in case of some 3120 * work can be done. 3121 */ 3122 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 3123 { 3124 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3125 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3126 u8 channels_to_check = tx_channel_count > rx_channel_count ? 3127 tx_channel_count : rx_channel_count; 3128 int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 3129 u8 chan; 3130 3131 /* Make sure we never check beyond our status buffer. */ 3132 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 3133 channels_to_check = ARRAY_SIZE(status); 3134 3135 for (chan = 0; chan < channels_to_check; chan++) 3136 status[chan] = stmmac_napi_check(priv, chan, 3137 DMA_DIR_RXTX); 3138 3139 for (chan = 0; chan < tx_channel_count; chan++) { 3140 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 3141 /* Try to bump up the dma threshold on this failure */ 3142 stmmac_bump_dma_threshold(priv, chan); 3143 } else if (unlikely(status[chan] == tx_hard_error)) { 3144 stmmac_tx_err(priv, chan); 3145 } 3146 } 3147 } 3148 3149 /** 3150 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 3151 * @priv: driver private structure 3152 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 3153 */ 3154 static void stmmac_mmc_setup(struct stmmac_priv *priv) 3155 { 3156 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 3157 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 3158 3159 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 3160 3161 if (priv->dma_cap.rmon) { 3162 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 3163 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 3164 } else 3165 netdev_info(priv->dev, "No MAC Management Counters available\n"); 3166 } 3167 3168 /** 3169 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 3170 * @priv: driver private structure 3171 * Description: 3172 * new GMAC chip generations have a new register to indicate the 3173 * presence of the optional feature/functions. 3174 * This can be also used to override the value passed through the 3175 * platform and necessary for old MAC10/100 and GMAC chips. 3176 */ 3177 static int stmmac_get_hw_features(struct stmmac_priv *priv) 3178 { 3179 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 3180 } 3181 3182 /** 3183 * stmmac_check_ether_addr - check if the MAC addr is valid 3184 * @priv: driver private structure 3185 * Description: 3186 * it is to verify if the MAC address is valid, in case of failures it 3187 * generates a random MAC address 3188 */ 3189 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 3190 { 3191 u8 addr[ETH_ALEN]; 3192 3193 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 3194 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 3195 if (is_valid_ether_addr(addr)) 3196 eth_hw_addr_set(priv->dev, addr); 3197 else 3198 eth_hw_addr_random(priv->dev); 3199 dev_info(priv->device, "device MAC address %pM\n", 3200 priv->dev->dev_addr); 3201 } 3202 } 3203 3204 int stmmac_get_phy_intf_sel(phy_interface_t interface) 3205 { 3206 int phy_intf_sel = -EINVAL; 3207 3208 if (interface == PHY_INTERFACE_MODE_MII || 3209 interface == PHY_INTERFACE_MODE_GMII) 3210 phy_intf_sel = PHY_INTF_SEL_GMII_MII; 3211 else if (phy_interface_mode_is_rgmii(interface)) 3212 phy_intf_sel = PHY_INTF_SEL_RGMII; 3213 else if (interface == PHY_INTERFACE_MODE_RMII) 3214 phy_intf_sel = PHY_INTF_SEL_RMII; 3215 else if (interface == PHY_INTERFACE_MODE_REVMII) 3216 phy_intf_sel = PHY_INTF_SEL_REVMII; 3217 3218 return phy_intf_sel; 3219 } 3220 EXPORT_SYMBOL_GPL(stmmac_get_phy_intf_sel); 3221 3222 static int stmmac_prereset_configure(struct stmmac_priv *priv) 3223 { 3224 struct plat_stmmacenet_data *plat_dat = priv->plat; 3225 phy_interface_t interface; 3226 struct phylink_pcs *pcs; 3227 int phy_intf_sel, ret; 3228 3229 if (!plat_dat->set_phy_intf_sel) 3230 return 0; 3231 3232 interface = plat_dat->phy_interface; 3233 3234 /* Check whether this mode uses a PCS */ 3235 pcs = stmmac_mac_select_pcs(&priv->phylink_config, interface); 3236 if (priv->integrated_pcs && pcs == &priv->integrated_pcs->pcs) { 3237 /* Request the phy_intf_sel from the integrated PCS */ 3238 phy_intf_sel = stmmac_integrated_pcs_get_phy_intf_sel(pcs, 3239 interface); 3240 } else { 3241 phy_intf_sel = stmmac_get_phy_intf_sel(interface); 3242 } 3243 3244 if (phy_intf_sel < 0) { 3245 netdev_err(priv->dev, 3246 "failed to get phy_intf_sel for %s: %pe\n", 3247 phy_modes(interface), ERR_PTR(phy_intf_sel)); 3248 return phy_intf_sel; 3249 } 3250 3251 ret = plat_dat->set_phy_intf_sel(plat_dat->bsp_priv, phy_intf_sel); 3252 if (ret == -EINVAL) 3253 netdev_err(priv->dev, "platform does not support %s\n", 3254 phy_modes(interface)); 3255 else if (ret < 0) 3256 netdev_err(priv->dev, 3257 "platform failed to set interface %s: %pe\n", 3258 phy_modes(interface), ERR_PTR(ret)); 3259 3260 return ret; 3261 } 3262 3263 /** 3264 * stmmac_init_dma_engine - DMA init. 3265 * @priv: driver private structure 3266 * Description: 3267 * It inits the DMA invoking the specific MAC/GMAC callback. 3268 * Some DMA parameters can be passed from the platform; 3269 * in case of these are not passed a default is kept for the MAC or GMAC. 3270 */ 3271 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 3272 { 3273 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3274 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3275 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 3276 struct stmmac_rx_queue *rx_q; 3277 struct stmmac_tx_queue *tx_q; 3278 int ret = 0; 3279 u8 chan; 3280 3281 ret = stmmac_prereset_configure(priv); 3282 if (ret) 3283 return ret; 3284 3285 ret = stmmac_reset(priv); 3286 if (ret) { 3287 netdev_err(priv->dev, "Failed to reset the dma\n"); 3288 return ret; 3289 } 3290 3291 /* DMA Configuration */ 3292 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); 3293 3294 if (priv->plat->axi) 3295 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3296 3297 /* DMA CSR Channel configuration */ 3298 for (chan = 0; chan < dma_csr_ch; chan++) { 3299 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3300 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3301 } 3302 3303 /* DMA RX Channel Configuration */ 3304 for (chan = 0; chan < rx_channels_count; chan++) { 3305 rx_q = &priv->dma_conf.rx_queue[chan]; 3306 3307 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3308 rx_q->dma_rx_phy, chan); 3309 3310 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 3311 rx_q->buf_alloc_num); 3312 } 3313 3314 /* DMA TX Channel Configuration */ 3315 for (chan = 0; chan < tx_channels_count; chan++) { 3316 tx_q = &priv->dma_conf.tx_queue[chan]; 3317 3318 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3319 tx_q->dma_tx_phy, chan); 3320 3321 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 3322 } 3323 3324 return ret; 3325 } 3326 3327 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3328 { 3329 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3330 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3331 struct stmmac_channel *ch; 3332 struct napi_struct *napi; 3333 3334 if (!tx_coal_timer) 3335 return; 3336 3337 ch = &priv->channel[queue]; 3338 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3339 3340 /* Arm timer only if napi is not already scheduled. 3341 * Try to cancel any timer if napi is scheduled, timer will be armed 3342 * again in the next scheduled napi. 3343 */ 3344 if (unlikely(!napi_is_scheduled(napi))) { 3345 if (unlikely(!(hrtimer_active(&tx_q->txtimer)))) 3346 hrtimer_start(&tx_q->txtimer, 3347 STMMAC_COAL_TIMER(tx_coal_timer), 3348 HRTIMER_MODE_REL); 3349 } else { 3350 hrtimer_try_to_cancel(&tx_q->txtimer); 3351 } 3352 } 3353 3354 /** 3355 * stmmac_tx_timer - mitigation sw timer for tx. 3356 * @t: data pointer 3357 * Description: 3358 * This is the timer handler to directly invoke the stmmac_tx_clean. 3359 */ 3360 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3361 { 3362 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3363 struct stmmac_priv *priv = tx_q->priv_data; 3364 struct stmmac_channel *ch; 3365 struct napi_struct *napi; 3366 3367 ch = &priv->channel[tx_q->queue_index]; 3368 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3369 3370 if (likely(napi_schedule_prep(napi))) { 3371 unsigned long flags; 3372 3373 spin_lock_irqsave(&ch->lock, flags); 3374 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3375 spin_unlock_irqrestore(&ch->lock, flags); 3376 __napi_schedule(napi); 3377 } 3378 3379 return HRTIMER_NORESTART; 3380 } 3381 3382 /** 3383 * stmmac_init_coalesce - init mitigation options. 3384 * @priv: driver private structure 3385 * Description: 3386 * This inits the coalesce parameters: i.e. timer rate, 3387 * timer handler and default threshold used for enabling the 3388 * interrupt on completion bit. 3389 */ 3390 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3391 { 3392 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3393 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3394 u8 chan; 3395 3396 for (chan = 0; chan < tx_channel_count; chan++) { 3397 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3398 3399 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3400 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3401 3402 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3403 } 3404 3405 for (chan = 0; chan < rx_channel_count; chan++) 3406 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3407 } 3408 3409 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3410 { 3411 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3412 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3413 u8 chan; 3414 3415 /* set TX ring length */ 3416 for (chan = 0; chan < tx_channels_count; chan++) 3417 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3418 (priv->dma_conf.dma_tx_size - 1), chan); 3419 3420 /* set RX ring length */ 3421 for (chan = 0; chan < rx_channels_count; chan++) 3422 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3423 (priv->dma_conf.dma_rx_size - 1), chan); 3424 } 3425 3426 /** 3427 * stmmac_set_tx_queue_weight - Set TX queue weight 3428 * @priv: driver private structure 3429 * Description: It is used for setting TX queues weight 3430 */ 3431 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3432 { 3433 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3434 u32 weight; 3435 u8 queue; 3436 3437 for (queue = 0; queue < tx_queues_count; queue++) { 3438 weight = priv->plat->tx_queues_cfg[queue].weight; 3439 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3440 } 3441 } 3442 3443 /** 3444 * stmmac_configure_cbs - Configure CBS in TX queue 3445 * @priv: driver private structure 3446 * Description: It is used for configuring CBS in AVB TX queues 3447 */ 3448 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3449 { 3450 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3451 u32 mode_to_use; 3452 u8 queue; 3453 3454 /* queue 0 is reserved for legacy traffic */ 3455 for (queue = 1; queue < tx_queues_count; queue++) { 3456 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3457 if (mode_to_use == MTL_QUEUE_DCB) 3458 continue; 3459 3460 stmmac_config_cbs(priv, priv->hw, 3461 priv->plat->tx_queues_cfg[queue].send_slope, 3462 priv->plat->tx_queues_cfg[queue].idle_slope, 3463 priv->plat->tx_queues_cfg[queue].high_credit, 3464 priv->plat->tx_queues_cfg[queue].low_credit, 3465 queue); 3466 } 3467 } 3468 3469 /** 3470 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3471 * @priv: driver private structure 3472 * Description: It is used for mapping RX queues to RX dma channels 3473 */ 3474 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3475 { 3476 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3477 u8 queue; 3478 u32 chan; 3479 3480 for (queue = 0; queue < rx_queues_count; queue++) { 3481 chan = priv->plat->rx_queues_cfg[queue].chan; 3482 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3483 } 3484 } 3485 3486 /** 3487 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3488 * @priv: driver private structure 3489 * Description: It is used for configuring the RX Queue Priority 3490 */ 3491 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3492 { 3493 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3494 u8 queue; 3495 u32 prio; 3496 3497 for (queue = 0; queue < rx_queues_count; queue++) { 3498 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3499 continue; 3500 3501 prio = priv->plat->rx_queues_cfg[queue].prio; 3502 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3503 } 3504 } 3505 3506 /** 3507 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3508 * @priv: driver private structure 3509 * Description: It is used for configuring the TX Queue Priority 3510 */ 3511 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3512 { 3513 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3514 u8 queue; 3515 u32 prio; 3516 3517 for (queue = 0; queue < tx_queues_count; queue++) { 3518 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3519 continue; 3520 3521 prio = priv->plat->tx_queues_cfg[queue].prio; 3522 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3523 } 3524 } 3525 3526 /** 3527 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3528 * @priv: driver private structure 3529 * Description: It is used for configuring the RX queue routing 3530 */ 3531 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3532 { 3533 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3534 u8 packet; 3535 u8 queue; 3536 3537 for (queue = 0; queue < rx_queues_count; queue++) { 3538 /* no specific packet type routing specified for the queue */ 3539 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3540 continue; 3541 3542 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3543 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3544 } 3545 } 3546 3547 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3548 { 3549 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3550 priv->rss.enable = false; 3551 return; 3552 } 3553 3554 if (priv->dev->features & NETIF_F_RXHASH) 3555 priv->rss.enable = true; 3556 else 3557 priv->rss.enable = false; 3558 3559 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3560 priv->plat->rx_queues_to_use); 3561 } 3562 3563 /** 3564 * stmmac_mtl_configuration - Configure MTL 3565 * @priv: driver private structure 3566 * Description: It is used for configuring MTL 3567 */ 3568 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3569 { 3570 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3571 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3572 3573 if (tx_queues_count > 1) 3574 stmmac_set_tx_queue_weight(priv); 3575 3576 /* Configure MTL RX algorithms */ 3577 if (rx_queues_count > 1) 3578 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3579 priv->plat->rx_sched_algorithm); 3580 3581 /* Configure MTL TX algorithms */ 3582 if (tx_queues_count > 1) 3583 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3584 priv->plat->tx_sched_algorithm); 3585 3586 /* Configure CBS in AVB TX queues */ 3587 if (tx_queues_count > 1) 3588 stmmac_configure_cbs(priv); 3589 3590 /* Map RX MTL to DMA channels */ 3591 stmmac_rx_queue_dma_chan_map(priv); 3592 3593 /* Enable MAC RX Queues */ 3594 stmmac_mac_enable_rx_queues(priv); 3595 3596 /* Set RX priorities */ 3597 if (rx_queues_count > 1) 3598 stmmac_mac_config_rx_queues_prio(priv); 3599 3600 /* Set TX priorities */ 3601 if (tx_queues_count > 1) 3602 stmmac_mac_config_tx_queues_prio(priv); 3603 3604 /* Set RX routing */ 3605 if (rx_queues_count > 1) 3606 stmmac_mac_config_rx_queues_routing(priv); 3607 3608 /* Receive Side Scaling */ 3609 if (rx_queues_count > 1) 3610 stmmac_mac_config_rss(priv); 3611 } 3612 3613 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3614 { 3615 if (priv->dma_cap.asp) { 3616 netdev_info(priv->dev, "Enabling Safety Features\n"); 3617 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3618 priv->plat->safety_feat_cfg); 3619 } else { 3620 netdev_info(priv->dev, "No Safety Features support found\n"); 3621 } 3622 } 3623 3624 /* STM32MP25xx (dwmac v5.3) states "Do not enable time-based scheduling for 3625 * channels on which the TSO feature is enabled." If we have a skb for a 3626 * channel which has TBS enabled, fall back to software GSO. 3627 */ 3628 static bool stmmac_tso_channel_permitted(struct stmmac_priv *priv, 3629 unsigned int chan) 3630 { 3631 /* TSO and TBS cannot co-exist */ 3632 return !(priv->dma_conf.tx_queue[chan].tbs & STMMAC_TBS_AVAIL); 3633 } 3634 3635 /** 3636 * stmmac_hw_setup - setup mac in a usable state. 3637 * @dev : pointer to the device structure. 3638 * Description: 3639 * this is the main function to setup the HW in a usable state because the 3640 * dma engine is reset, the core registers are configured (e.g. AXI, 3641 * Checksum features, timers). The DMA is ready to start receiving and 3642 * transmitting. 3643 * Return value: 3644 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3645 * file on failure. 3646 */ 3647 static int stmmac_hw_setup(struct net_device *dev) 3648 { 3649 struct stmmac_priv *priv = netdev_priv(dev); 3650 u8 rx_cnt = priv->plat->rx_queues_to_use; 3651 u8 tx_cnt = priv->plat->tx_queues_to_use; 3652 bool sph_en; 3653 u8 chan; 3654 int ret; 3655 3656 /* Make sure RX clock is enabled */ 3657 if (priv->hw->phylink_pcs) 3658 phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs); 3659 3660 /* Note that clk_rx_i must be running for reset to complete. This 3661 * clock may also be required when setting the MAC address. 3662 * 3663 * Block the receive clock stop for LPI mode at the PHY in case 3664 * the link is established with EEE mode active. 3665 */ 3666 phylink_rx_clk_stop_block(priv->phylink); 3667 3668 /* DMA initialization and SW reset */ 3669 ret = stmmac_init_dma_engine(priv); 3670 if (ret < 0) { 3671 phylink_rx_clk_stop_unblock(priv->phylink); 3672 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3673 __func__); 3674 return ret; 3675 } 3676 3677 /* Copy the MAC addr into the HW */ 3678 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3679 phylink_rx_clk_stop_unblock(priv->phylink); 3680 3681 /* Initialize the MAC Core */ 3682 stmmac_core_init(priv, priv->hw, dev); 3683 3684 /* Initialize MTL*/ 3685 stmmac_mtl_configuration(priv); 3686 3687 /* Initialize Safety Features */ 3688 stmmac_safety_feat_configuration(priv); 3689 3690 ret = stmmac_rx_ipc(priv, priv->hw); 3691 if (!ret) { 3692 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3693 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3694 priv->hw->rx_csum = 0; 3695 } 3696 3697 /* Enable the MAC Rx/Tx */ 3698 stmmac_mac_set(priv, priv->ioaddr, true); 3699 3700 /* Set the HW DMA mode and the COE */ 3701 stmmac_dma_operation_mode(priv); 3702 3703 stmmac_mmc_setup(priv); 3704 3705 if (priv->use_riwt) { 3706 u32 queue; 3707 3708 for (queue = 0; queue < rx_cnt; queue++) { 3709 if (!priv->rx_riwt[queue]) 3710 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3711 3712 stmmac_rx_watchdog(priv, priv->ioaddr, 3713 priv->rx_riwt[queue], queue); 3714 } 3715 } 3716 3717 /* set TX and RX rings length */ 3718 stmmac_set_rings_length(priv); 3719 3720 /* Enable TSO */ 3721 if (priv->dma_cap.tsoen && priv->plat->flags & STMMAC_FLAG_TSO_EN) { 3722 for (chan = 0; chan < tx_cnt; chan++) { 3723 if (!stmmac_tso_channel_permitted(priv, chan)) 3724 continue; 3725 3726 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3727 } 3728 } 3729 3730 /* Enable Split Header */ 3731 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 3732 for (chan = 0; chan < rx_cnt; chan++) 3733 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3734 3735 3736 /* VLAN Tag Insertion */ 3737 if (priv->dma_cap.vlins) 3738 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3739 3740 /* TBS */ 3741 for (chan = 0; chan < tx_cnt; chan++) { 3742 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3743 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3744 3745 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3746 } 3747 3748 /* Configure real RX and TX queues */ 3749 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3750 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3751 3752 /* Start the ball rolling... */ 3753 stmmac_start_all_dma(priv); 3754 3755 phylink_rx_clk_stop_block(priv->phylink); 3756 stmmac_set_hw_vlan_mode(priv, priv->hw); 3757 phylink_rx_clk_stop_unblock(priv->phylink); 3758 3759 return 0; 3760 } 3761 3762 static void stmmac_free_irq(struct net_device *dev, 3763 enum request_irq_err irq_err, int irq_idx) 3764 { 3765 struct stmmac_priv *priv = netdev_priv(dev); 3766 struct stmmac_msi *msi = priv->msi; 3767 int j; 3768 3769 switch (irq_err) { 3770 case REQ_IRQ_ERR_ALL: 3771 irq_idx = priv->plat->tx_queues_to_use; 3772 fallthrough; 3773 case REQ_IRQ_ERR_TX: 3774 for (j = irq_idx - 1; msi && j >= 0; j--) { 3775 if (msi->tx_irq[j] > 0) { 3776 irq_set_affinity_hint(msi->tx_irq[j], NULL); 3777 free_irq(msi->tx_irq[j], 3778 &priv->dma_conf.tx_queue[j]); 3779 } 3780 } 3781 irq_idx = priv->plat->rx_queues_to_use; 3782 fallthrough; 3783 case REQ_IRQ_ERR_RX: 3784 for (j = irq_idx - 1; msi && j >= 0; j--) { 3785 if (msi->rx_irq[j] > 0) { 3786 irq_set_affinity_hint(msi->rx_irq[j], NULL); 3787 free_irq(msi->rx_irq[j], 3788 &priv->dma_conf.rx_queue[j]); 3789 } 3790 } 3791 3792 if (msi && msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) 3793 free_irq(msi->sfty_ue_irq, dev); 3794 fallthrough; 3795 case REQ_IRQ_ERR_SFTY_UE: 3796 if (msi && msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) 3797 free_irq(msi->sfty_ce_irq, dev); 3798 fallthrough; 3799 case REQ_IRQ_ERR_SFTY_CE: 3800 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3801 free_irq(priv->wol_irq, dev); 3802 fallthrough; 3803 case REQ_IRQ_ERR_SFTY: 3804 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) 3805 free_irq(priv->sfty_irq, dev); 3806 fallthrough; 3807 case REQ_IRQ_ERR_WOL: 3808 free_irq(dev->irq, dev); 3809 fallthrough; 3810 case REQ_IRQ_ERR_MAC: 3811 case REQ_IRQ_ERR_NO: 3812 /* If MAC IRQ request error, no more IRQ to free */ 3813 break; 3814 } 3815 } 3816 3817 static int stmmac_msi_init(struct stmmac_priv *priv, 3818 struct stmmac_resources *res) 3819 { 3820 int i; 3821 3822 priv->msi = devm_kmalloc(priv->device, sizeof(*priv->msi), GFP_KERNEL); 3823 if (!priv->msi) 3824 return -ENOMEM; 3825 3826 priv->msi->sfty_ce_irq = res->sfty_ce_irq; 3827 priv->msi->sfty_ue_irq = res->sfty_ue_irq; 3828 3829 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 3830 priv->msi->rx_irq[i] = res->rx_irq[i]; 3831 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 3832 priv->msi->tx_irq[i] = res->tx_irq[i]; 3833 3834 return 0; 3835 } 3836 3837 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3838 { 3839 struct stmmac_priv *priv = netdev_priv(dev); 3840 struct stmmac_msi *msi = priv->msi; 3841 enum request_irq_err irq_err; 3842 int irq_idx = 0; 3843 char *int_name; 3844 int ret; 3845 int i; 3846 3847 /* For common interrupt */ 3848 int_name = msi->int_name_mac; 3849 sprintf(int_name, "%s:%s", dev->name, "mac"); 3850 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3851 0, int_name, dev); 3852 if (unlikely(ret < 0)) { 3853 netdev_err(priv->dev, 3854 "%s: alloc mac MSI %d (error: %d)\n", 3855 __func__, dev->irq, ret); 3856 irq_err = REQ_IRQ_ERR_MAC; 3857 goto irq_error; 3858 } 3859 3860 /* Request the Wake IRQ in case of another line 3861 * is used for WoL 3862 */ 3863 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3864 int_name = msi->int_name_wol; 3865 sprintf(int_name, "%s:%s", dev->name, "wol"); 3866 ret = request_irq(priv->wol_irq, 3867 stmmac_mac_interrupt, 3868 0, int_name, dev); 3869 if (unlikely(ret < 0)) { 3870 netdev_err(priv->dev, 3871 "%s: alloc wol MSI %d (error: %d)\n", 3872 __func__, priv->wol_irq, ret); 3873 irq_err = REQ_IRQ_ERR_WOL; 3874 goto irq_error; 3875 } 3876 } 3877 3878 /* Request the common Safety Feature Correctible/Uncorrectible 3879 * Error line in case of another line is used 3880 */ 3881 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3882 int_name = msi->int_name_sfty; 3883 sprintf(int_name, "%s:%s", dev->name, "safety"); 3884 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3885 0, int_name, dev); 3886 if (unlikely(ret < 0)) { 3887 netdev_err(priv->dev, 3888 "%s: alloc sfty MSI %d (error: %d)\n", 3889 __func__, priv->sfty_irq, ret); 3890 irq_err = REQ_IRQ_ERR_SFTY; 3891 goto irq_error; 3892 } 3893 } 3894 3895 /* Request the Safety Feature Correctible Error line in 3896 * case of another line is used 3897 */ 3898 if (msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) { 3899 int_name = msi->int_name_sfty_ce; 3900 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3901 ret = request_irq(msi->sfty_ce_irq, 3902 stmmac_safety_interrupt, 3903 0, int_name, dev); 3904 if (unlikely(ret < 0)) { 3905 netdev_err(priv->dev, 3906 "%s: alloc sfty ce MSI %d (error: %d)\n", 3907 __func__, msi->sfty_ce_irq, ret); 3908 irq_err = REQ_IRQ_ERR_SFTY_CE; 3909 goto irq_error; 3910 } 3911 } 3912 3913 /* Request the Safety Feature Uncorrectible Error line in 3914 * case of another line is used 3915 */ 3916 if (msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) { 3917 int_name = msi->int_name_sfty_ue; 3918 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3919 ret = request_irq(msi->sfty_ue_irq, 3920 stmmac_safety_interrupt, 3921 0, int_name, dev); 3922 if (unlikely(ret < 0)) { 3923 netdev_err(priv->dev, 3924 "%s: alloc sfty ue MSI %d (error: %d)\n", 3925 __func__, msi->sfty_ue_irq, ret); 3926 irq_err = REQ_IRQ_ERR_SFTY_UE; 3927 goto irq_error; 3928 } 3929 } 3930 3931 /* Request Rx MSI irq */ 3932 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3933 if (i >= MTL_MAX_RX_QUEUES) 3934 break; 3935 if (msi->rx_irq[i] == 0) 3936 continue; 3937 3938 int_name = msi->int_name_rx_irq[i]; 3939 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3940 ret = request_irq(msi->rx_irq[i], 3941 stmmac_msi_intr_rx, 3942 0, int_name, &priv->dma_conf.rx_queue[i]); 3943 if (unlikely(ret < 0)) { 3944 netdev_err(priv->dev, 3945 "%s: alloc rx-%d MSI %d (error: %d)\n", 3946 __func__, i, msi->rx_irq[i], ret); 3947 irq_err = REQ_IRQ_ERR_RX; 3948 irq_idx = i; 3949 goto irq_error; 3950 } 3951 irq_set_affinity_hint(msi->rx_irq[i], 3952 cpumask_of(i % num_online_cpus())); 3953 } 3954 3955 /* Request Tx MSI irq */ 3956 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3957 if (i >= MTL_MAX_TX_QUEUES) 3958 break; 3959 if (msi->tx_irq[i] == 0) 3960 continue; 3961 3962 int_name = msi->int_name_tx_irq[i]; 3963 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3964 ret = request_irq(msi->tx_irq[i], 3965 stmmac_msi_intr_tx, 3966 0, int_name, &priv->dma_conf.tx_queue[i]); 3967 if (unlikely(ret < 0)) { 3968 netdev_err(priv->dev, 3969 "%s: alloc tx-%d MSI %d (error: %d)\n", 3970 __func__, i, msi->tx_irq[i], ret); 3971 irq_err = REQ_IRQ_ERR_TX; 3972 irq_idx = i; 3973 goto irq_error; 3974 } 3975 irq_set_affinity_hint(msi->tx_irq[i], 3976 cpumask_of(i % num_online_cpus())); 3977 } 3978 3979 return 0; 3980 3981 irq_error: 3982 stmmac_free_irq(dev, irq_err, irq_idx); 3983 return ret; 3984 } 3985 3986 static int stmmac_request_irq_single(struct net_device *dev) 3987 { 3988 struct stmmac_priv *priv = netdev_priv(dev); 3989 enum request_irq_err irq_err; 3990 int ret; 3991 3992 ret = request_irq(dev->irq, stmmac_interrupt, 3993 IRQF_SHARED, dev->name, dev); 3994 if (unlikely(ret < 0)) { 3995 netdev_err(priv->dev, 3996 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3997 __func__, dev->irq, ret); 3998 irq_err = REQ_IRQ_ERR_MAC; 3999 goto irq_error; 4000 } 4001 4002 /* Request the Wake IRQ in case of another line 4003 * is used for WoL 4004 */ 4005 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 4006 ret = request_irq(priv->wol_irq, stmmac_interrupt, 4007 IRQF_SHARED, dev->name, dev); 4008 if (unlikely(ret < 0)) { 4009 netdev_err(priv->dev, 4010 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 4011 __func__, priv->wol_irq, ret); 4012 irq_err = REQ_IRQ_ERR_WOL; 4013 goto irq_error; 4014 } 4015 } 4016 4017 /* Request the common Safety Feature Correctible/Uncorrectible 4018 * Error line in case of another line is used 4019 */ 4020 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 4021 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 4022 IRQF_SHARED, dev->name, dev); 4023 if (unlikely(ret < 0)) { 4024 netdev_err(priv->dev, 4025 "%s: ERROR: allocating the sfty IRQ %d (%d)\n", 4026 __func__, priv->sfty_irq, ret); 4027 irq_err = REQ_IRQ_ERR_SFTY; 4028 goto irq_error; 4029 } 4030 } 4031 4032 return 0; 4033 4034 irq_error: 4035 stmmac_free_irq(dev, irq_err, 0); 4036 return ret; 4037 } 4038 4039 static int stmmac_request_irq(struct net_device *dev) 4040 { 4041 struct stmmac_priv *priv = netdev_priv(dev); 4042 int ret; 4043 4044 /* Request the IRQ lines */ 4045 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 4046 ret = stmmac_request_irq_multi_msi(dev); 4047 else 4048 ret = stmmac_request_irq_single(dev); 4049 4050 return ret; 4051 } 4052 4053 /** 4054 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 4055 * @priv: driver private structure 4056 * @mtu: MTU to setup the dma queue and buf with 4057 * Description: Allocate and generate a dma_conf based on the provided MTU. 4058 * Allocate the Tx/Rx DMA queue and init them. 4059 * Return value: 4060 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 4061 */ 4062 static struct stmmac_dma_conf * 4063 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 4064 { 4065 struct stmmac_dma_conf *dma_conf; 4066 int bfsize, ret; 4067 u8 chan; 4068 4069 dma_conf = kzalloc_obj(*dma_conf); 4070 if (!dma_conf) { 4071 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 4072 __func__); 4073 return ERR_PTR(-ENOMEM); 4074 } 4075 4076 /* Returns 0 or BUF_SIZE_16KiB if mtu > 8KiB and dwmac4 or ring mode */ 4077 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 4078 if (bfsize < 0) 4079 bfsize = 0; 4080 4081 if (bfsize < BUF_SIZE_16KiB) 4082 bfsize = stmmac_set_bfsize(mtu); 4083 4084 dma_conf->dma_buf_sz = bfsize; 4085 /* Chose the tx/rx size from the already defined one in the 4086 * priv struct. (if defined) 4087 */ 4088 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 4089 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 4090 4091 if (!dma_conf->dma_tx_size) 4092 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 4093 if (!dma_conf->dma_rx_size) 4094 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 4095 4096 /* Earlier check for TBS */ 4097 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 4098 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 4099 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 4100 4101 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 4102 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 4103 } 4104 4105 ret = alloc_dma_desc_resources(priv, dma_conf); 4106 if (ret < 0) { 4107 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 4108 __func__); 4109 goto alloc_error; 4110 } 4111 4112 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 4113 if (ret < 0) { 4114 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 4115 __func__); 4116 goto init_error; 4117 } 4118 4119 return dma_conf; 4120 4121 init_error: 4122 free_dma_desc_resources(priv, dma_conf); 4123 alloc_error: 4124 kfree(dma_conf); 4125 return ERR_PTR(ret); 4126 } 4127 4128 /** 4129 * __stmmac_open - open entry point of the driver 4130 * @dev : pointer to the device structure. 4131 * @dma_conf : structure to take the dma data 4132 * Description: 4133 * This function is the open entry point of the driver. 4134 * Return value: 4135 * 0 on success and an appropriate (-)ve integer as defined in errno.h 4136 * file on failure. 4137 */ 4138 static int __stmmac_open(struct net_device *dev, 4139 struct stmmac_dma_conf *dma_conf) 4140 { 4141 struct stmmac_priv *priv = netdev_priv(dev); 4142 u8 chan; 4143 int ret; 4144 4145 for (int i = 0; i < priv->plat->tx_queues_to_use; i++) 4146 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 4147 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 4148 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 4149 4150 stmmac_reset_queues_param(priv); 4151 4152 ret = stmmac_hw_setup(dev); 4153 if (ret < 0) { 4154 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 4155 goto init_error; 4156 } 4157 4158 stmmac_setup_ptp(priv); 4159 4160 stmmac_init_coalesce(priv); 4161 4162 phylink_start(priv->phylink); 4163 4164 stmmac_vlan_restore(priv); 4165 4166 ret = stmmac_request_irq(dev); 4167 if (ret) 4168 goto irq_error; 4169 4170 stmmac_enable_all_queues(priv); 4171 netif_tx_start_all_queues(priv->dev); 4172 stmmac_enable_all_dma_irq(priv); 4173 4174 return 0; 4175 4176 irq_error: 4177 phylink_stop(priv->phylink); 4178 4179 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4180 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4181 4182 stmmac_release_ptp(priv); 4183 init_error: 4184 return ret; 4185 } 4186 4187 static int stmmac_open(struct net_device *dev) 4188 { 4189 struct stmmac_priv *priv = netdev_priv(dev); 4190 struct stmmac_dma_conf *dma_conf; 4191 int ret; 4192 4193 /* Initialise the tx lpi timer, converting from msec to usec */ 4194 if (!priv->tx_lpi_timer) 4195 priv->tx_lpi_timer = eee_timer * 1000; 4196 4197 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 4198 if (IS_ERR(dma_conf)) 4199 return PTR_ERR(dma_conf); 4200 4201 ret = pm_runtime_resume_and_get(priv->device); 4202 if (ret < 0) 4203 goto err_dma_resources; 4204 4205 ret = stmmac_init_phy(dev); 4206 if (ret) 4207 goto err_runtime_pm; 4208 4209 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 4210 ret = stmmac_legacy_serdes_power_up(priv); 4211 if (ret < 0) 4212 goto err_disconnect_phy; 4213 } 4214 4215 ret = __stmmac_open(dev, dma_conf); 4216 if (ret) 4217 goto err_serdes; 4218 4219 kfree(dma_conf); 4220 4221 /* We may have called phylink_speed_down before */ 4222 phylink_speed_up(priv->phylink); 4223 4224 return ret; 4225 4226 err_serdes: 4227 stmmac_legacy_serdes_power_down(priv); 4228 err_disconnect_phy: 4229 phylink_disconnect_phy(priv->phylink); 4230 err_runtime_pm: 4231 pm_runtime_put(priv->device); 4232 err_dma_resources: 4233 free_dma_desc_resources(priv, dma_conf); 4234 kfree(dma_conf); 4235 return ret; 4236 } 4237 4238 static void __stmmac_release(struct net_device *dev) 4239 { 4240 struct stmmac_priv *priv = netdev_priv(dev); 4241 u8 chan; 4242 4243 /* Stop and disconnect the PHY */ 4244 phylink_stop(priv->phylink); 4245 4246 stmmac_disable_all_queues(priv); 4247 4248 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4249 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4250 4251 netif_tx_disable(dev); 4252 4253 /* Free the IRQ lines */ 4254 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4255 4256 /* Stop TX/RX DMA and clear the descriptors */ 4257 stmmac_stop_all_dma(priv); 4258 4259 /* Release and free the Rx/Tx resources */ 4260 free_dma_desc_resources(priv, &priv->dma_conf); 4261 4262 stmmac_release_ptp(priv); 4263 4264 if (stmmac_fpe_supported(priv)) 4265 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 4266 } 4267 4268 /** 4269 * stmmac_release - close entry point of the driver 4270 * @dev : device pointer. 4271 * Description: 4272 * This is the stop entry point of the driver. 4273 */ 4274 static int stmmac_release(struct net_device *dev) 4275 { 4276 struct stmmac_priv *priv = netdev_priv(dev); 4277 4278 /* If the PHY or MAC has WoL enabled, then the PHY will not be 4279 * suspended when phylink_stop() is called below. Set the PHY 4280 * to its slowest speed to save power. 4281 */ 4282 if (device_may_wakeup(priv->device)) 4283 phylink_speed_down(priv->phylink, false); 4284 4285 __stmmac_release(dev); 4286 4287 stmmac_legacy_serdes_power_down(priv); 4288 phylink_disconnect_phy(priv->phylink); 4289 pm_runtime_put(priv->device); 4290 4291 return 0; 4292 } 4293 4294 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4295 struct stmmac_tx_queue *tx_q) 4296 { 4297 struct dma_desc *p; 4298 u16 tag = 0x0; 4299 4300 if (!priv->dma_cap.vlins || !skb_vlan_tag_present(skb)) 4301 return false; 4302 4303 tag = skb_vlan_tag_get(skb); 4304 4305 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4306 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4307 else 4308 p = &tx_q->dma_tx[tx_q->cur_tx]; 4309 4310 if (stmmac_set_desc_vlan_tag(priv, p, tag, 0x0, 0x0)) 4311 return false; 4312 4313 stmmac_set_tx_owner(priv, p); 4314 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4315 return true; 4316 } 4317 4318 /** 4319 * stmmac_tso_allocator - close entry point of the driver 4320 * @priv: driver private structure 4321 * @des: buffer start address 4322 * @total_len: total length to fill in descriptors 4323 * @last_segment: condition for the last descriptor 4324 * @queue: TX queue index 4325 * Description: 4326 * This function fills descriptor and request new descriptors according to 4327 * buffer length to fill 4328 */ 4329 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4330 int total_len, bool last_segment, u32 queue) 4331 { 4332 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4333 struct dma_desc *desc; 4334 u32 buff_size; 4335 int tmp_len; 4336 4337 tmp_len = total_len; 4338 4339 while (tmp_len > 0) { 4340 dma_addr_t curr_addr; 4341 4342 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, 4343 priv->dma_conf.dma_tx_size); 4344 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4345 4346 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4347 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4348 else 4349 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4350 4351 curr_addr = des + (total_len - tmp_len); 4352 stmmac_set_desc_addr(priv, desc, curr_addr); 4353 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4354 TSO_MAX_BUFF_SIZE : tmp_len; 4355 4356 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4357 0, 1, 4358 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4359 0, 0); 4360 4361 tmp_len -= TSO_MAX_BUFF_SIZE; 4362 } 4363 } 4364 4365 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4366 { 4367 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4368 4369 /* The own bit must be the latest setting done when prepare the 4370 * descriptor and then barrier is needed to make sure that 4371 * all is coherent before granting the DMA engine. 4372 */ 4373 wmb(); 4374 4375 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx); 4376 } 4377 4378 static void stmmac_set_gso_types(struct stmmac_priv *priv, bool tso) 4379 { 4380 if (!tso) { 4381 priv->gso_enabled_types = 0; 4382 } else { 4383 /* Manage oversized TCP frames for GMAC4 device */ 4384 priv->gso_enabled_types = SKB_GSO_TCPV4 | SKB_GSO_TCPV6; 4385 if (priv->plat->core_type == DWMAC_CORE_GMAC4) 4386 priv->gso_enabled_types |= SKB_GSO_UDP_L4; 4387 } 4388 } 4389 4390 static void stmmac_set_gso_features(struct net_device *ndev) 4391 { 4392 struct stmmac_priv *priv = netdev_priv(ndev); 4393 const struct stmmac_dma_cfg *dma_cfg; 4394 int txpbl; 4395 4396 if (priv->dma_cap.tsoen) 4397 dev_info(priv->device, "TSO supported\n"); 4398 4399 if (!(priv->plat->flags & STMMAC_FLAG_TSO_EN)) 4400 return; 4401 4402 if (!priv->dma_cap.tsoen) { 4403 dev_warn(priv->device, "platform requests unsupported TSO\n"); 4404 return; 4405 } 4406 4407 /* FIXME: 4408 * STM32MP151 (v4.2 userver v4.0) states that TxPBL must be >= 4. It 4409 * is not clear whether PBLx8 (which multiplies the PBL value by 8) 4410 * influences this. 4411 */ 4412 dma_cfg = priv->plat->dma_cfg; 4413 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl; 4414 if (txpbl < 4) { 4415 dev_warn(priv->device, "txpbl(%d) is too low for TSO\n", txpbl); 4416 return; 4417 } 4418 4419 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4420 if (priv->plat->core_type == DWMAC_CORE_GMAC4) 4421 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 4422 4423 stmmac_set_gso_types(priv, true); 4424 4425 dev_info(priv->device, "TSO feature enabled\n"); 4426 } 4427 4428 static size_t stmmac_tso_header_size(struct sk_buff *skb) 4429 { 4430 size_t size; 4431 4432 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4433 size = skb_transport_offset(skb) + sizeof(struct udphdr); 4434 else 4435 size = skb_tcp_all_headers(skb); 4436 4437 return size; 4438 } 4439 4440 /* STM32MP151 (dwmac v4.2) and STM32MP25xx (dwmac v5.3) states for TDES2 normal 4441 * (read format) descriptor that the maximum header length supported for the 4442 * TSO feature is 1023 bytes. 4443 * 4444 * While IPv4 is limited to MAC+VLAN+IPv4+ext+TCP+ext = 138 bytes, the IPv6 4445 * extension headers aren't similarly limited. 4446 * 4447 * Fall back to software GSO for these skbs. Also check that the MSS is >= 4448 * the recommended 64 bytes (documented in ETH_DMACxCR register description), 4449 * and that a the header plus MSS is not larger than 16383 (documented in 4450 * "Building the Descriptor and the packet for the TSO feature"). 4451 */ 4452 static bool stmmac_tso_valid_packet(struct sk_buff *skb) 4453 { 4454 size_t header_len = stmmac_tso_header_size(skb); 4455 unsigned int gso_size = skb_shinfo(skb)->gso_size; 4456 4457 return header_len <= 1023 && gso_size >= 64 && 4458 header_len + gso_size < 16383; 4459 } 4460 4461 /** 4462 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4463 * @skb : the socket buffer 4464 * @dev : device pointer 4465 * Description: this is the transmit function that is called on TSO frames 4466 * (support available on GMAC4 and newer chips). 4467 * Diagram below show the ring programming in case of TSO frames: 4468 * 4469 * First Descriptor 4470 * -------- 4471 * | DES0 |---> buffer1 = L2/L3/L4 header 4472 * | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address 4473 * | | width is 32-bit, but we never use it. 4474 * | | Also can be used as the most-significant 8-bits or 16-bits of 4475 * | | buffer1 address pointer if the DMA AXI address width is 40-bit 4476 * | | or 48-bit, and we always use it. 4477 * | DES2 |---> buffer1 len 4478 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4479 * -------- 4480 * -------- 4481 * | DES0 |---> buffer1 = TCP Payload (can continue on next descr...) 4482 * | DES1 |---> same as the First Descriptor 4483 * | DES2 |---> buffer1 len 4484 * | DES3 | 4485 * -------- 4486 * | 4487 * ... 4488 * | 4489 * -------- 4490 * | DES0 |---> buffer1 = Split TCP Payload 4491 * | DES1 |---> same as the First Descriptor 4492 * | DES2 |---> buffer1 len 4493 * | DES3 | 4494 * -------- 4495 * 4496 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4497 */ 4498 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4499 { 4500 struct dma_desc *desc, *first, *mss_desc = NULL; 4501 struct stmmac_priv *priv = netdev_priv(dev); 4502 unsigned int first_entry, tx_packets; 4503 struct stmmac_txq_stats *txq_stats; 4504 struct stmmac_tx_queue *tx_q; 4505 bool set_ic, is_last_segment; 4506 u32 pay_len, mss, queue; 4507 int i, first_tx, nfrags; 4508 u8 proto_hdr_len, hdr; 4509 dma_addr_t des; 4510 4511 nfrags = skb_shinfo(skb)->nr_frags; 4512 queue = skb_get_queue_mapping(skb); 4513 4514 tx_q = &priv->dma_conf.tx_queue[queue]; 4515 txq_stats = &priv->xstats.txq_stats[queue]; 4516 first_tx = tx_q->cur_tx; 4517 4518 /* Compute header lengths */ 4519 proto_hdr_len = stmmac_tso_header_size(skb); 4520 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4521 hdr = sizeof(struct udphdr); 4522 else 4523 hdr = tcp_hdrlen(skb); 4524 4525 /* Desc availability based on threshold should be enough safe */ 4526 if (unlikely(stmmac_tx_avail(priv, queue) < 4527 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4528 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4529 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4530 queue)); 4531 /* This is a hard error, log it. */ 4532 netdev_err(priv->dev, 4533 "%s: Tx Ring full when queue awake\n", 4534 __func__); 4535 } 4536 return NETDEV_TX_BUSY; 4537 } 4538 4539 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4540 4541 mss = skb_shinfo(skb)->gso_size; 4542 4543 /* set new MSS value if needed */ 4544 if (mss != tx_q->mss) { 4545 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4546 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4547 else 4548 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4549 4550 stmmac_set_mss(priv, mss_desc, mss); 4551 tx_q->mss = mss; 4552 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, 4553 priv->dma_conf.dma_tx_size); 4554 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4555 } 4556 4557 if (netif_msg_tx_queued(priv)) { 4558 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4559 __func__, hdr, proto_hdr_len, pay_len, mss); 4560 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4561 skb->data_len); 4562 } 4563 4564 first_entry = tx_q->cur_tx; 4565 WARN_ON(tx_q->tx_skbuff[first_entry]); 4566 4567 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4568 desc = &tx_q->dma_entx[first_entry].basic; 4569 else 4570 desc = &tx_q->dma_tx[first_entry]; 4571 first = desc; 4572 4573 /* first descriptor: fill Headers on Buf1 */ 4574 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4575 DMA_TO_DEVICE); 4576 if (dma_mapping_error(priv->device, des)) 4577 goto dma_map_err; 4578 4579 stmmac_set_desc_addr(priv, first, des); 4580 stmmac_tso_allocator(priv, des + proto_hdr_len, pay_len, 4581 (nfrags == 0), queue); 4582 4583 /* In case two or more DMA transmit descriptors are allocated for this 4584 * non-paged SKB data, the DMA buffer address should be saved to 4585 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4586 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4587 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4588 * since the tail areas of the DMA buffer can be accessed by DMA engine 4589 * sooner or later. 4590 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4591 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4592 * this DMA buffer right after the DMA engine completely finishes the 4593 * full buffer transmission. 4594 */ 4595 stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des, skb_headlen(skb), 4596 false); 4597 4598 /* Prepare fragments */ 4599 for (i = 0; i < nfrags; i++) { 4600 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4601 4602 des = skb_frag_dma_map(priv->device, frag, 0, 4603 skb_frag_size(frag), 4604 DMA_TO_DEVICE); 4605 if (dma_mapping_error(priv->device, des)) 4606 goto dma_map_err; 4607 4608 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4609 (i == nfrags - 1), queue); 4610 4611 stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des, 4612 skb_frag_size(frag), true); 4613 } 4614 4615 stmmac_set_tx_dma_last_segment(tx_q, tx_q->cur_tx); 4616 4617 /* Only the last descriptor gets to point to the skb. */ 4618 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4619 4620 /* Manage tx mitigation */ 4621 tx_packets = CIRC_CNT(tx_q->cur_tx + 1, first_tx, 4622 priv->dma_conf.dma_tx_size); 4623 tx_q->tx_count_frames += tx_packets; 4624 4625 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4626 set_ic = true; 4627 else if (!priv->tx_coal_frames[queue]) 4628 set_ic = false; 4629 else if (tx_packets > priv->tx_coal_frames[queue]) 4630 set_ic = true; 4631 else if ((tx_q->tx_count_frames % 4632 priv->tx_coal_frames[queue]) < tx_packets) 4633 set_ic = true; 4634 else 4635 set_ic = false; 4636 4637 if (set_ic) { 4638 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4639 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4640 else 4641 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4642 4643 tx_q->tx_count_frames = 0; 4644 stmmac_set_tx_ic(priv, desc); 4645 } 4646 4647 /* We've used all descriptors we need for this skb, however, 4648 * advance cur_tx so that it references a fresh descriptor. 4649 * ndo_start_xmit will fill this descriptor the next time it's 4650 * called and stmmac_tx_clean may clean up to this descriptor. 4651 */ 4652 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4653 4654 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4655 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4656 __func__); 4657 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4658 } 4659 4660 u64_stats_update_begin(&txq_stats->q_syncp); 4661 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4662 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4663 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4664 if (set_ic) 4665 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4666 u64_stats_update_end(&txq_stats->q_syncp); 4667 4668 if (priv->sarc_type) 4669 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4670 4671 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4672 priv->hwts_tx_en)) { 4673 /* declare that device is doing timestamping */ 4674 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4675 stmmac_enable_tx_timestamp(priv, first); 4676 } 4677 4678 /* If we only have one entry used, then the first entry is the last 4679 * segment. 4680 */ 4681 is_last_segment = CIRC_CNT(tx_q->cur_tx, first_entry, 4682 priv->dma_conf.dma_tx_size) == 1; 4683 4684 /* Complete the first descriptor before granting the DMA */ 4685 stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, 4686 is_last_segment, hdr / 4, 4687 skb->len - proto_hdr_len); 4688 4689 /* If context desc is used to change MSS */ 4690 if (mss_desc) { 4691 /* Make sure that first descriptor has been completely 4692 * written, including its own bit. This is because MSS is 4693 * actually before first descriptor, so we need to make 4694 * sure that MSS's own bit is the last thing written. 4695 */ 4696 dma_wmb(); 4697 stmmac_set_tx_owner(priv, mss_desc); 4698 } 4699 4700 if (netif_msg_pktdata(priv)) { 4701 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4702 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4703 tx_q->cur_tx, first, nfrags); 4704 pr_info(">>> frame to be transmitted: "); 4705 print_pkt(skb->data, skb_headlen(skb)); 4706 } 4707 4708 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4709 skb_tx_timestamp(skb); 4710 4711 stmmac_flush_tx_descriptors(priv, queue); 4712 stmmac_tx_timer_arm(priv, queue); 4713 4714 return NETDEV_TX_OK; 4715 4716 dma_map_err: 4717 dev_err(priv->device, "Tx dma map failed\n"); 4718 dev_kfree_skb(skb); 4719 priv->xstats.tx_dropped++; 4720 return NETDEV_TX_OK; 4721 } 4722 4723 /** 4724 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4725 * @skb: socket buffer to check 4726 * 4727 * Check if a packet has an ethertype that will trigger the IP header checks 4728 * and IP/TCP checksum engine of the stmmac core. 4729 * 4730 * Return: true if the ethertype can trigger the checksum engine, false 4731 * otherwise 4732 */ 4733 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4734 { 4735 int depth = 0; 4736 __be16 proto; 4737 4738 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4739 &depth); 4740 4741 return (depth <= ETH_HLEN) && 4742 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4743 } 4744 4745 /** 4746 * stmmac_xmit - Tx entry point of the driver 4747 * @skb : the socket buffer 4748 * @dev : device pointer 4749 * Description : this is the tx entry point of the driver. 4750 * It programs the chain or the ring and supports oversized frames 4751 * and SG feature. 4752 */ 4753 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4754 { 4755 bool enh_desc, has_vlan, set_ic, is_jumbo = false; 4756 struct stmmac_priv *priv = netdev_priv(dev); 4757 unsigned int nopaged_len = skb_headlen(skb); 4758 u32 queue = skb_get_queue_mapping(skb); 4759 int nfrags = skb_shinfo(skb)->nr_frags; 4760 unsigned int first_entry, tx_packets; 4761 struct stmmac_txq_stats *txq_stats; 4762 struct dma_desc *desc, *first_desc; 4763 struct stmmac_tx_queue *tx_q; 4764 int i, csum_insertion = 0; 4765 int entry, first_tx; 4766 dma_addr_t dma_addr; 4767 u32 sdu_len; 4768 4769 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4770 stmmac_stop_sw_lpi(priv); 4771 4772 if (skb_is_gso(skb) && 4773 skb_shinfo(skb)->gso_type & priv->gso_enabled_types) 4774 return stmmac_tso_xmit(skb, dev); 4775 4776 if (priv->est && priv->est->enable && 4777 priv->est->max_sdu[queue]) { 4778 sdu_len = skb->len; 4779 /* Add VLAN tag length if VLAN tag insertion offload is requested */ 4780 if (priv->dma_cap.vlins && skb_vlan_tag_present(skb)) 4781 sdu_len += VLAN_HLEN; 4782 if (sdu_len > priv->est->max_sdu[queue]) { 4783 priv->xstats.max_sdu_txq_drop[queue]++; 4784 goto max_sdu_err; 4785 } 4786 } 4787 4788 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4789 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4790 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4791 queue)); 4792 /* This is a hard error, log it. */ 4793 netdev_err(priv->dev, 4794 "%s: Tx Ring full when queue awake\n", 4795 __func__); 4796 } 4797 return NETDEV_TX_BUSY; 4798 } 4799 4800 tx_q = &priv->dma_conf.tx_queue[queue]; 4801 first_tx = tx_q->cur_tx; 4802 4803 /* Check if VLAN can be inserted by HW */ 4804 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4805 4806 entry = tx_q->cur_tx; 4807 first_entry = entry; 4808 WARN_ON(tx_q->tx_skbuff[first_entry]); 4809 4810 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4811 first_desc = desc; 4812 4813 if (has_vlan) 4814 stmmac_set_desc_vlan(priv, first_desc, STMMAC_VLAN_INSERT); 4815 4816 enh_desc = priv->plat->enh_desc; 4817 /* To program the descriptors according to the size of the frame */ 4818 if (enh_desc) 4819 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4820 4821 csum_insertion = skb->ip_summed == CHECKSUM_PARTIAL; 4822 4823 if (unlikely(is_jumbo)) { 4824 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4825 if (unlikely(entry < 0) && (entry != -EINVAL)) 4826 goto dma_map_err; 4827 } else { 4828 bool last_segment = (nfrags == 0); 4829 4830 dma_addr = dma_map_single(priv->device, skb->data, 4831 nopaged_len, DMA_TO_DEVICE); 4832 if (dma_mapping_error(priv->device, dma_addr)) 4833 goto dma_map_err; 4834 4835 stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr, 4836 nopaged_len, false); 4837 4838 stmmac_set_desc_addr(priv, first_desc, dma_addr); 4839 4840 if (last_segment) 4841 stmmac_set_tx_dma_last_segment(tx_q, first_entry); 4842 4843 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4844 priv->hwts_tx_en)) { 4845 /* declare that device is doing timestamping */ 4846 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4847 stmmac_enable_tx_timestamp(priv, first_desc); 4848 } 4849 4850 /* Prepare the first descriptor without setting the OWN bit */ 4851 stmmac_prepare_tx_desc(priv, first_desc, 1, nopaged_len, 4852 csum_insertion, priv->descriptor_mode, 4853 0, last_segment, skb->len); 4854 } 4855 4856 if (priv->sarc_type) 4857 stmmac_set_desc_sarc(priv, first_desc, priv->sarc_type); 4858 4859 /* STMMAC_TBS_EN can only be set if STMMAC_TBS_AVAIL has already 4860 * been set, which means the underlying type of the descriptors 4861 * will be struct stmmac_edesc. Therefore, it is safe to convert 4862 * the basic descriptor to the enhanced descriptor here. 4863 */ 4864 if (tx_q->tbs & STMMAC_TBS_EN) { 4865 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4866 4867 stmmac_set_desc_tbs(priv, dma_desc_to_edesc(first_desc), 4868 ts.tv_sec, ts.tv_nsec); 4869 } 4870 4871 for (i = 0; i < nfrags; i++) { 4872 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4873 unsigned int frag_size = skb_frag_size(frag); 4874 bool last_segment = (i == (nfrags - 1)); 4875 4876 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4877 WARN_ON(tx_q->tx_skbuff[entry]); 4878 4879 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4880 4881 dma_addr = skb_frag_dma_map(priv->device, frag, 0, frag_size, 4882 DMA_TO_DEVICE); 4883 if (dma_mapping_error(priv->device, dma_addr)) 4884 goto dma_map_err; /* should reuse desc w/o issues */ 4885 4886 stmmac_set_tx_skb_dma_entry(tx_q, entry, dma_addr, frag_size, 4887 true); 4888 stmmac_set_desc_addr(priv, desc, dma_addr); 4889 4890 /* Prepare the descriptor and set the own bit too */ 4891 stmmac_prepare_tx_desc(priv, desc, 0, frag_size, csum_insertion, 4892 priv->descriptor_mode, 1, last_segment, 4893 skb->len); 4894 } 4895 4896 stmmac_set_tx_dma_last_segment(tx_q, entry); 4897 4898 /* Only the last descriptor gets to point to the skb. */ 4899 tx_q->tx_skbuff[entry] = skb; 4900 4901 /* According to the coalesce parameter the IC bit for the latest 4902 * segment is reset and the timer re-started to clean the tx status. 4903 * This approach takes care about the fragments: desc is the first 4904 * element in case of no SG. 4905 */ 4906 tx_packets = CIRC_CNT(entry + 1, first_tx, priv->dma_conf.dma_tx_size); 4907 tx_q->tx_count_frames += tx_packets; 4908 4909 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4910 set_ic = true; 4911 else if (!priv->tx_coal_frames[queue]) 4912 set_ic = false; 4913 else if (tx_packets > priv->tx_coal_frames[queue]) 4914 set_ic = true; 4915 else if ((tx_q->tx_count_frames % 4916 priv->tx_coal_frames[queue]) < tx_packets) 4917 set_ic = true; 4918 else 4919 set_ic = false; 4920 4921 if (set_ic) { 4922 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4923 tx_q->tx_count_frames = 0; 4924 stmmac_set_tx_ic(priv, desc); 4925 } 4926 4927 /* We've used all descriptors we need for this skb, however, 4928 * advance cur_tx so that it references a fresh descriptor. 4929 * ndo_start_xmit will fill this descriptor the next time it's 4930 * called and stmmac_tx_clean may clean up to this descriptor. 4931 */ 4932 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4933 tx_q->cur_tx = entry; 4934 4935 if (netif_msg_pktdata(priv)) { 4936 netdev_dbg(priv->dev, 4937 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4938 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4939 entry, first_desc, nfrags); 4940 4941 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4942 print_pkt(skb->data, skb->len); 4943 } 4944 4945 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4946 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4947 __func__); 4948 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4949 } 4950 4951 txq_stats = &priv->xstats.txq_stats[queue]; 4952 u64_stats_update_begin(&txq_stats->q_syncp); 4953 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4954 if (set_ic) 4955 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4956 u64_stats_update_end(&txq_stats->q_syncp); 4957 4958 /* Set the OWN bit on the first descriptor now that all descriptors 4959 * for this skb are populated. 4960 */ 4961 stmmac_set_tx_owner(priv, first_desc); 4962 4963 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4964 4965 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 4966 skb_tx_timestamp(skb); 4967 stmmac_flush_tx_descriptors(priv, queue); 4968 stmmac_tx_timer_arm(priv, queue); 4969 4970 return NETDEV_TX_OK; 4971 4972 dma_map_err: 4973 netdev_err(priv->dev, "Tx DMA map failed\n"); 4974 max_sdu_err: 4975 dev_kfree_skb(skb); 4976 priv->xstats.tx_dropped++; 4977 return NETDEV_TX_OK; 4978 } 4979 4980 static netdev_features_t stmmac_features_check(struct sk_buff *skb, 4981 struct net_device *dev, 4982 netdev_features_t features) 4983 { 4984 struct stmmac_priv *priv = netdev_priv(dev); 4985 u16 queue = skb_get_queue_mapping(skb); 4986 4987 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 4988 * queues. In that case, checksum offloading for those queues that don't 4989 * support tx coe needs to fallback to software checksum calculation. 4990 * 4991 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 4992 * also have to be checksummed in software. 4993 * 4994 * Note that disabling hardware checksumming also disables TSO. See 4995 * harmonize_features() in net/core/dev.c 4996 */ 4997 if (priv->plat->tx_queues_cfg[queue].coe_unsupported || 4998 !stmmac_has_ip_ethertype(skb)) 4999 features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 5000 5001 if (skb_is_gso(skb)) { 5002 if (!stmmac_tso_channel_permitted(priv, queue) || 5003 !stmmac_tso_valid_packet(skb)) 5004 features &= ~NETIF_F_GSO_MASK; 5005 5006 /* If we are going to be using hardware TSO, always insert 5007 * VLAN tag to SKB payload for TSO frames. 5008 * 5009 * Never insert VLAN tag by HW, since segments split by 5010 * TSO engine will be un-tagged by mistake. 5011 */ 5012 if (features & NETIF_F_GSO_MASK) 5013 features &= ~(NETIF_F_HW_VLAN_STAG_TX | 5014 NETIF_F_HW_VLAN_CTAG_TX); 5015 } 5016 5017 return vlan_features_check(skb, features); 5018 } 5019 5020 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 5021 { 5022 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 5023 __be16 vlan_proto = veth->h_vlan_proto; 5024 u16 vlanid; 5025 5026 if ((vlan_proto == htons(ETH_P_8021Q) && 5027 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 5028 (vlan_proto == htons(ETH_P_8021AD) && 5029 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 5030 /* pop the vlan tag */ 5031 vlanid = ntohs(veth->h_vlan_TCI); 5032 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 5033 skb_pull(skb, VLAN_HLEN); 5034 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 5035 } 5036 } 5037 5038 /** 5039 * stmmac_rx_refill - refill used skb preallocated buffers 5040 * @priv: driver private structure 5041 * @queue: RX queue index 5042 * Description : this is to reallocate the skb for the reception process 5043 * that is based on zero-copy. 5044 */ 5045 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 5046 { 5047 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5048 int dirty = stmmac_rx_dirty(priv, queue); 5049 unsigned int entry = rx_q->dirty_rx; 5050 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 5051 5052 if (priv->dma_cap.host_dma_width <= 32) 5053 gfp |= GFP_DMA32; 5054 5055 while (dirty-- > 0) { 5056 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5057 struct dma_desc *p; 5058 bool use_rx_wd; 5059 5060 p = stmmac_get_rx_desc(priv, rx_q, entry); 5061 5062 if (!buf->page) { 5063 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5064 if (!buf->page) 5065 break; 5066 } 5067 5068 if (priv->sph_active && !buf->sec_page) { 5069 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5070 if (!buf->sec_page) 5071 break; 5072 5073 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 5074 } 5075 5076 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 5077 5078 stmmac_set_desc_addr(priv, p, buf->addr); 5079 if (priv->sph_active) 5080 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 5081 else 5082 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 5083 stmmac_refill_desc3(priv, rx_q, p); 5084 5085 rx_q->rx_count_frames++; 5086 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5087 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5088 rx_q->rx_count_frames = 0; 5089 5090 use_rx_wd = !priv->rx_coal_frames[queue]; 5091 use_rx_wd |= rx_q->rx_count_frames > 0; 5092 if (!priv->use_riwt) 5093 use_rx_wd = false; 5094 5095 dma_wmb(); 5096 stmmac_set_rx_owner(priv, p, use_rx_wd); 5097 5098 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5099 } 5100 rx_q->dirty_rx = entry; 5101 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5102 /* Wake up Rx DMA from the suspend state if required */ 5103 stmmac_enable_dma_reception(priv, priv->ioaddr, queue); 5104 } 5105 5106 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 5107 struct dma_desc *p, 5108 int status, unsigned int len) 5109 { 5110 unsigned int plen = 0, hlen = 0; 5111 int coe = priv->hw->rx_csum; 5112 5113 /* Not first descriptor, buffer is always zero */ 5114 if (priv->sph_active && len) 5115 return 0; 5116 5117 /* First descriptor, get split header length */ 5118 stmmac_get_rx_header_len(priv, p, &hlen); 5119 if (priv->sph_active && hlen) { 5120 priv->xstats.rx_split_hdr_pkt_n++; 5121 return hlen; 5122 } 5123 5124 /* First descriptor, not last descriptor and not split header */ 5125 if (status & rx_not_ls) 5126 return priv->dma_conf.dma_buf_sz; 5127 5128 plen = stmmac_get_rx_frame_len(priv, p, coe); 5129 5130 /* First descriptor and last descriptor and not split header */ 5131 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 5132 } 5133 5134 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 5135 struct dma_desc *p, 5136 int status, unsigned int len) 5137 { 5138 int coe = priv->hw->rx_csum; 5139 unsigned int plen = 0; 5140 5141 /* Not split header, buffer is not available */ 5142 if (!priv->sph_active) 5143 return 0; 5144 5145 /* For GMAC4, when split header is enabled, in some rare cases, the 5146 * hardware does not fill buf2 of the first descriptor with payload. 5147 * Thus we cannot assume buf2 is always fully filled if it is not 5148 * the last descriptor. Otherwise, the length of buf2 of the second 5149 * descriptor will be calculated wrong and cause an oops. 5150 * 5151 * If this is the last descriptor, 'plen' is the length of the 5152 * received packet that was transferred to system memory. 5153 * Otherwise, it is the accumulated number of bytes that have been 5154 * transferred for the current packet. 5155 * 5156 * Thus 'plen - len' always gives the correct length of buf2. 5157 */ 5158 5159 /* Not GMAC4 and not last descriptor */ 5160 if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) 5161 return priv->dma_conf.dma_buf_sz; 5162 5163 /* GMAC4 or last descriptor */ 5164 plen = stmmac_get_rx_frame_len(priv, p, coe); 5165 5166 return plen - len; 5167 } 5168 5169 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 5170 struct xdp_frame *xdpf, bool dma_map) 5171 { 5172 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 5173 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 5174 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 5175 unsigned int entry = tx_q->cur_tx; 5176 enum stmmac_txbuf_type buf_type; 5177 struct dma_desc *tx_desc; 5178 dma_addr_t dma_addr; 5179 bool set_ic; 5180 5181 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 5182 return STMMAC_XDP_CONSUMED; 5183 5184 if (priv->est && priv->est->enable && 5185 priv->est->max_sdu[queue] && 5186 xdpf->len > priv->est->max_sdu[queue]) { 5187 priv->xstats.max_sdu_txq_drop[queue]++; 5188 return STMMAC_XDP_CONSUMED; 5189 } 5190 5191 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 5192 if (dma_map) { 5193 dma_addr = dma_map_single(priv->device, xdpf->data, 5194 xdpf->len, DMA_TO_DEVICE); 5195 if (dma_mapping_error(priv->device, dma_addr)) 5196 return STMMAC_XDP_CONSUMED; 5197 5198 buf_type = STMMAC_TXBUF_T_XDP_NDO; 5199 } else { 5200 struct page *page = virt_to_page(xdpf->data); 5201 5202 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 5203 xdpf->headroom; 5204 dma_sync_single_for_device(priv->device, dma_addr, 5205 xdpf->len, DMA_BIDIRECTIONAL); 5206 5207 buf_type = STMMAC_TXBUF_T_XDP_TX; 5208 } 5209 5210 stmmac_set_tx_dma_entry(tx_q, entry, buf_type, dma_addr, xdpf->len, 5211 false); 5212 stmmac_set_tx_dma_last_segment(tx_q, entry); 5213 5214 tx_q->xdpf[entry] = xdpf; 5215 5216 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 5217 5218 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 5219 csum, priv->descriptor_mode, true, true, 5220 xdpf->len); 5221 5222 tx_q->tx_count_frames++; 5223 5224 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 5225 set_ic = true; 5226 else 5227 set_ic = false; 5228 5229 if (set_ic) { 5230 tx_q->tx_count_frames = 0; 5231 stmmac_set_tx_ic(priv, tx_desc); 5232 u64_stats_update_begin(&txq_stats->q_syncp); 5233 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 5234 u64_stats_update_end(&txq_stats->q_syncp); 5235 } 5236 5237 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5238 5239 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 5240 tx_q->cur_tx = entry; 5241 5242 return STMMAC_XDP_TX; 5243 } 5244 5245 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 5246 int cpu) 5247 { 5248 int index = cpu; 5249 5250 if (unlikely(index < 0)) 5251 index = 0; 5252 5253 while (index >= priv->plat->tx_queues_to_use) 5254 index -= priv->plat->tx_queues_to_use; 5255 5256 return index; 5257 } 5258 5259 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 5260 struct xdp_buff *xdp) 5261 { 5262 bool zc = !!(xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL); 5263 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 5264 int cpu = smp_processor_id(); 5265 struct netdev_queue *nq; 5266 int queue; 5267 int res; 5268 5269 if (unlikely(!xdpf)) 5270 return STMMAC_XDP_CONSUMED; 5271 5272 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5273 nq = netdev_get_tx_queue(priv->dev, queue); 5274 5275 __netif_tx_lock(nq, cpu); 5276 /* Avoids TX time-out as we are sharing with slow path */ 5277 txq_trans_cond_update(nq); 5278 5279 /* For zero copy XDP_TX action, dma_map is true */ 5280 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, zc); 5281 if (res == STMMAC_XDP_TX) { 5282 stmmac_flush_tx_descriptors(priv, queue); 5283 } else if (res == STMMAC_XDP_CONSUMED && zc) { 5284 /* xdp has been freed by xdp_convert_buff_to_frame(), 5285 * no need to call xsk_buff_free() again, so return 5286 * STMMAC_XSK_CONSUMED. 5287 */ 5288 res = STMMAC_XSK_CONSUMED; 5289 xdp_return_frame(xdpf); 5290 } 5291 5292 __netif_tx_unlock(nq); 5293 5294 return res; 5295 } 5296 5297 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 5298 struct bpf_prog *prog, 5299 struct xdp_buff *xdp) 5300 { 5301 u32 act; 5302 int res; 5303 5304 act = bpf_prog_run_xdp(prog, xdp); 5305 switch (act) { 5306 case XDP_PASS: 5307 res = STMMAC_XDP_PASS; 5308 break; 5309 case XDP_TX: 5310 res = stmmac_xdp_xmit_back(priv, xdp); 5311 break; 5312 case XDP_REDIRECT: 5313 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 5314 res = STMMAC_XDP_CONSUMED; 5315 else 5316 res = STMMAC_XDP_REDIRECT; 5317 break; 5318 default: 5319 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5320 fallthrough; 5321 case XDP_ABORTED: 5322 trace_xdp_exception(priv->dev, prog, act); 5323 fallthrough; 5324 case XDP_DROP: 5325 res = STMMAC_XDP_CONSUMED; 5326 break; 5327 } 5328 5329 return res; 5330 } 5331 5332 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5333 struct xdp_buff *xdp) 5334 { 5335 struct bpf_prog *prog; 5336 int res; 5337 5338 prog = READ_ONCE(priv->xdp_prog); 5339 if (!prog) { 5340 res = STMMAC_XDP_PASS; 5341 goto out; 5342 } 5343 5344 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5345 out: 5346 return ERR_PTR(-res); 5347 } 5348 5349 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5350 int xdp_status) 5351 { 5352 int cpu = smp_processor_id(); 5353 int queue; 5354 5355 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5356 5357 if (xdp_status & STMMAC_XDP_TX) 5358 stmmac_tx_timer_arm(priv, queue); 5359 5360 if (xdp_status & STMMAC_XDP_REDIRECT) 5361 xdp_do_flush(); 5362 } 5363 5364 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5365 struct xdp_buff *xdp) 5366 { 5367 unsigned int metasize = xdp->data - xdp->data_meta; 5368 unsigned int datasize = xdp->data_end - xdp->data; 5369 struct sk_buff *skb; 5370 5371 skb = napi_alloc_skb(&ch->rxtx_napi, 5372 xdp->data_end - xdp->data_hard_start); 5373 if (unlikely(!skb)) 5374 return NULL; 5375 5376 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5377 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5378 if (metasize) 5379 skb_metadata_set(skb, metasize); 5380 5381 return skb; 5382 } 5383 5384 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5385 struct dma_desc *p, struct dma_desc *np, 5386 struct xdp_buff *xdp) 5387 { 5388 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5389 struct stmmac_channel *ch = &priv->channel[queue]; 5390 unsigned int len = xdp->data_end - xdp->data; 5391 enum pkt_hash_types hash_type; 5392 int coe = priv->hw->rx_csum; 5393 struct sk_buff *skb; 5394 u32 hash; 5395 5396 skb = stmmac_construct_skb_zc(ch, xdp); 5397 if (!skb) { 5398 priv->xstats.rx_dropped++; 5399 return; 5400 } 5401 5402 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5403 if (priv->hw->hw_vlan_en) 5404 /* MAC level stripping. */ 5405 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5406 else 5407 /* Driver level stripping. */ 5408 stmmac_rx_vlan(priv->dev, skb); 5409 skb->protocol = eth_type_trans(skb, priv->dev); 5410 5411 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5412 skb_checksum_none_assert(skb); 5413 else 5414 skb->ip_summed = CHECKSUM_UNNECESSARY; 5415 5416 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5417 skb_set_hash(skb, hash, hash_type); 5418 5419 skb_record_rx_queue(skb, queue); 5420 napi_gro_receive(&ch->rxtx_napi, skb); 5421 5422 u64_stats_update_begin(&rxq_stats->napi_syncp); 5423 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5424 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5425 u64_stats_update_end(&rxq_stats->napi_syncp); 5426 } 5427 5428 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5429 { 5430 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5431 unsigned int entry = rx_q->dirty_rx; 5432 struct dma_desc *rx_desc = NULL; 5433 bool ret = true; 5434 5435 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5436 5437 while (budget-- > 0 && entry != rx_q->cur_rx) { 5438 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5439 dma_addr_t dma_addr; 5440 bool use_rx_wd; 5441 5442 if (!buf->xdp) { 5443 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5444 if (!buf->xdp) { 5445 ret = false; 5446 break; 5447 } 5448 } 5449 5450 rx_desc = stmmac_get_rx_desc(priv, rx_q, entry); 5451 5452 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5453 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5454 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5455 stmmac_refill_desc3(priv, rx_q, rx_desc); 5456 5457 rx_q->rx_count_frames++; 5458 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5459 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5460 rx_q->rx_count_frames = 0; 5461 5462 use_rx_wd = !priv->rx_coal_frames[queue]; 5463 use_rx_wd |= rx_q->rx_count_frames > 0; 5464 if (!priv->use_riwt) 5465 use_rx_wd = false; 5466 5467 dma_wmb(); 5468 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5469 5470 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5471 } 5472 5473 if (rx_desc) { 5474 rx_q->dirty_rx = entry; 5475 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5476 } 5477 5478 return ret; 5479 } 5480 5481 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5482 { 5483 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5484 * to represent incoming packet, whereas cb field in the same structure 5485 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5486 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5487 */ 5488 return (struct stmmac_xdp_buff *)xdp; 5489 } 5490 5491 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5492 { 5493 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5494 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5495 unsigned int count = 0, error = 0, len = 0; 5496 int dirty = stmmac_rx_dirty(priv, queue); 5497 unsigned int next_entry = rx_q->cur_rx; 5498 u32 rx_errors = 0, rx_dropped = 0; 5499 unsigned int desc_size; 5500 struct bpf_prog *prog; 5501 bool failure = false; 5502 int xdp_status = 0; 5503 int status = 0; 5504 5505 if (netif_msg_rx_status(priv)) { 5506 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5507 5508 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5509 desc_size = stmmac_get_rx_desc_size(priv); 5510 5511 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5512 rx_q->dma_rx_phy, desc_size); 5513 } 5514 while (count < limit) { 5515 struct stmmac_rx_buffer *buf; 5516 struct stmmac_xdp_buff *ctx; 5517 unsigned int buf1_len = 0; 5518 struct dma_desc *np, *p; 5519 int entry; 5520 int res; 5521 5522 if (!count && rx_q->state_saved) { 5523 error = rx_q->state.error; 5524 len = rx_q->state.len; 5525 } else { 5526 rx_q->state_saved = false; 5527 error = 0; 5528 len = 0; 5529 } 5530 5531 read_again: 5532 if (count >= limit) 5533 break; 5534 5535 buf1_len = 0; 5536 entry = next_entry; 5537 buf = &rx_q->buf_pool[entry]; 5538 5539 if (dirty >= STMMAC_RX_FILL_BATCH) { 5540 failure = failure || 5541 !stmmac_rx_refill_zc(priv, queue, dirty); 5542 dirty = 0; 5543 } 5544 5545 p = stmmac_get_rx_desc(priv, rx_q, entry); 5546 5547 /* read the status of the incoming frame */ 5548 status = stmmac_rx_status(priv, &priv->xstats, p); 5549 /* check if managed by the DMA otherwise go ahead */ 5550 if (unlikely(status & dma_own)) 5551 break; 5552 5553 /* Prefetch the next RX descriptor */ 5554 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5555 priv->dma_conf.dma_rx_size); 5556 if (unlikely(next_entry == rx_q->dirty_rx)) 5557 break; 5558 5559 rx_q->cur_rx = next_entry; 5560 5561 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5562 5563 prefetch(np); 5564 5565 /* Ensure a valid XSK buffer before proceed */ 5566 if (!buf->xdp) 5567 break; 5568 5569 if (priv->extend_desc) 5570 stmmac_rx_extended_status(priv, &priv->xstats, 5571 rx_q->dma_erx + entry); 5572 if (unlikely(status == discard_frame)) { 5573 xsk_buff_free(buf->xdp); 5574 buf->xdp = NULL; 5575 dirty++; 5576 error = 1; 5577 if (!priv->hwts_rx_en) 5578 rx_errors++; 5579 } 5580 5581 if (unlikely(error && (status & rx_not_ls))) 5582 goto read_again; 5583 if (unlikely(error)) { 5584 count++; 5585 continue; 5586 } 5587 5588 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5589 if (likely(status & rx_not_ls)) { 5590 xsk_buff_free(buf->xdp); 5591 buf->xdp = NULL; 5592 dirty++; 5593 count++; 5594 goto read_again; 5595 } 5596 5597 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5598 ctx->priv = priv; 5599 ctx->desc = p; 5600 ctx->ndesc = np; 5601 5602 /* XDP ZC Frame only support primary buffers for now */ 5603 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5604 len += buf1_len; 5605 5606 /* ACS is disabled; strip manually. */ 5607 if (likely(!(status & rx_not_ls))) { 5608 buf1_len -= ETH_FCS_LEN; 5609 len -= ETH_FCS_LEN; 5610 } 5611 5612 /* RX buffer is good and fit into a XSK pool buffer */ 5613 buf->xdp->data_end = buf->xdp->data + buf1_len; 5614 xsk_buff_dma_sync_for_cpu(buf->xdp); 5615 5616 prog = READ_ONCE(priv->xdp_prog); 5617 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5618 5619 switch (res) { 5620 case STMMAC_XDP_PASS: 5621 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5622 xsk_buff_free(buf->xdp); 5623 break; 5624 case STMMAC_XDP_CONSUMED: 5625 xsk_buff_free(buf->xdp); 5626 fallthrough; 5627 case STMMAC_XSK_CONSUMED: 5628 rx_dropped++; 5629 break; 5630 case STMMAC_XDP_TX: 5631 case STMMAC_XDP_REDIRECT: 5632 xdp_status |= res; 5633 break; 5634 } 5635 5636 buf->xdp = NULL; 5637 dirty++; 5638 count++; 5639 } 5640 5641 if (status & rx_not_ls) { 5642 rx_q->state_saved = true; 5643 rx_q->state.error = error; 5644 rx_q->state.len = len; 5645 } 5646 5647 stmmac_finalize_xdp_rx(priv, xdp_status); 5648 5649 u64_stats_update_begin(&rxq_stats->napi_syncp); 5650 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5651 u64_stats_update_end(&rxq_stats->napi_syncp); 5652 5653 priv->xstats.rx_dropped += rx_dropped; 5654 priv->xstats.rx_errors += rx_errors; 5655 5656 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5657 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5658 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5659 else 5660 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5661 5662 return (int)count; 5663 } 5664 5665 return failure ? limit : (int)count; 5666 } 5667 5668 /** 5669 * stmmac_rx - manage the receive process 5670 * @priv: driver private structure 5671 * @limit: napi bugget 5672 * @queue: RX queue index. 5673 * Description : this the function called by the napi poll method. 5674 * It gets all the frames inside the ring. 5675 */ 5676 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5677 { 5678 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5679 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5680 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5681 struct stmmac_channel *ch = &priv->channel[queue]; 5682 unsigned int count = 0, error = 0, len = 0; 5683 int status = 0, coe = priv->hw->rx_csum; 5684 unsigned int next_entry = rx_q->cur_rx; 5685 enum dma_data_direction dma_dir; 5686 unsigned int desc_size; 5687 struct sk_buff *skb = NULL; 5688 struct stmmac_xdp_buff ctx; 5689 int xdp_status = 0; 5690 int bufsz; 5691 5692 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5693 bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5694 5695 if (netif_msg_rx_status(priv)) { 5696 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5697 5698 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5699 desc_size = stmmac_get_rx_desc_size(priv); 5700 5701 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5702 rx_q->dma_rx_phy, desc_size); 5703 } 5704 while (count < limit) { 5705 unsigned int buf1_len = 0, buf2_len = 0; 5706 enum pkt_hash_types hash_type; 5707 struct stmmac_rx_buffer *buf; 5708 struct dma_desc *np, *p; 5709 int entry; 5710 u32 hash; 5711 5712 if (!count && rx_q->state_saved) { 5713 skb = rx_q->state.skb; 5714 error = rx_q->state.error; 5715 len = rx_q->state.len; 5716 } else { 5717 rx_q->state_saved = false; 5718 skb = NULL; 5719 error = 0; 5720 len = 0; 5721 } 5722 5723 read_again: 5724 if (count >= limit) 5725 break; 5726 5727 buf1_len = 0; 5728 buf2_len = 0; 5729 entry = next_entry; 5730 buf = &rx_q->buf_pool[entry]; 5731 5732 p = stmmac_get_rx_desc(priv, rx_q, entry); 5733 5734 /* read the status of the incoming frame */ 5735 status = stmmac_rx_status(priv, &priv->xstats, p); 5736 /* check if managed by the DMA otherwise go ahead */ 5737 if (unlikely(status & dma_own)) 5738 break; 5739 5740 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5741 priv->dma_conf.dma_rx_size); 5742 if (unlikely(next_entry == rx_q->dirty_rx)) 5743 break; 5744 5745 rx_q->cur_rx = next_entry; 5746 5747 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5748 5749 prefetch(np); 5750 5751 if (priv->extend_desc) 5752 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5753 if (unlikely(status == discard_frame)) { 5754 page_pool_put_page(rx_q->page_pool, buf->page, 0, true); 5755 buf->page = NULL; 5756 error = 1; 5757 if (!priv->hwts_rx_en) 5758 rx_errors++; 5759 } 5760 5761 if (unlikely(error && (status & rx_not_ls))) 5762 goto read_again; 5763 if (unlikely(error)) { 5764 dev_kfree_skb(skb); 5765 skb = NULL; 5766 count++; 5767 continue; 5768 } 5769 5770 /* Buffer is good. Go on. */ 5771 5772 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5773 len += buf1_len; 5774 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5775 len += buf2_len; 5776 5777 /* ACS is disabled; strip manually. */ 5778 if (likely(!(status & rx_not_ls))) { 5779 if (buf2_len) { 5780 buf2_len -= ETH_FCS_LEN; 5781 len -= ETH_FCS_LEN; 5782 } else if (buf1_len) { 5783 buf1_len -= ETH_FCS_LEN; 5784 len -= ETH_FCS_LEN; 5785 } 5786 } 5787 5788 if (!skb) { 5789 unsigned int pre_len, sync_len; 5790 5791 dma_sync_single_for_cpu(priv->device, buf->addr, 5792 buf1_len, dma_dir); 5793 net_prefetch(page_address(buf->page) + 5794 buf->page_offset); 5795 5796 xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq); 5797 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5798 buf->page_offset, buf1_len, true); 5799 5800 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5801 buf->page_offset; 5802 5803 ctx.priv = priv; 5804 ctx.desc = p; 5805 ctx.ndesc = np; 5806 5807 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5808 /* Due xdp_adjust_tail: DMA sync for_device 5809 * cover max len CPU touch 5810 */ 5811 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5812 buf->page_offset; 5813 sync_len = max(sync_len, pre_len); 5814 5815 /* For Not XDP_PASS verdict */ 5816 if (IS_ERR(skb)) { 5817 unsigned int xdp_res = -PTR_ERR(skb); 5818 5819 if (xdp_res & STMMAC_XDP_CONSUMED) { 5820 page_pool_put_page(rx_q->page_pool, 5821 virt_to_head_page(ctx.xdp.data), 5822 sync_len, true); 5823 buf->page = NULL; 5824 rx_dropped++; 5825 5826 /* Clear skb as it was set as 5827 * status by XDP program. 5828 */ 5829 skb = NULL; 5830 5831 if (unlikely((status & rx_not_ls))) 5832 goto read_again; 5833 5834 count++; 5835 continue; 5836 } else if (xdp_res & (STMMAC_XDP_TX | 5837 STMMAC_XDP_REDIRECT)) { 5838 xdp_status |= xdp_res; 5839 buf->page = NULL; 5840 skb = NULL; 5841 count++; 5842 continue; 5843 } 5844 } 5845 } 5846 5847 if (!skb) { 5848 unsigned int head_pad_len; 5849 5850 /* XDP program may expand or reduce tail */ 5851 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5852 5853 skb = napi_build_skb(page_address(buf->page), 5854 rx_q->napi_skb_frag_size); 5855 if (!skb) { 5856 page_pool_recycle_direct(rx_q->page_pool, 5857 buf->page); 5858 rx_dropped++; 5859 count++; 5860 goto drain_data; 5861 } 5862 5863 /* XDP program may adjust header */ 5864 head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; 5865 skb_reserve(skb, head_pad_len); 5866 skb_put(skb, buf1_len); 5867 skb_mark_for_recycle(skb); 5868 buf->page = NULL; 5869 } else if (buf1_len) { 5870 dma_sync_single_for_cpu(priv->device, buf->addr, 5871 buf1_len, dma_dir); 5872 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5873 buf->page, buf->page_offset, buf1_len, 5874 priv->dma_conf.dma_buf_sz); 5875 buf->page = NULL; 5876 } 5877 5878 if (buf2_len) { 5879 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5880 buf2_len, dma_dir); 5881 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5882 buf->sec_page, 0, buf2_len, 5883 priv->dma_conf.dma_buf_sz); 5884 buf->sec_page = NULL; 5885 } 5886 5887 drain_data: 5888 if (likely(status & rx_not_ls)) 5889 goto read_again; 5890 if (!skb) 5891 continue; 5892 5893 /* Got entire packet into SKB. Finish it. */ 5894 5895 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5896 5897 if (priv->hw->hw_vlan_en) 5898 /* MAC level stripping. */ 5899 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5900 else 5901 /* Driver level stripping. */ 5902 stmmac_rx_vlan(priv->dev, skb); 5903 5904 skb->protocol = eth_type_trans(skb, priv->dev); 5905 5906 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) || 5907 (status & csum_none)) 5908 skb_checksum_none_assert(skb); 5909 else 5910 skb->ip_summed = CHECKSUM_UNNECESSARY; 5911 5912 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5913 skb_set_hash(skb, hash, hash_type); 5914 5915 skb_record_rx_queue(skb, queue); 5916 napi_gro_receive(&ch->rx_napi, skb); 5917 skb = NULL; 5918 5919 rx_packets++; 5920 rx_bytes += len; 5921 count++; 5922 } 5923 5924 if (status & rx_not_ls || skb) { 5925 rx_q->state_saved = true; 5926 rx_q->state.skb = skb; 5927 rx_q->state.error = error; 5928 rx_q->state.len = len; 5929 } 5930 5931 stmmac_finalize_xdp_rx(priv, xdp_status); 5932 5933 stmmac_rx_refill(priv, queue); 5934 5935 u64_stats_update_begin(&rxq_stats->napi_syncp); 5936 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5937 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5938 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5939 u64_stats_update_end(&rxq_stats->napi_syncp); 5940 5941 priv->xstats.rx_dropped += rx_dropped; 5942 priv->xstats.rx_errors += rx_errors; 5943 5944 return count; 5945 } 5946 5947 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5948 { 5949 struct stmmac_channel *ch = 5950 container_of(napi, struct stmmac_channel, rx_napi); 5951 struct stmmac_priv *priv = ch->priv_data; 5952 struct stmmac_rxq_stats *rxq_stats; 5953 u32 chan = ch->index; 5954 int work_done; 5955 5956 rxq_stats = &priv->xstats.rxq_stats[chan]; 5957 u64_stats_update_begin(&rxq_stats->napi_syncp); 5958 u64_stats_inc(&rxq_stats->napi.poll); 5959 u64_stats_update_end(&rxq_stats->napi_syncp); 5960 5961 work_done = stmmac_rx(priv, budget, chan); 5962 if (work_done < budget && napi_complete_done(napi, work_done)) { 5963 unsigned long flags; 5964 5965 spin_lock_irqsave(&ch->lock, flags); 5966 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5967 spin_unlock_irqrestore(&ch->lock, flags); 5968 } 5969 5970 return work_done; 5971 } 5972 5973 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5974 { 5975 struct stmmac_channel *ch = 5976 container_of(napi, struct stmmac_channel, tx_napi); 5977 struct stmmac_priv *priv = ch->priv_data; 5978 struct stmmac_txq_stats *txq_stats; 5979 bool pending_packets = false; 5980 u32 chan = ch->index; 5981 int work_done; 5982 5983 txq_stats = &priv->xstats.txq_stats[chan]; 5984 u64_stats_update_begin(&txq_stats->napi_syncp); 5985 u64_stats_inc(&txq_stats->napi.poll); 5986 u64_stats_update_end(&txq_stats->napi_syncp); 5987 5988 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 5989 work_done = min(work_done, budget); 5990 5991 if (work_done < budget && napi_complete_done(napi, work_done)) { 5992 unsigned long flags; 5993 5994 spin_lock_irqsave(&ch->lock, flags); 5995 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5996 spin_unlock_irqrestore(&ch->lock, flags); 5997 } 5998 5999 /* TX still have packet to handle, check if we need to arm tx timer */ 6000 if (pending_packets) 6001 stmmac_tx_timer_arm(priv, chan); 6002 6003 return work_done; 6004 } 6005 6006 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 6007 { 6008 struct stmmac_channel *ch = 6009 container_of(napi, struct stmmac_channel, rxtx_napi); 6010 struct stmmac_priv *priv = ch->priv_data; 6011 bool tx_pending_packets = false; 6012 int rx_done, tx_done, rxtx_done; 6013 struct stmmac_rxq_stats *rxq_stats; 6014 struct stmmac_txq_stats *txq_stats; 6015 u32 chan = ch->index; 6016 6017 rxq_stats = &priv->xstats.rxq_stats[chan]; 6018 u64_stats_update_begin(&rxq_stats->napi_syncp); 6019 u64_stats_inc(&rxq_stats->napi.poll); 6020 u64_stats_update_end(&rxq_stats->napi_syncp); 6021 6022 txq_stats = &priv->xstats.txq_stats[chan]; 6023 u64_stats_update_begin(&txq_stats->napi_syncp); 6024 u64_stats_inc(&txq_stats->napi.poll); 6025 u64_stats_update_end(&txq_stats->napi_syncp); 6026 6027 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 6028 tx_done = min(tx_done, budget); 6029 6030 rx_done = stmmac_rx_zc(priv, budget, chan); 6031 6032 rxtx_done = max(tx_done, rx_done); 6033 6034 /* If either TX or RX work is not complete, return budget 6035 * and keep pooling 6036 */ 6037 if (rxtx_done >= budget) 6038 return budget; 6039 6040 /* all work done, exit the polling mode */ 6041 if (napi_complete_done(napi, rxtx_done)) { 6042 unsigned long flags; 6043 6044 spin_lock_irqsave(&ch->lock, flags); 6045 /* Both RX and TX work done are complete, 6046 * so enable both RX & TX IRQs. 6047 */ 6048 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6049 spin_unlock_irqrestore(&ch->lock, flags); 6050 } 6051 6052 /* TX still have packet to handle, check if we need to arm tx timer */ 6053 if (tx_pending_packets) 6054 stmmac_tx_timer_arm(priv, chan); 6055 6056 return min(rxtx_done, budget - 1); 6057 } 6058 6059 /** 6060 * stmmac_tx_timeout 6061 * @dev : Pointer to net device structure 6062 * @txqueue: the index of the hanging transmit queue 6063 * Description: this function is called when a packet transmission fails to 6064 * complete within a reasonable time. The driver will mark the error in the 6065 * netdev structure and arrange for the device to be reset to a sane state 6066 * in order to transmit a new packet. 6067 */ 6068 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 6069 { 6070 struct stmmac_priv *priv = netdev_priv(dev); 6071 6072 stmmac_global_err(priv); 6073 } 6074 6075 /** 6076 * stmmac_set_rx_mode - entry point for multicast addressing 6077 * @dev : pointer to the device structure 6078 * Description: 6079 * This function is a driver entry point which gets called by the kernel 6080 * whenever multicast addresses must be enabled/disabled. 6081 * Return value: 6082 * void. 6083 * 6084 * FIXME: This may need RXC to be running, but it may be called with BH 6085 * disabled, which means we can't call phylink_rx_clk_stop*(). 6086 */ 6087 static void stmmac_set_rx_mode(struct net_device *dev) 6088 { 6089 struct stmmac_priv *priv = netdev_priv(dev); 6090 6091 stmmac_set_filter(priv, priv->hw, dev); 6092 } 6093 6094 /** 6095 * stmmac_change_mtu - entry point to change MTU size for the device. 6096 * @dev : device pointer. 6097 * @new_mtu : the new MTU size for the device. 6098 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 6099 * to drive packet transmission. Ethernet has an MTU of 1500 octets 6100 * (ETH_DATA_LEN). This value can be changed with ifconfig. 6101 * Return value: 6102 * 0 on success and an appropriate (-)ve integer as defined in errno.h 6103 * file on failure. 6104 */ 6105 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 6106 { 6107 struct stmmac_priv *priv = netdev_priv(dev); 6108 int txfifosz = priv->plat->tx_fifo_size; 6109 struct stmmac_dma_conf *dma_conf; 6110 const int mtu = new_mtu; 6111 int ret; 6112 6113 if (txfifosz == 0) 6114 txfifosz = priv->dma_cap.tx_fifo_size; 6115 6116 txfifosz /= priv->plat->tx_queues_to_use; 6117 6118 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 6119 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 6120 return -EINVAL; 6121 } 6122 6123 new_mtu = STMMAC_ALIGN(new_mtu); 6124 6125 /* If condition true, FIFO is too small or MTU too large */ 6126 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 6127 return -EINVAL; 6128 6129 if (netif_running(dev)) { 6130 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 6131 /* Try to allocate the new DMA conf with the new mtu */ 6132 dma_conf = stmmac_setup_dma_desc(priv, mtu); 6133 if (IS_ERR(dma_conf)) { 6134 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 6135 mtu); 6136 return PTR_ERR(dma_conf); 6137 } 6138 6139 __stmmac_release(dev); 6140 6141 ret = __stmmac_open(dev, dma_conf); 6142 if (ret) { 6143 free_dma_desc_resources(priv, dma_conf); 6144 kfree(dma_conf); 6145 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 6146 return ret; 6147 } 6148 6149 kfree(dma_conf); 6150 6151 stmmac_set_rx_mode(dev); 6152 } 6153 6154 WRITE_ONCE(dev->mtu, mtu); 6155 netdev_update_features(dev); 6156 6157 return 0; 6158 } 6159 6160 static netdev_features_t stmmac_fix_features(struct net_device *dev, 6161 netdev_features_t features) 6162 { 6163 struct stmmac_priv *priv = netdev_priv(dev); 6164 6165 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 6166 features &= ~NETIF_F_RXCSUM; 6167 6168 if (!priv->plat->tx_coe) 6169 features &= ~NETIF_F_CSUM_MASK; 6170 6171 /* Some GMAC devices have a bugged Jumbo frame support that 6172 * needs to have the Tx COE disabled for oversized frames 6173 * (due to limited buffer sizes). In this case we disable 6174 * the TX csum insertion in the TDES and not use SF. 6175 */ 6176 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 6177 features &= ~NETIF_F_CSUM_MASK; 6178 6179 return features; 6180 } 6181 6182 static int stmmac_set_features(struct net_device *netdev, 6183 netdev_features_t features) 6184 { 6185 struct stmmac_priv *priv = netdev_priv(netdev); 6186 6187 /* Keep the COE Type in case of csum is supporting */ 6188 if (features & NETIF_F_RXCSUM) 6189 priv->hw->rx_csum = priv->plat->rx_coe; 6190 else 6191 priv->hw->rx_csum = 0; 6192 /* No check needed because rx_coe has been set before and it will be 6193 * fixed in case of issue. 6194 */ 6195 stmmac_rx_ipc(priv, priv->hw); 6196 6197 if (priv->sph_capable) { 6198 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 6199 u8 chan; 6200 6201 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 6202 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6203 } 6204 6205 stmmac_set_gso_types(priv, features & NETIF_F_TSO); 6206 6207 if (features & NETIF_F_HW_VLAN_CTAG_RX) 6208 priv->hw->hw_vlan_en = true; 6209 else 6210 priv->hw->hw_vlan_en = false; 6211 6212 phylink_rx_clk_stop_block(priv->phylink); 6213 stmmac_set_hw_vlan_mode(priv, priv->hw); 6214 phylink_rx_clk_stop_unblock(priv->phylink); 6215 6216 return 0; 6217 } 6218 6219 static void stmmac_common_interrupt(struct stmmac_priv *priv) 6220 { 6221 u8 rx_cnt = priv->plat->rx_queues_to_use; 6222 u8 tx_cnt = priv->plat->tx_queues_to_use; 6223 u8 queues_count; 6224 bool xmac; 6225 u8 queue; 6226 6227 xmac = dwmac_is_xmac(priv->plat->core_type); 6228 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 6229 6230 if (priv->irq_wake) 6231 pm_wakeup_event(priv->device, 0); 6232 6233 if (priv->dma_cap.estsel) 6234 stmmac_est_irq_status(priv, priv, priv->dev, 6235 &priv->xstats, tx_cnt); 6236 6237 if (stmmac_fpe_supported(priv)) 6238 stmmac_fpe_irq_status(priv); 6239 6240 /* To handle GMAC own interrupts */ 6241 if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) { 6242 int status = stmmac_host_irq_status(priv, &priv->xstats); 6243 6244 if (unlikely(status)) { 6245 /* For LPI we need to save the tx status */ 6246 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 6247 priv->tx_path_in_lpi_mode = true; 6248 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 6249 priv->tx_path_in_lpi_mode = false; 6250 } 6251 6252 for (queue = 0; queue < queues_count; queue++) 6253 stmmac_host_mtl_irq_status(priv, priv->hw, queue); 6254 6255 stmmac_timestamp_interrupt(priv, priv); 6256 } 6257 } 6258 6259 /** 6260 * stmmac_interrupt - main ISR 6261 * @irq: interrupt number. 6262 * @dev_id: to pass the net device pointer. 6263 * Description: this is the main driver interrupt service routine. 6264 * It can call: 6265 * o DMA service routine (to manage incoming frame reception and transmission 6266 * status) 6267 * o Core interrupts to manage: remote wake-up, management counter, LPI 6268 * interrupts. 6269 */ 6270 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6271 { 6272 struct net_device *dev = (struct net_device *)dev_id; 6273 struct stmmac_priv *priv = netdev_priv(dev); 6274 6275 /* Check if adapter is up */ 6276 if (test_bit(STMMAC_DOWN, &priv->state)) 6277 return IRQ_HANDLED; 6278 6279 /* Check ASP error if it isn't delivered via an individual IRQ */ 6280 if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) 6281 return IRQ_HANDLED; 6282 6283 /* To handle Common interrupts */ 6284 stmmac_common_interrupt(priv); 6285 6286 /* To handle DMA interrupts */ 6287 stmmac_dma_interrupt(priv); 6288 6289 return IRQ_HANDLED; 6290 } 6291 6292 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6293 { 6294 struct net_device *dev = (struct net_device *)dev_id; 6295 struct stmmac_priv *priv = netdev_priv(dev); 6296 6297 /* Check if adapter is up */ 6298 if (test_bit(STMMAC_DOWN, &priv->state)) 6299 return IRQ_HANDLED; 6300 6301 /* To handle Common interrupts */ 6302 stmmac_common_interrupt(priv); 6303 6304 return IRQ_HANDLED; 6305 } 6306 6307 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6308 { 6309 struct net_device *dev = (struct net_device *)dev_id; 6310 struct stmmac_priv *priv = netdev_priv(dev); 6311 6312 /* Check if adapter is up */ 6313 if (test_bit(STMMAC_DOWN, &priv->state)) 6314 return IRQ_HANDLED; 6315 6316 /* Check if a fatal error happened */ 6317 stmmac_safety_feat_interrupt(priv); 6318 6319 return IRQ_HANDLED; 6320 } 6321 6322 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6323 { 6324 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6325 struct stmmac_dma_conf *dma_conf; 6326 int chan = tx_q->queue_index; 6327 struct stmmac_priv *priv; 6328 int status; 6329 6330 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6331 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6332 6333 /* Check if adapter is up */ 6334 if (test_bit(STMMAC_DOWN, &priv->state)) 6335 return IRQ_HANDLED; 6336 6337 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6338 6339 if (unlikely(status & tx_hard_error_bump_tc)) { 6340 /* Try to bump up the dma threshold on this failure */ 6341 stmmac_bump_dma_threshold(priv, chan); 6342 } else if (unlikely(status == tx_hard_error)) { 6343 stmmac_tx_err(priv, chan); 6344 } 6345 6346 return IRQ_HANDLED; 6347 } 6348 6349 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6350 { 6351 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6352 struct stmmac_dma_conf *dma_conf; 6353 int chan = rx_q->queue_index; 6354 struct stmmac_priv *priv; 6355 6356 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6357 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6358 6359 /* Check if adapter is up */ 6360 if (test_bit(STMMAC_DOWN, &priv->state)) 6361 return IRQ_HANDLED; 6362 6363 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6364 6365 return IRQ_HANDLED; 6366 } 6367 6368 /** 6369 * stmmac_ioctl - Entry point for the Ioctl 6370 * @dev: Device pointer. 6371 * @rq: An IOCTL specific structure, that can contain a pointer to 6372 * a proprietary structure used to pass information to the driver. 6373 * @cmd: IOCTL command 6374 * Description: 6375 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6376 */ 6377 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6378 { 6379 struct stmmac_priv *priv = netdev_priv (dev); 6380 int ret = -EOPNOTSUPP; 6381 6382 if (!netif_running(dev)) 6383 return -EINVAL; 6384 6385 switch (cmd) { 6386 case SIOCGMIIPHY: 6387 case SIOCGMIIREG: 6388 case SIOCSMIIREG: 6389 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6390 break; 6391 default: 6392 break; 6393 } 6394 6395 return ret; 6396 } 6397 6398 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6399 void *cb_priv) 6400 { 6401 struct stmmac_priv *priv = cb_priv; 6402 int ret = -EOPNOTSUPP; 6403 6404 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6405 return ret; 6406 6407 __stmmac_disable_all_queues(priv); 6408 6409 switch (type) { 6410 case TC_SETUP_CLSU32: 6411 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6412 break; 6413 case TC_SETUP_CLSFLOWER: 6414 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6415 break; 6416 default: 6417 break; 6418 } 6419 6420 stmmac_enable_all_queues(priv); 6421 return ret; 6422 } 6423 6424 static LIST_HEAD(stmmac_block_cb_list); 6425 6426 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6427 void *type_data) 6428 { 6429 struct stmmac_priv *priv = netdev_priv(ndev); 6430 6431 switch (type) { 6432 case TC_QUERY_CAPS: 6433 return stmmac_tc_query_caps(priv, priv, type_data); 6434 case TC_SETUP_QDISC_MQPRIO: 6435 return stmmac_tc_setup_mqprio(priv, priv, type_data); 6436 case TC_SETUP_BLOCK: 6437 return flow_block_cb_setup_simple(type_data, 6438 &stmmac_block_cb_list, 6439 stmmac_setup_tc_block_cb, 6440 priv, priv, true); 6441 case TC_SETUP_QDISC_CBS: 6442 return stmmac_tc_setup_cbs(priv, priv, type_data); 6443 case TC_SETUP_QDISC_TAPRIO: 6444 return stmmac_tc_setup_taprio(priv, priv, type_data); 6445 case TC_SETUP_QDISC_ETF: 6446 return stmmac_tc_setup_etf(priv, priv, type_data); 6447 default: 6448 return -EOPNOTSUPP; 6449 } 6450 } 6451 6452 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6453 struct net_device *sb_dev) 6454 { 6455 int gso = skb_shinfo(skb)->gso_type; 6456 6457 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6458 /* 6459 * There is no way to determine the number of TSO/USO 6460 * capable Queues. Let's use always the Queue 0 6461 * because if TSO/USO is supported then at least this 6462 * one will be capable. 6463 */ 6464 return 0; 6465 } 6466 6467 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6468 } 6469 6470 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6471 { 6472 struct stmmac_priv *priv = netdev_priv(ndev); 6473 int ret = 0; 6474 6475 ret = pm_runtime_resume_and_get(priv->device); 6476 if (ret < 0) 6477 return ret; 6478 6479 ret = eth_mac_addr(ndev, addr); 6480 if (ret) 6481 goto set_mac_error; 6482 6483 phylink_rx_clk_stop_block(priv->phylink); 6484 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6485 phylink_rx_clk_stop_unblock(priv->phylink); 6486 6487 set_mac_error: 6488 pm_runtime_put(priv->device); 6489 6490 return ret; 6491 } 6492 6493 #ifdef CONFIG_DEBUG_FS 6494 static struct dentry *stmmac_fs_dir; 6495 6496 static void sysfs_display_ring(void *head, int size, int extend_desc, 6497 struct seq_file *seq, dma_addr_t dma_phy_addr) 6498 { 6499 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6500 struct dma_desc *p = (struct dma_desc *)head; 6501 unsigned int desc_size; 6502 dma_addr_t dma_addr; 6503 int i; 6504 6505 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6506 for (i = 0; i < size; i++) { 6507 dma_addr = dma_phy_addr + i * desc_size; 6508 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6509 i, &dma_addr, 6510 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6511 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6512 if (extend_desc) 6513 p = &(++ep)->basic; 6514 else 6515 p++; 6516 } 6517 } 6518 6519 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6520 { 6521 struct net_device *dev = seq->private; 6522 struct stmmac_priv *priv = netdev_priv(dev); 6523 u8 rx_count = priv->plat->rx_queues_to_use; 6524 u8 tx_count = priv->plat->tx_queues_to_use; 6525 u8 queue; 6526 6527 if ((dev->flags & IFF_UP) == 0) 6528 return 0; 6529 6530 for (queue = 0; queue < rx_count; queue++) { 6531 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6532 6533 seq_printf(seq, "RX Queue %d:\n", queue); 6534 6535 if (priv->extend_desc) { 6536 seq_printf(seq, "Extended descriptor ring:\n"); 6537 sysfs_display_ring((void *)rx_q->dma_erx, 6538 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6539 } else { 6540 seq_printf(seq, "Descriptor ring:\n"); 6541 sysfs_display_ring((void *)rx_q->dma_rx, 6542 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6543 } 6544 } 6545 6546 for (queue = 0; queue < tx_count; queue++) { 6547 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6548 6549 seq_printf(seq, "TX Queue %d:\n", queue); 6550 6551 if (priv->extend_desc) { 6552 seq_printf(seq, "Extended descriptor ring:\n"); 6553 sysfs_display_ring((void *)tx_q->dma_etx, 6554 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6555 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6556 seq_printf(seq, "Descriptor ring:\n"); 6557 sysfs_display_ring((void *)tx_q->dma_tx, 6558 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6559 } 6560 } 6561 6562 return 0; 6563 } 6564 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6565 6566 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6567 { 6568 static const char * const dwxgmac_timestamp_source[] = { 6569 "None", 6570 "Internal", 6571 "External", 6572 "Both", 6573 }; 6574 static const char * const dwxgmac_safety_feature_desc[] = { 6575 "No", 6576 "All Safety Features with ECC and Parity", 6577 "All Safety Features without ECC or Parity", 6578 "All Safety Features with Parity Only", 6579 "ECC Only", 6580 "UNDEFINED", 6581 "UNDEFINED", 6582 "UNDEFINED", 6583 }; 6584 struct net_device *dev = seq->private; 6585 struct stmmac_priv *priv = netdev_priv(dev); 6586 6587 if (!priv->hw_cap_support) { 6588 seq_printf(seq, "DMA HW features not supported\n"); 6589 return 0; 6590 } 6591 6592 seq_printf(seq, "==============================\n"); 6593 seq_printf(seq, "\tDMA HW features\n"); 6594 seq_printf(seq, "==============================\n"); 6595 6596 seq_printf(seq, "\t10/100 Mbps: %s\n", 6597 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6598 seq_printf(seq, "\t1000 Mbps: %s\n", 6599 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6600 seq_printf(seq, "\tHalf duplex: %s\n", 6601 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6602 if (priv->plat->core_type == DWMAC_CORE_XGMAC) { 6603 seq_printf(seq, 6604 "\tNumber of Additional MAC address registers: %d\n", 6605 priv->dma_cap.multi_addr); 6606 } else { 6607 seq_printf(seq, "\tHash Filter: %s\n", 6608 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6609 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6610 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6611 } 6612 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6613 (priv->dma_cap.pcs) ? "Y" : "N"); 6614 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6615 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6616 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6617 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6618 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6619 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6620 seq_printf(seq, "\tRMON module: %s\n", 6621 (priv->dma_cap.rmon) ? "Y" : "N"); 6622 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6623 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6624 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6625 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6626 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 6627 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6628 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6629 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6630 (priv->dma_cap.eee) ? "Y" : "N"); 6631 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6632 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6633 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6634 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6635 priv->plat->core_type == DWMAC_CORE_XGMAC) { 6636 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6637 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6638 } else { 6639 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6640 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6641 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6642 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6643 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6644 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6645 } 6646 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6647 priv->dma_cap.number_rx_channel); 6648 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6649 priv->dma_cap.number_tx_channel); 6650 seq_printf(seq, "\tNumber of Additional RX queues: %u\n", 6651 priv->dma_cap.number_rx_queues); 6652 seq_printf(seq, "\tNumber of Additional TX queues: %u\n", 6653 priv->dma_cap.number_tx_queues); 6654 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6655 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6656 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6657 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6658 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6659 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6660 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6661 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6662 priv->dma_cap.pps_out_num); 6663 seq_printf(seq, "\tSafety Features: %s\n", 6664 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6665 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6666 priv->dma_cap.frpsel ? "Y" : "N"); 6667 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6668 priv->dma_cap.host_dma_width); 6669 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6670 priv->dma_cap.rssen ? "Y" : "N"); 6671 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6672 priv->dma_cap.vlhash ? "Y" : "N"); 6673 seq_printf(seq, "\tSplit Header: %s\n", 6674 priv->dma_cap.sphen ? "Y" : "N"); 6675 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6676 priv->dma_cap.vlins ? "Y" : "N"); 6677 seq_printf(seq, "\tDouble VLAN: %s\n", 6678 priv->dma_cap.dvlan ? "Y" : "N"); 6679 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6680 priv->dma_cap.l3l4fnum); 6681 seq_printf(seq, "\tARP Offloading: %s\n", 6682 priv->dma_cap.arpoffsel ? "Y" : "N"); 6683 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6684 priv->dma_cap.estsel ? "Y" : "N"); 6685 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6686 priv->dma_cap.fpesel ? "Y" : "N"); 6687 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6688 priv->dma_cap.tbssel ? "Y" : "N"); 6689 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6690 priv->dma_cap.tbs_ch_num); 6691 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6692 priv->dma_cap.sgfsel ? "Y" : "N"); 6693 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6694 BIT(priv->dma_cap.ttsfd) >> 1); 6695 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6696 priv->dma_cap.numtc); 6697 seq_printf(seq, "\tDCB Feature: %s\n", 6698 priv->dma_cap.dcben ? "Y" : "N"); 6699 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6700 priv->dma_cap.advthword ? "Y" : "N"); 6701 seq_printf(seq, "\tPTP Offload: %s\n", 6702 priv->dma_cap.ptoen ? "Y" : "N"); 6703 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6704 priv->dma_cap.osten ? "Y" : "N"); 6705 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6706 priv->dma_cap.pfcen ? "Y" : "N"); 6707 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6708 BIT(priv->dma_cap.frpes) << 6); 6709 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6710 BIT(priv->dma_cap.frpbs) << 6); 6711 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6712 priv->dma_cap.frppipe_num); 6713 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6714 priv->dma_cap.nrvf_num ? 6715 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6716 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6717 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6718 seq_printf(seq, "\tDepth of GCL: %lu\n", 6719 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6720 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6721 priv->dma_cap.cbtisel ? "Y" : "N"); 6722 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6723 priv->dma_cap.aux_snapshot_n); 6724 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6725 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6726 seq_printf(seq, "\tEnhanced DMA: %s\n", 6727 priv->dma_cap.edma ? "Y" : "N"); 6728 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6729 priv->dma_cap.ediffc ? "Y" : "N"); 6730 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6731 priv->dma_cap.vxn ? "Y" : "N"); 6732 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6733 priv->dma_cap.dbgmem ? "Y" : "N"); 6734 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6735 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6736 return 0; 6737 } 6738 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6739 6740 /* Use network device events to rename debugfs file entries. 6741 */ 6742 static int stmmac_device_event(struct notifier_block *unused, 6743 unsigned long event, void *ptr) 6744 { 6745 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6746 struct stmmac_priv *priv = netdev_priv(dev); 6747 6748 if (dev->netdev_ops != &stmmac_netdev_ops) 6749 goto done; 6750 6751 switch (event) { 6752 case NETDEV_CHANGENAME: 6753 debugfs_change_name(priv->dbgfs_dir, "%s", dev->name); 6754 break; 6755 } 6756 done: 6757 return NOTIFY_DONE; 6758 } 6759 6760 static struct notifier_block stmmac_notifier = { 6761 .notifier_call = stmmac_device_event, 6762 }; 6763 6764 static void stmmac_init_fs(struct net_device *dev) 6765 { 6766 struct stmmac_priv *priv = netdev_priv(dev); 6767 6768 rtnl_lock(); 6769 6770 /* Create per netdev entries */ 6771 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6772 6773 /* Entry to report DMA RX/TX rings */ 6774 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6775 &stmmac_rings_status_fops); 6776 6777 /* Entry to report the DMA HW features */ 6778 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6779 &stmmac_dma_cap_fops); 6780 6781 rtnl_unlock(); 6782 } 6783 6784 static void stmmac_exit_fs(struct net_device *dev) 6785 { 6786 struct stmmac_priv *priv = netdev_priv(dev); 6787 6788 debugfs_remove_recursive(priv->dbgfs_dir); 6789 } 6790 #endif /* CONFIG_DEBUG_FS */ 6791 6792 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6793 { 6794 unsigned char *data = (unsigned char *)&vid_le; 6795 unsigned char data_byte = 0; 6796 u32 crc = ~0x0; 6797 u32 temp = 0; 6798 int i, bits; 6799 6800 bits = get_bitmask_order(VLAN_VID_MASK); 6801 for (i = 0; i < bits; i++) { 6802 if ((i % 8) == 0) 6803 data_byte = data[i / 8]; 6804 6805 temp = ((crc & 1) ^ data_byte) & 1; 6806 crc >>= 1; 6807 data_byte >>= 1; 6808 6809 if (temp) 6810 crc ^= 0xedb88320; 6811 } 6812 6813 return crc; 6814 } 6815 6816 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6817 { 6818 u32 crc, hash = 0; 6819 u16 pmatch = 0; 6820 int count = 0; 6821 u16 vid = 0; 6822 6823 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6824 __le16 vid_le = cpu_to_le16(vid); 6825 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6826 hash |= (1 << crc); 6827 count++; 6828 } 6829 6830 if (!priv->dma_cap.vlhash) { 6831 if (count > 2) /* VID = 0 always passes filter */ 6832 return -EOPNOTSUPP; 6833 6834 pmatch = vid; 6835 hash = 0; 6836 } 6837 6838 if (!netif_running(priv->dev)) 6839 return 0; 6840 6841 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6842 } 6843 6844 /* FIXME: This may need RXC to be running, but it may be called with BH 6845 * disabled, which means we can't call phylink_rx_clk_stop*(). 6846 */ 6847 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6848 { 6849 struct stmmac_priv *priv = netdev_priv(ndev); 6850 unsigned int num_double_vlans; 6851 bool is_double = false; 6852 int ret; 6853 6854 ret = pm_runtime_resume_and_get(priv->device); 6855 if (ret < 0) 6856 return ret; 6857 6858 if (be16_to_cpu(proto) == ETH_P_8021AD) 6859 is_double = true; 6860 6861 set_bit(vid, priv->active_vlans); 6862 num_double_vlans = priv->num_double_vlans + is_double; 6863 ret = stmmac_vlan_update(priv, num_double_vlans); 6864 if (ret) { 6865 clear_bit(vid, priv->active_vlans); 6866 goto err_pm_put; 6867 } 6868 6869 if (priv->hw->num_vlan) { 6870 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6871 if (ret) { 6872 clear_bit(vid, priv->active_vlans); 6873 stmmac_vlan_update(priv, priv->num_double_vlans); 6874 goto err_pm_put; 6875 } 6876 } 6877 6878 priv->num_double_vlans = num_double_vlans; 6879 6880 err_pm_put: 6881 pm_runtime_put(priv->device); 6882 6883 return ret; 6884 } 6885 6886 /* FIXME: This may need RXC to be running, but it may be called with BH 6887 * disabled, which means we can't call phylink_rx_clk_stop*(). 6888 */ 6889 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6890 { 6891 struct stmmac_priv *priv = netdev_priv(ndev); 6892 unsigned int num_double_vlans; 6893 bool is_double = false; 6894 int ret; 6895 6896 ret = pm_runtime_resume_and_get(priv->device); 6897 if (ret < 0) 6898 return ret; 6899 6900 if (be16_to_cpu(proto) == ETH_P_8021AD) 6901 is_double = true; 6902 6903 clear_bit(vid, priv->active_vlans); 6904 num_double_vlans = priv->num_double_vlans - is_double; 6905 ret = stmmac_vlan_update(priv, num_double_vlans); 6906 if (ret) { 6907 set_bit(vid, priv->active_vlans); 6908 goto del_vlan_error; 6909 } 6910 6911 if (priv->hw->num_vlan) { 6912 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6913 if (ret) { 6914 set_bit(vid, priv->active_vlans); 6915 stmmac_vlan_update(priv, priv->num_double_vlans); 6916 goto del_vlan_error; 6917 } 6918 } 6919 6920 priv->num_double_vlans = num_double_vlans; 6921 6922 del_vlan_error: 6923 pm_runtime_put(priv->device); 6924 6925 return ret; 6926 } 6927 6928 static void stmmac_vlan_restore(struct stmmac_priv *priv) 6929 { 6930 if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) 6931 return; 6932 6933 if (priv->hw->num_vlan) 6934 stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); 6935 6936 stmmac_vlan_update(priv, priv->num_double_vlans); 6937 } 6938 6939 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6940 { 6941 struct stmmac_priv *priv = netdev_priv(dev); 6942 6943 switch (bpf->command) { 6944 case XDP_SETUP_PROG: 6945 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6946 case XDP_SETUP_XSK_POOL: 6947 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6948 bpf->xsk.queue_id); 6949 default: 6950 return -EOPNOTSUPP; 6951 } 6952 } 6953 6954 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6955 struct xdp_frame **frames, u32 flags) 6956 { 6957 struct stmmac_priv *priv = netdev_priv(dev); 6958 int cpu = smp_processor_id(); 6959 struct netdev_queue *nq; 6960 int i, nxmit = 0; 6961 int queue; 6962 6963 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6964 return -ENETDOWN; 6965 6966 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6967 return -EINVAL; 6968 6969 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6970 nq = netdev_get_tx_queue(priv->dev, queue); 6971 6972 __netif_tx_lock(nq, cpu); 6973 /* Avoids TX time-out as we are sharing with slow path */ 6974 txq_trans_cond_update(nq); 6975 6976 for (i = 0; i < num_frames; i++) { 6977 int res; 6978 6979 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6980 if (res == STMMAC_XDP_CONSUMED) 6981 break; 6982 6983 nxmit++; 6984 } 6985 6986 if (flags & XDP_XMIT_FLUSH) { 6987 stmmac_flush_tx_descriptors(priv, queue); 6988 stmmac_tx_timer_arm(priv, queue); 6989 } 6990 6991 __netif_tx_unlock(nq); 6992 6993 return nxmit; 6994 } 6995 6996 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6997 { 6998 struct stmmac_channel *ch = &priv->channel[queue]; 6999 unsigned long flags; 7000 7001 spin_lock_irqsave(&ch->lock, flags); 7002 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7003 spin_unlock_irqrestore(&ch->lock, flags); 7004 7005 stmmac_stop_rx_dma(priv, queue); 7006 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7007 } 7008 7009 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 7010 { 7011 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7012 struct stmmac_channel *ch = &priv->channel[queue]; 7013 unsigned long flags; 7014 int ret; 7015 7016 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7017 if (ret) { 7018 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 7019 return; 7020 } 7021 7022 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 7023 if (ret) { 7024 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7025 netdev_err(priv->dev, "Failed to init RX desc.\n"); 7026 return; 7027 } 7028 7029 stmmac_reset_rx_queue(priv, queue); 7030 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 7031 7032 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7033 rx_q->dma_rx_phy, queue); 7034 7035 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->buf_alloc_num); 7036 7037 stmmac_set_queue_rx_buf_size(priv, rx_q, queue); 7038 7039 stmmac_start_rx_dma(priv, queue); 7040 7041 spin_lock_irqsave(&ch->lock, flags); 7042 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7043 spin_unlock_irqrestore(&ch->lock, flags); 7044 } 7045 7046 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 7047 { 7048 struct stmmac_channel *ch = &priv->channel[queue]; 7049 unsigned long flags; 7050 7051 spin_lock_irqsave(&ch->lock, flags); 7052 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7053 spin_unlock_irqrestore(&ch->lock, flags); 7054 7055 stmmac_stop_tx_dma(priv, queue); 7056 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7057 } 7058 7059 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 7060 { 7061 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7062 struct stmmac_channel *ch = &priv->channel[queue]; 7063 unsigned long flags; 7064 int ret; 7065 7066 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7067 if (ret) { 7068 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 7069 return; 7070 } 7071 7072 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 7073 if (ret) { 7074 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7075 netdev_err(priv->dev, "Failed to init TX desc.\n"); 7076 return; 7077 } 7078 7079 stmmac_reset_tx_queue(priv, queue); 7080 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 7081 7082 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7083 tx_q->dma_tx_phy, queue); 7084 7085 if (tx_q->tbs & STMMAC_TBS_AVAIL) 7086 stmmac_enable_tbs(priv, priv->ioaddr, 1, queue); 7087 7088 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, 0); 7089 7090 stmmac_start_tx_dma(priv, queue); 7091 7092 spin_lock_irqsave(&ch->lock, flags); 7093 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7094 spin_unlock_irqrestore(&ch->lock, flags); 7095 } 7096 7097 void stmmac_xdp_release(struct net_device *dev) 7098 { 7099 struct stmmac_priv *priv = netdev_priv(dev); 7100 u8 chan; 7101 7102 /* Ensure tx function is not running */ 7103 netif_tx_disable(dev); 7104 7105 /* Disable NAPI process */ 7106 stmmac_disable_all_queues(priv); 7107 7108 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7109 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7110 7111 /* Free the IRQ lines */ 7112 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 7113 7114 /* Stop TX/RX DMA channels */ 7115 stmmac_stop_all_dma(priv); 7116 7117 /* Release and free the Rx/Tx resources */ 7118 free_dma_desc_resources(priv, &priv->dma_conf); 7119 7120 /* Disable the MAC Rx/Tx */ 7121 stmmac_mac_set(priv, priv->ioaddr, false); 7122 7123 /* set trans_start so we don't get spurious 7124 * watchdogs during reset 7125 */ 7126 netif_trans_update(dev); 7127 netif_carrier_off(dev); 7128 } 7129 7130 int stmmac_xdp_open(struct net_device *dev) 7131 { 7132 struct stmmac_priv *priv = netdev_priv(dev); 7133 u8 rx_cnt = priv->plat->rx_queues_to_use; 7134 u8 tx_cnt = priv->plat->tx_queues_to_use; 7135 u8 dma_csr_ch = max(rx_cnt, tx_cnt); 7136 struct stmmac_rx_queue *rx_q; 7137 struct stmmac_tx_queue *tx_q; 7138 bool sph_en; 7139 u8 chan; 7140 int ret; 7141 7142 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 7143 if (ret < 0) { 7144 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 7145 __func__); 7146 goto dma_desc_error; 7147 } 7148 7149 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 7150 if (ret < 0) { 7151 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 7152 __func__); 7153 goto init_error; 7154 } 7155 7156 stmmac_reset_queues_param(priv); 7157 7158 /* DMA CSR Channel configuration */ 7159 for (chan = 0; chan < dma_csr_ch; chan++) { 7160 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 7161 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 7162 } 7163 7164 /* Adjust Split header */ 7165 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 7166 7167 /* DMA RX Channel Configuration */ 7168 for (chan = 0; chan < rx_cnt; chan++) { 7169 rx_q = &priv->dma_conf.rx_queue[chan]; 7170 7171 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7172 rx_q->dma_rx_phy, chan); 7173 7174 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 7175 rx_q->buf_alloc_num); 7176 7177 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 7178 7179 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 7180 } 7181 7182 /* DMA TX Channel Configuration */ 7183 for (chan = 0; chan < tx_cnt; chan++) { 7184 tx_q = &priv->dma_conf.tx_queue[chan]; 7185 7186 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7187 tx_q->dma_tx_phy, chan); 7188 7189 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 7190 7191 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 7192 } 7193 7194 /* Enable the MAC Rx/Tx */ 7195 stmmac_mac_set(priv, priv->ioaddr, true); 7196 7197 /* Start Rx & Tx DMA Channels */ 7198 stmmac_start_all_dma(priv); 7199 7200 ret = stmmac_request_irq(dev); 7201 if (ret) 7202 goto irq_error; 7203 7204 /* Enable NAPI process*/ 7205 stmmac_enable_all_queues(priv); 7206 netif_carrier_on(dev); 7207 netif_tx_start_all_queues(dev); 7208 stmmac_enable_all_dma_irq(priv); 7209 7210 return 0; 7211 7212 irq_error: 7213 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7214 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7215 7216 init_error: 7217 free_dma_desc_resources(priv, &priv->dma_conf); 7218 dma_desc_error: 7219 return ret; 7220 } 7221 7222 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 7223 { 7224 struct stmmac_priv *priv = netdev_priv(dev); 7225 struct stmmac_rx_queue *rx_q; 7226 struct stmmac_tx_queue *tx_q; 7227 struct stmmac_channel *ch; 7228 7229 if (test_bit(STMMAC_DOWN, &priv->state) || 7230 !netif_carrier_ok(priv->dev)) 7231 return -ENETDOWN; 7232 7233 if (!stmmac_xdp_is_enabled(priv)) 7234 return -EINVAL; 7235 7236 if (queue >= priv->plat->rx_queues_to_use || 7237 queue >= priv->plat->tx_queues_to_use) 7238 return -EINVAL; 7239 7240 rx_q = &priv->dma_conf.rx_queue[queue]; 7241 tx_q = &priv->dma_conf.tx_queue[queue]; 7242 ch = &priv->channel[queue]; 7243 7244 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 7245 return -EINVAL; 7246 7247 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 7248 /* EQoS does not have per-DMA channel SW interrupt, 7249 * so we schedule RX Napi straight-away. 7250 */ 7251 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 7252 __napi_schedule(&ch->rxtx_napi); 7253 } 7254 7255 return 0; 7256 } 7257 7258 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7259 { 7260 struct stmmac_priv *priv = netdev_priv(dev); 7261 u8 tx_cnt = priv->plat->tx_queues_to_use; 7262 u8 rx_cnt = priv->plat->rx_queues_to_use; 7263 unsigned int start; 7264 u8 q; 7265 7266 for (q = 0; q < tx_cnt; q++) { 7267 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7268 u64 tx_packets; 7269 u64 tx_bytes; 7270 7271 do { 7272 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 7273 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 7274 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 7275 do { 7276 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 7277 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 7278 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 7279 7280 stats->tx_packets += tx_packets; 7281 stats->tx_bytes += tx_bytes; 7282 } 7283 7284 for (q = 0; q < rx_cnt; q++) { 7285 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7286 u64 rx_packets; 7287 u64 rx_bytes; 7288 7289 do { 7290 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 7291 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 7292 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 7293 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 7294 7295 stats->rx_packets += rx_packets; 7296 stats->rx_bytes += rx_bytes; 7297 } 7298 7299 stats->rx_dropped = priv->xstats.rx_dropped; 7300 stats->rx_errors = priv->xstats.rx_errors; 7301 stats->tx_dropped = priv->xstats.tx_dropped; 7302 stats->tx_errors = priv->xstats.tx_errors; 7303 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7304 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7305 stats->rx_length_errors = priv->xstats.rx_length; 7306 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7307 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7308 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7309 } 7310 7311 static const struct net_device_ops stmmac_netdev_ops = { 7312 .ndo_open = stmmac_open, 7313 .ndo_start_xmit = stmmac_xmit, 7314 .ndo_features_check = stmmac_features_check, 7315 .ndo_stop = stmmac_release, 7316 .ndo_change_mtu = stmmac_change_mtu, 7317 .ndo_fix_features = stmmac_fix_features, 7318 .ndo_set_features = stmmac_set_features, 7319 .ndo_set_rx_mode = stmmac_set_rx_mode, 7320 .ndo_tx_timeout = stmmac_tx_timeout, 7321 .ndo_eth_ioctl = stmmac_ioctl, 7322 .ndo_get_stats64 = stmmac_get_stats64, 7323 .ndo_setup_tc = stmmac_setup_tc, 7324 .ndo_select_queue = stmmac_select_queue, 7325 .ndo_set_mac_address = stmmac_set_mac_address, 7326 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7327 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7328 .ndo_bpf = stmmac_bpf, 7329 .ndo_xdp_xmit = stmmac_xdp_xmit, 7330 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7331 .ndo_hwtstamp_get = stmmac_hwtstamp_get, 7332 .ndo_hwtstamp_set = stmmac_hwtstamp_set, 7333 }; 7334 7335 static void stmmac_reset_subtask(struct stmmac_priv *priv) 7336 { 7337 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7338 return; 7339 if (test_bit(STMMAC_DOWN, &priv->state)) 7340 return; 7341 7342 netdev_err(priv->dev, "Reset adapter.\n"); 7343 7344 rtnl_lock(); 7345 netif_trans_update(priv->dev); 7346 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7347 usleep_range(1000, 2000); 7348 7349 set_bit(STMMAC_DOWN, &priv->state); 7350 dev_close(priv->dev); 7351 dev_open(priv->dev, NULL); 7352 clear_bit(STMMAC_DOWN, &priv->state); 7353 clear_bit(STMMAC_RESETING, &priv->state); 7354 rtnl_unlock(); 7355 } 7356 7357 static void stmmac_service_task(struct work_struct *work) 7358 { 7359 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7360 service_task); 7361 7362 stmmac_reset_subtask(priv); 7363 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7364 } 7365 7366 static void stmmac_print_actphyif(struct stmmac_priv *priv) 7367 { 7368 const char **phyif_table; 7369 const char *actphyif_str; 7370 size_t phyif_table_size; 7371 7372 switch (priv->plat->core_type) { 7373 case DWMAC_CORE_MAC100: 7374 return; 7375 7376 case DWMAC_CORE_GMAC: 7377 case DWMAC_CORE_GMAC4: 7378 phyif_table = stmmac_dwmac_actphyif; 7379 phyif_table_size = ARRAY_SIZE(stmmac_dwmac_actphyif); 7380 break; 7381 7382 case DWMAC_CORE_XGMAC: 7383 phyif_table = stmmac_dwxgmac_phyif; 7384 phyif_table_size = ARRAY_SIZE(stmmac_dwxgmac_phyif); 7385 break; 7386 } 7387 7388 if (priv->dma_cap.actphyif < phyif_table_size) 7389 actphyif_str = phyif_table[priv->dma_cap.actphyif]; 7390 else 7391 actphyif_str = NULL; 7392 7393 if (!actphyif_str) 7394 actphyif_str = "unknown"; 7395 7396 dev_info(priv->device, "Active PHY interface: %s (%u)\n", 7397 actphyif_str, priv->dma_cap.actphyif); 7398 } 7399 7400 /** 7401 * stmmac_hw_init - Init the MAC device 7402 * @priv: driver private structure 7403 * Description: this function is to configure the MAC device according to 7404 * some platform parameters or the HW capability register. It prepares the 7405 * driver to use either ring or chain modes and to setup either enhanced or 7406 * normal descriptors. 7407 */ 7408 static int stmmac_hw_init(struct stmmac_priv *priv) 7409 { 7410 int ret; 7411 7412 /* dwmac-sun8i only work in chain mode */ 7413 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7414 chain_mode = 1; 7415 priv->chain_mode = !!chain_mode; 7416 7417 /* Initialize HW Interface */ 7418 ret = stmmac_hwif_init(priv); 7419 if (ret) 7420 return ret; 7421 7422 /* Get the HW capability (new GMAC newer than 3.50a) */ 7423 priv->hw_cap_support = stmmac_get_hw_features(priv); 7424 if (priv->hw_cap_support) { 7425 dev_info(priv->device, "DMA HW capability register supported\n"); 7426 7427 /* We can override some gmac/dma configuration fields: e.g. 7428 * enh_desc, tx_coe (e.g. that are passed through the 7429 * platform) with the values from the HW capability 7430 * register (if supported). 7431 */ 7432 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7433 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7434 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7435 if (priv->dma_cap.hash_tb_sz) { 7436 priv->hw->multicast_filter_bins = 7437 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7438 priv->hw->mcast_bits_log2 = 7439 ilog2(priv->hw->multicast_filter_bins); 7440 } 7441 7442 /* TXCOE doesn't work in thresh DMA mode */ 7443 if (priv->plat->force_thresh_dma_mode) 7444 priv->plat->tx_coe = false; 7445 else 7446 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7447 7448 /* In case of GMAC4 rx_coe is from HW cap register. */ 7449 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7450 7451 if (priv->dma_cap.rx_coe_type2) 7452 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7453 else if (priv->dma_cap.rx_coe_type1) 7454 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7455 7456 stmmac_print_actphyif(priv); 7457 } else { 7458 dev_info(priv->device, "No HW DMA feature register supported\n"); 7459 } 7460 7461 if (priv->plat->rx_coe) { 7462 priv->hw->rx_csum = priv->plat->rx_coe; 7463 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7464 if (priv->synopsys_id < DWMAC_CORE_4_00) 7465 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7466 } 7467 if (priv->plat->tx_coe) 7468 dev_info(priv->device, "TX Checksum insertion supported\n"); 7469 7470 if (priv->plat->pmt) { 7471 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7472 device_set_wakeup_capable(priv->device, 1); 7473 devm_pm_set_wake_irq(priv->device, priv->wol_irq); 7474 } 7475 7476 if (priv->dma_cap.number_rx_queues && 7477 priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) { 7478 dev_warn(priv->device, 7479 "Number of Rx queues (%u) exceeds dma capability\n", 7480 priv->plat->rx_queues_to_use); 7481 priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues; 7482 } 7483 if (priv->dma_cap.number_tx_queues && 7484 priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) { 7485 dev_warn(priv->device, 7486 "Number of Tx queues (%u) exceeds dma capability\n", 7487 priv->plat->tx_queues_to_use); 7488 priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues; 7489 } 7490 7491 if (priv->dma_cap.rx_fifo_size && 7492 priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) { 7493 dev_warn(priv->device, 7494 "Rx FIFO size (%u) exceeds dma capability\n", 7495 priv->plat->rx_fifo_size); 7496 priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size; 7497 } 7498 if (priv->dma_cap.tx_fifo_size && 7499 priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) { 7500 dev_warn(priv->device, 7501 "Tx FIFO size (%u) exceeds dma capability\n", 7502 priv->plat->tx_fifo_size); 7503 priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size; 7504 } 7505 7506 priv->hw->vlan_fail_q_en = 7507 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7508 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7509 7510 /* Run HW quirks, if any */ 7511 if (priv->hwif_quirks) { 7512 ret = priv->hwif_quirks(priv); 7513 if (ret) 7514 return ret; 7515 } 7516 7517 /* Set alternate descriptor size (which tells the hardware that 7518 * descriptors are 8 32-bit words) when using extended descriptors 7519 * with ring mode. Only applicable for pre-v4.0 cores. Platform glue 7520 * is not expected to change this. 7521 */ 7522 priv->plat->dma_cfg->atds = priv->extend_desc && 7523 priv->descriptor_mode == STMMAC_RING_MODE; 7524 7525 /* Rx Watchdog is available in the COREs newer than the 3.40. 7526 * In some case, for example on bugged HW this feature 7527 * has to be disable and this can be done by passing the 7528 * riwt_off field from the platform. 7529 */ 7530 if ((priv->synopsys_id >= DWMAC_CORE_3_50 || 7531 priv->plat->core_type == DWMAC_CORE_XGMAC) && 7532 !priv->plat->riwt_off) { 7533 priv->use_riwt = 1; 7534 dev_info(priv->device, 7535 "Enable RX Mitigation via HW Watchdog Timer\n"); 7536 } 7537 7538 /* Unimplemented PCS init (as indicated by stmmac_do_callback() 7539 * perversely returning -EINVAL) is non-fatal. 7540 */ 7541 ret = stmmac_mac_pcs_init(priv); 7542 if (ret != -EINVAL) 7543 return ret; 7544 7545 return 0; 7546 } 7547 7548 static void stmmac_napi_add(struct net_device *dev) 7549 { 7550 struct stmmac_priv *priv = netdev_priv(dev); 7551 u8 queue, maxq; 7552 7553 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7554 7555 for (queue = 0; queue < maxq; queue++) { 7556 struct stmmac_channel *ch = &priv->channel[queue]; 7557 7558 ch->priv_data = priv; 7559 ch->index = queue; 7560 spin_lock_init(&ch->lock); 7561 7562 if (queue < priv->plat->rx_queues_to_use) { 7563 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7564 } 7565 if (queue < priv->plat->tx_queues_to_use) { 7566 netif_napi_add_tx(dev, &ch->tx_napi, 7567 stmmac_napi_poll_tx); 7568 } 7569 if (queue < priv->plat->rx_queues_to_use && 7570 queue < priv->plat->tx_queues_to_use) { 7571 netif_napi_add(dev, &ch->rxtx_napi, 7572 stmmac_napi_poll_rxtx); 7573 } 7574 } 7575 } 7576 7577 static void stmmac_napi_del(struct net_device *dev) 7578 { 7579 struct stmmac_priv *priv = netdev_priv(dev); 7580 u8 queue, maxq; 7581 7582 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7583 7584 for (queue = 0; queue < maxq; queue++) { 7585 struct stmmac_channel *ch = &priv->channel[queue]; 7586 7587 if (queue < priv->plat->rx_queues_to_use) 7588 netif_napi_del(&ch->rx_napi); 7589 if (queue < priv->plat->tx_queues_to_use) 7590 netif_napi_del(&ch->tx_napi); 7591 if (queue < priv->plat->rx_queues_to_use && 7592 queue < priv->plat->tx_queues_to_use) { 7593 netif_napi_del(&ch->rxtx_napi); 7594 } 7595 } 7596 } 7597 7598 int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt) 7599 { 7600 struct stmmac_priv *priv = netdev_priv(dev); 7601 int ret = 0, i; 7602 7603 if (netif_running(dev)) 7604 stmmac_release(dev); 7605 7606 stmmac_napi_del(dev); 7607 7608 priv->plat->rx_queues_to_use = rx_cnt; 7609 priv->plat->tx_queues_to_use = tx_cnt; 7610 if (!netif_is_rxfh_configured(dev)) 7611 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7612 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7613 rx_cnt); 7614 7615 stmmac_napi_add(dev); 7616 7617 if (netif_running(dev)) 7618 ret = stmmac_open(dev); 7619 7620 return ret; 7621 } 7622 7623 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7624 { 7625 struct stmmac_priv *priv = netdev_priv(dev); 7626 int ret = 0; 7627 7628 if (netif_running(dev)) 7629 stmmac_release(dev); 7630 7631 priv->dma_conf.dma_rx_size = rx_size; 7632 priv->dma_conf.dma_tx_size = tx_size; 7633 7634 if (netif_running(dev)) 7635 ret = stmmac_open(dev); 7636 7637 return ret; 7638 } 7639 7640 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7641 { 7642 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7643 struct dma_desc *desc_contains_ts = ctx->desc; 7644 struct stmmac_priv *priv = ctx->priv; 7645 struct dma_desc *ndesc = ctx->ndesc; 7646 struct dma_desc *desc = ctx->desc; 7647 u64 ns = 0; 7648 7649 if (!priv->hwts_rx_en) 7650 return -ENODATA; 7651 7652 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7653 if (dwmac_is_xmac(priv->plat->core_type)) 7654 desc_contains_ts = ndesc; 7655 7656 /* Check if timestamp is available */ 7657 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7658 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7659 ns -= priv->plat->cdc_error_adj; 7660 *timestamp = ns_to_ktime(ns); 7661 return 0; 7662 } 7663 7664 return -ENODATA; 7665 } 7666 7667 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7668 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7669 }; 7670 7671 static int stmmac_dl_ts_coarse_set(struct devlink *dl, u32 id, 7672 struct devlink_param_gset_ctx *ctx, 7673 struct netlink_ext_ack *extack) 7674 { 7675 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7676 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7677 7678 priv->tsfupdt_coarse = ctx->val.vbool; 7679 7680 if (priv->tsfupdt_coarse) 7681 priv->systime_flags &= ~PTP_TCR_TSCFUPDT; 7682 else 7683 priv->systime_flags |= PTP_TCR_TSCFUPDT; 7684 7685 /* In Coarse mode, we can use a smaller subsecond increment, let's 7686 * reconfigure the systime, subsecond increment and addend. 7687 */ 7688 stmmac_update_subsecond_increment(priv); 7689 7690 return 0; 7691 } 7692 7693 static int stmmac_dl_ts_coarse_get(struct devlink *dl, u32 id, 7694 struct devlink_param_gset_ctx *ctx, 7695 struct netlink_ext_ack *extack) 7696 { 7697 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7698 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7699 7700 ctx->val.vbool = priv->tsfupdt_coarse; 7701 7702 return 0; 7703 } 7704 7705 static const struct devlink_param stmmac_devlink_params[] = { 7706 DEVLINK_PARAM_DRIVER(STMMAC_DEVLINK_PARAM_ID_TS_COARSE, "phc_coarse_adj", 7707 DEVLINK_PARAM_TYPE_BOOL, 7708 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 7709 stmmac_dl_ts_coarse_get, 7710 stmmac_dl_ts_coarse_set, NULL), 7711 }; 7712 7713 /* None of the generic devlink parameters are implemented */ 7714 static const struct devlink_ops stmmac_devlink_ops = {}; 7715 7716 static int stmmac_register_devlink(struct stmmac_priv *priv) 7717 { 7718 struct stmmac_devlink_priv *dl_priv; 7719 int ret; 7720 7721 /* For now, what is exposed over devlink is only relevant when 7722 * timestamping is available and we have a valid ptp clock rate 7723 */ 7724 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp) || 7725 !priv->plat->clk_ptp_rate) 7726 return 0; 7727 7728 priv->devlink = devlink_alloc(&stmmac_devlink_ops, sizeof(*dl_priv), 7729 priv->device); 7730 if (!priv->devlink) 7731 return -ENOMEM; 7732 7733 dl_priv = devlink_priv(priv->devlink); 7734 dl_priv->stmmac_priv = priv; 7735 7736 ret = devlink_params_register(priv->devlink, stmmac_devlink_params, 7737 ARRAY_SIZE(stmmac_devlink_params)); 7738 if (ret) 7739 goto dl_free; 7740 7741 devlink_register(priv->devlink); 7742 return 0; 7743 7744 dl_free: 7745 devlink_free(priv->devlink); 7746 7747 return ret; 7748 } 7749 7750 static void stmmac_unregister_devlink(struct stmmac_priv *priv) 7751 { 7752 if (!priv->devlink) 7753 return; 7754 7755 devlink_unregister(priv->devlink); 7756 devlink_params_unregister(priv->devlink, stmmac_devlink_params, 7757 ARRAY_SIZE(stmmac_devlink_params)); 7758 devlink_free(priv->devlink); 7759 } 7760 7761 struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev) 7762 { 7763 struct plat_stmmacenet_data *plat_dat; 7764 int i; 7765 7766 plat_dat = devm_kzalloc(dev, sizeof(*plat_dat), GFP_KERNEL); 7767 if (!plat_dat) 7768 return NULL; 7769 7770 plat_dat->dma_cfg = &plat_dat->__dma_cfg; 7771 7772 /* Set the defaults: 7773 * - phy autodetection 7774 * - determine GMII_Address CR field from CSR clock 7775 * - allow MTU up to JUMBO_LEN 7776 * - hash table size 7777 * - one unicast filter entry 7778 */ 7779 plat_dat->phy_addr = -1; 7780 plat_dat->clk_csr = -1; 7781 plat_dat->maxmtu = JUMBO_LEN; 7782 plat_dat->multicast_filter_bins = HASH_TABLE_SIZE; 7783 plat_dat->unicast_filter_entries = 1; 7784 7785 /* Set the mtl defaults */ 7786 plat_dat->tx_queues_to_use = 1; 7787 plat_dat->rx_queues_to_use = 1; 7788 7789 /* Setup the default RX queue channel map */ 7790 for (i = 0; i < ARRAY_SIZE(plat_dat->rx_queues_cfg); i++) 7791 plat_dat->rx_queues_cfg[i].chan = i; 7792 7793 return plat_dat; 7794 } 7795 EXPORT_SYMBOL_GPL(stmmac_plat_dat_alloc); 7796 7797 static int __stmmac_dvr_probe(struct device *device, 7798 struct plat_stmmacenet_data *plat_dat, 7799 struct stmmac_resources *res) 7800 { 7801 struct net_device *ndev = NULL; 7802 struct stmmac_priv *priv; 7803 int i, ret = 0; 7804 u8 rxq; 7805 7806 if (!plat_dat->dma_cfg || !plat_dat->dma_cfg->pbl) { 7807 dev_err(device, "invalid DMA configuration\n"); 7808 return -EINVAL; 7809 } 7810 7811 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7812 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7813 if (!ndev) 7814 return -ENOMEM; 7815 7816 SET_NETDEV_DEV(ndev, device); 7817 7818 priv = netdev_priv(ndev); 7819 priv->device = device; 7820 priv->dev = ndev; 7821 7822 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7823 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7824 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7825 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7826 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7827 } 7828 7829 priv->xstats.pcpu_stats = 7830 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7831 if (!priv->xstats.pcpu_stats) 7832 return -ENOMEM; 7833 7834 stmmac_set_ethtool_ops(ndev); 7835 priv->pause_time = pause; 7836 priv->plat = plat_dat; 7837 priv->ioaddr = res->addr; 7838 priv->dev->base_addr = (unsigned long)res->addr; 7839 priv->plat->dma_cfg->multi_msi_en = 7840 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7841 7842 priv->dev->irq = res->irq; 7843 priv->wol_irq = res->wol_irq; 7844 priv->sfty_irq = res->sfty_irq; 7845 7846 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) { 7847 ret = stmmac_msi_init(priv, res); 7848 if (ret) 7849 return ret; 7850 } 7851 7852 if (!is_zero_ether_addr(res->mac)) 7853 eth_hw_addr_set(priv->dev, res->mac); 7854 7855 dev_set_drvdata(device, priv->dev); 7856 7857 /* Verify driver arguments */ 7858 stmmac_verify_args(); 7859 7860 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7861 if (!priv->af_xdp_zc_qps) 7862 return -ENOMEM; 7863 7864 /* Allocate workqueue */ 7865 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7866 if (!priv->wq) { 7867 dev_err(priv->device, "failed to create workqueue\n"); 7868 ret = -ENOMEM; 7869 goto error_wq_init; 7870 } 7871 7872 INIT_WORK(&priv->service_task, stmmac_service_task); 7873 7874 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 7875 7876 /* Override with kernel parameters if supplied XXX CRS XXX 7877 * this needs to have multiple instances 7878 */ 7879 if ((phyaddr >= 0) && (phyaddr <= 31)) 7880 priv->plat->phy_addr = phyaddr; 7881 7882 if (priv->plat->stmmac_rst) { 7883 ret = reset_control_assert(priv->plat->stmmac_rst); 7884 reset_control_deassert(priv->plat->stmmac_rst); 7885 /* Some reset controllers have only reset callback instead of 7886 * assert + deassert callbacks pair. 7887 */ 7888 if (ret == -ENOTSUPP) 7889 reset_control_reset(priv->plat->stmmac_rst); 7890 } 7891 7892 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7893 if (ret == -ENOTSUPP) 7894 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7895 ERR_PTR(ret)); 7896 7897 /* Wait a bit for the reset to take effect */ 7898 udelay(10); 7899 7900 /* Init MAC and get the capabilities */ 7901 ret = stmmac_hw_init(priv); 7902 if (ret) 7903 goto error_hw_init; 7904 7905 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7906 */ 7907 if (priv->synopsys_id < DWMAC_CORE_5_20) 7908 priv->plat->dma_cfg->dche = false; 7909 7910 stmmac_check_ether_addr(priv); 7911 7912 ndev->netdev_ops = &stmmac_netdev_ops; 7913 7914 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7915 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7916 7917 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7918 NETIF_F_RXCSUM; 7919 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7920 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7921 7922 ret = stmmac_tc_init(priv, priv); 7923 if (!ret) { 7924 ndev->hw_features |= NETIF_F_HW_TC; 7925 } 7926 7927 stmmac_set_gso_features(ndev); 7928 7929 if (priv->dma_cap.sphen && 7930 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7931 ndev->hw_features |= NETIF_F_GRO; 7932 priv->sph_capable = true; 7933 priv->sph_active = priv->sph_capable; 7934 dev_info(priv->device, "SPH feature enabled\n"); 7935 } 7936 7937 /* Ideally our host DMA address width is the same as for the 7938 * device. However, it may differ and then we have to use our 7939 * host DMA width for allocation and the device DMA width for 7940 * register handling. 7941 */ 7942 if (priv->plat->host_dma_width) 7943 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7944 else 7945 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7946 7947 if (priv->dma_cap.host_dma_width) { 7948 ret = dma_set_mask_and_coherent(device, 7949 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7950 if (!ret) { 7951 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7952 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7953 7954 /* 7955 * If more than 32 bits can be addressed, make sure to 7956 * enable enhanced addressing mode. 7957 */ 7958 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7959 priv->plat->dma_cfg->eame = true; 7960 } else { 7961 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7962 if (ret) { 7963 dev_err(priv->device, "Failed to set DMA Mask\n"); 7964 goto error_hw_init; 7965 } 7966 7967 priv->dma_cap.host_dma_width = 32; 7968 } 7969 } 7970 7971 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7972 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7973 #ifdef STMMAC_VLAN_TAG_USED 7974 /* Both mac100 and gmac support receive VLAN tag detection */ 7975 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7976 if (dwmac_is_xmac(priv->plat->core_type)) { 7977 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 7978 priv->hw->hw_vlan_en = true; 7979 } 7980 if (priv->dma_cap.vlhash) { 7981 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7982 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7983 } 7984 if (priv->dma_cap.vlins) 7985 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7986 #endif 7987 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7988 7989 priv->xstats.threshold = tc; 7990 7991 /* Initialize RSS */ 7992 rxq = priv->plat->rx_queues_to_use; 7993 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7994 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7995 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7996 7997 if (priv->dma_cap.rssen && priv->plat->rss_en) 7998 ndev->features |= NETIF_F_RXHASH; 7999 8000 ndev->vlan_features |= ndev->features; 8001 8002 /* MTU range: 46 - hw-specific max */ 8003 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 8004 8005 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 8006 ndev->max_mtu = XGMAC_JUMBO_LEN; 8007 else if (priv->plat->enh_desc || priv->synopsys_id >= DWMAC_CORE_4_00) 8008 ndev->max_mtu = JUMBO_LEN; 8009 else 8010 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 8011 8012 /* Warn if the platform's maxmtu is smaller than the minimum MTU, 8013 * otherwise clamp the maximum MTU above to the platform's maxmtu. 8014 */ 8015 if (priv->plat->maxmtu < ndev->min_mtu) 8016 dev_warn(priv->device, 8017 "%s: warning: maxmtu having invalid value (%d)\n", 8018 __func__, priv->plat->maxmtu); 8019 else if (priv->plat->maxmtu < ndev->max_mtu) 8020 ndev->max_mtu = priv->plat->maxmtu; 8021 8022 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 8023 8024 /* Setup channels NAPI */ 8025 stmmac_napi_add(ndev); 8026 8027 mutex_init(&priv->lock); 8028 8029 stmmac_fpe_init(priv); 8030 8031 stmmac_check_pcs_mode(priv); 8032 8033 pm_runtime_get_noresume(device); 8034 pm_runtime_set_active(device); 8035 if (!pm_runtime_enabled(device)) 8036 pm_runtime_enable(device); 8037 8038 ret = stmmac_mdio_register(ndev); 8039 if (ret < 0) { 8040 dev_err_probe(priv->device, ret, 8041 "MDIO bus (id: %d) registration failed\n", 8042 priv->plat->bus_id); 8043 goto error_mdio_register; 8044 } 8045 8046 ret = stmmac_pcs_setup(ndev); 8047 if (ret) 8048 goto error_pcs_setup; 8049 8050 ret = stmmac_phylink_setup(priv); 8051 if (ret) { 8052 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 8053 goto error_phy_setup; 8054 } 8055 8056 ret = stmmac_register_devlink(priv); 8057 if (ret) 8058 goto error_devlink_setup; 8059 8060 ret = register_netdev(ndev); 8061 if (ret) { 8062 dev_err(priv->device, "%s: ERROR %i registering the device\n", 8063 __func__, ret); 8064 goto error_netdev_register; 8065 } 8066 8067 #ifdef CONFIG_DEBUG_FS 8068 stmmac_init_fs(ndev); 8069 #endif 8070 8071 if (priv->plat->dump_debug_regs) 8072 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 8073 8074 /* Let pm_runtime_put() disable the clocks. 8075 * If CONFIG_PM is not enabled, the clocks will stay powered. 8076 */ 8077 pm_runtime_put(device); 8078 8079 return ret; 8080 8081 error_netdev_register: 8082 stmmac_unregister_devlink(priv); 8083 error_devlink_setup: 8084 phylink_destroy(priv->phylink); 8085 error_phy_setup: 8086 stmmac_pcs_clean(ndev); 8087 error_pcs_setup: 8088 stmmac_mdio_unregister(ndev); 8089 error_mdio_register: 8090 stmmac_napi_del(ndev); 8091 error_hw_init: 8092 destroy_workqueue(priv->wq); 8093 error_wq_init: 8094 bitmap_free(priv->af_xdp_zc_qps); 8095 8096 return ret; 8097 } 8098 8099 /** 8100 * stmmac_dvr_probe 8101 * @dev: device pointer 8102 * @plat_dat: platform data pointer 8103 * @res: stmmac resource pointer 8104 * Description: this is the main probe function used to 8105 * call the alloc_etherdev, allocate the priv structure. 8106 * Return: 8107 * returns 0 on success, otherwise errno. 8108 */ 8109 int stmmac_dvr_probe(struct device *dev, struct plat_stmmacenet_data *plat_dat, 8110 struct stmmac_resources *res) 8111 { 8112 int ret; 8113 8114 if (plat_dat->init) { 8115 ret = plat_dat->init(dev, plat_dat->bsp_priv); 8116 if (ret) 8117 return ret; 8118 } 8119 8120 ret = __stmmac_dvr_probe(dev, plat_dat, res); 8121 if (ret && plat_dat->exit) 8122 plat_dat->exit(dev, plat_dat->bsp_priv); 8123 8124 return ret; 8125 } 8126 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 8127 8128 /** 8129 * stmmac_dvr_remove 8130 * @dev: device pointer 8131 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 8132 * changes the link status, releases the DMA descriptor rings. 8133 */ 8134 void stmmac_dvr_remove(struct device *dev) 8135 { 8136 struct net_device *ndev = dev_get_drvdata(dev); 8137 struct stmmac_priv *priv = netdev_priv(ndev); 8138 8139 netdev_info(priv->dev, "%s: removing driver", __func__); 8140 8141 pm_runtime_get_sync(dev); 8142 8143 unregister_netdev(ndev); 8144 8145 #ifdef CONFIG_DEBUG_FS 8146 stmmac_exit_fs(ndev); 8147 #endif 8148 stmmac_unregister_devlink(priv); 8149 8150 phylink_destroy(priv->phylink); 8151 if (priv->plat->stmmac_rst) 8152 reset_control_assert(priv->plat->stmmac_rst); 8153 reset_control_assert(priv->plat->stmmac_ahb_rst); 8154 8155 stmmac_pcs_clean(ndev); 8156 stmmac_mdio_unregister(ndev); 8157 8158 destroy_workqueue(priv->wq); 8159 mutex_destroy(&priv->lock); 8160 bitmap_free(priv->af_xdp_zc_qps); 8161 8162 pm_runtime_disable(dev); 8163 pm_runtime_put_noidle(dev); 8164 8165 if (priv->plat->exit) 8166 priv->plat->exit(dev, priv->plat->bsp_priv); 8167 } 8168 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 8169 8170 /** 8171 * stmmac_suspend - suspend callback 8172 * @dev: device pointer 8173 * Description: this is the function to suspend the device and it is called 8174 * by the platform driver to stop the network queue, release the resources, 8175 * program the PMT register (for WoL), clean and release driver resources. 8176 */ 8177 int stmmac_suspend(struct device *dev) 8178 { 8179 struct net_device *ndev = dev_get_drvdata(dev); 8180 struct stmmac_priv *priv = netdev_priv(ndev); 8181 u8 chan; 8182 8183 if (!ndev || !netif_running(ndev)) 8184 goto suspend_bsp; 8185 8186 mutex_lock(&priv->lock); 8187 8188 netif_device_detach(ndev); 8189 8190 stmmac_disable_all_queues(priv); 8191 8192 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 8193 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 8194 8195 if (priv->eee_sw_timer_en) { 8196 priv->tx_path_in_lpi_mode = false; 8197 timer_delete_sync(&priv->eee_ctrl_timer); 8198 } 8199 8200 /* Stop TX/RX DMA */ 8201 stmmac_stop_all_dma(priv); 8202 8203 stmmac_legacy_serdes_power_down(priv); 8204 8205 /* Enable Power down mode by programming the PMT regs */ 8206 if (priv->wolopts) { 8207 stmmac_pmt(priv, priv->hw, priv->wolopts); 8208 priv->irq_wake = 1; 8209 } else { 8210 stmmac_mac_set(priv, priv->ioaddr, false); 8211 pinctrl_pm_select_sleep_state(priv->device); 8212 } 8213 8214 mutex_unlock(&priv->lock); 8215 8216 rtnl_lock(); 8217 phylink_suspend(priv->phylink, !!priv->wolopts); 8218 rtnl_unlock(); 8219 8220 if (stmmac_fpe_supported(priv)) 8221 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 8222 8223 suspend_bsp: 8224 if (priv->plat->suspend) 8225 return priv->plat->suspend(dev, priv->plat->bsp_priv); 8226 8227 return 0; 8228 } 8229 EXPORT_SYMBOL_GPL(stmmac_suspend); 8230 8231 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 8232 { 8233 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 8234 8235 rx_q->cur_rx = 0; 8236 rx_q->dirty_rx = 0; 8237 } 8238 8239 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 8240 { 8241 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 8242 8243 tx_q->cur_tx = 0; 8244 tx_q->dirty_tx = 0; 8245 tx_q->mss = 0; 8246 8247 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 8248 } 8249 8250 /** 8251 * stmmac_reset_queues_param - reset queue parameters 8252 * @priv: device pointer 8253 */ 8254 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 8255 { 8256 u8 rx_cnt = priv->plat->rx_queues_to_use; 8257 u8 tx_cnt = priv->plat->tx_queues_to_use; 8258 u8 queue; 8259 8260 for (queue = 0; queue < rx_cnt; queue++) 8261 stmmac_reset_rx_queue(priv, queue); 8262 8263 for (queue = 0; queue < tx_cnt; queue++) 8264 stmmac_reset_tx_queue(priv, queue); 8265 } 8266 8267 /** 8268 * stmmac_resume - resume callback 8269 * @dev: device pointer 8270 * Description: when resume this function is invoked to setup the DMA and CORE 8271 * in a usable state. 8272 */ 8273 int stmmac_resume(struct device *dev) 8274 { 8275 struct net_device *ndev = dev_get_drvdata(dev); 8276 struct stmmac_priv *priv = netdev_priv(ndev); 8277 int ret; 8278 8279 if (priv->plat->resume) { 8280 ret = priv->plat->resume(dev, priv->plat->bsp_priv); 8281 if (ret) 8282 return ret; 8283 } 8284 8285 if (!netif_running(ndev)) 8286 return 0; 8287 8288 /* Power Down bit, into the PM register, is cleared 8289 * automatically as soon as a magic packet or a Wake-up frame 8290 * is received. Anyway, it's better to manually clear 8291 * this bit because it can generate problems while resuming 8292 * from another devices (e.g. serial console). 8293 */ 8294 if (priv->wolopts) { 8295 mutex_lock(&priv->lock); 8296 stmmac_pmt(priv, priv->hw, 0); 8297 mutex_unlock(&priv->lock); 8298 priv->irq_wake = 0; 8299 } else { 8300 pinctrl_pm_select_default_state(priv->device); 8301 /* reset the phy so that it's ready */ 8302 if (priv->mii) 8303 stmmac_mdio_reset(priv->mii); 8304 } 8305 8306 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 8307 ret = stmmac_legacy_serdes_power_up(priv); 8308 if (ret < 0) 8309 return ret; 8310 } 8311 8312 rtnl_lock(); 8313 8314 /* Prepare the PHY to resume, ensuring that its clocks which are 8315 * necessary for the MAC DMA reset to complete are running 8316 */ 8317 phylink_prepare_resume(priv->phylink); 8318 8319 mutex_lock(&priv->lock); 8320 8321 stmmac_reset_queues_param(priv); 8322 8323 stmmac_free_tx_skbufs(priv); 8324 stmmac_clear_descriptors(priv, &priv->dma_conf); 8325 8326 ret = stmmac_hw_setup(ndev); 8327 if (ret < 0) { 8328 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 8329 stmmac_legacy_serdes_power_down(priv); 8330 mutex_unlock(&priv->lock); 8331 rtnl_unlock(); 8332 return ret; 8333 } 8334 8335 stmmac_init_timestamping(priv); 8336 8337 stmmac_init_coalesce(priv); 8338 phylink_rx_clk_stop_block(priv->phylink); 8339 stmmac_set_rx_mode(ndev); 8340 phylink_rx_clk_stop_unblock(priv->phylink); 8341 8342 stmmac_vlan_restore(priv); 8343 8344 stmmac_enable_all_queues(priv); 8345 stmmac_enable_all_dma_irq(priv); 8346 8347 mutex_unlock(&priv->lock); 8348 8349 /* phylink_resume() must be called after the hardware has been 8350 * initialised because it may bring the link up immediately in a 8351 * workqueue thread, which will race with initialisation. 8352 */ 8353 phylink_resume(priv->phylink); 8354 rtnl_unlock(); 8355 8356 netif_device_attach(ndev); 8357 8358 return 0; 8359 } 8360 EXPORT_SYMBOL_GPL(stmmac_resume); 8361 8362 /* This is not the same as EXPORT_GPL_SIMPLE_DEV_PM_OPS() when CONFIG_PM=n */ 8363 DEFINE_SIMPLE_DEV_PM_OPS(stmmac_simple_pm_ops, stmmac_suspend, stmmac_resume); 8364 EXPORT_SYMBOL_GPL(stmmac_simple_pm_ops); 8365 8366 #ifndef MODULE 8367 static int __init stmmac_cmdline_opt(char *str) 8368 { 8369 char *opt; 8370 8371 if (!str || !*str) 8372 return 1; 8373 while ((opt = strsep(&str, ",")) != NULL) { 8374 if (!strncmp(opt, "debug:", 6)) { 8375 if (kstrtoint(opt + 6, 0, &debug)) 8376 goto err; 8377 } else if (!strncmp(opt, "phyaddr:", 8)) { 8378 if (kstrtoint(opt + 8, 0, &phyaddr)) 8379 goto err; 8380 } else if (!strncmp(opt, "tc:", 3)) { 8381 if (kstrtoint(opt + 3, 0, &tc)) 8382 goto err; 8383 } else if (!strncmp(opt, "watchdog:", 9)) { 8384 if (kstrtoint(opt + 9, 0, &watchdog)) 8385 goto err; 8386 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 8387 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 8388 goto err; 8389 } else if (!strncmp(opt, "pause:", 6)) { 8390 if (kstrtoint(opt + 6, 0, &pause)) 8391 goto err; 8392 } else if (!strncmp(opt, "eee_timer:", 10)) { 8393 if (kstrtoint(opt + 10, 0, &eee_timer)) 8394 goto err; 8395 } else if (!strncmp(opt, "chain_mode:", 11)) { 8396 if (kstrtoint(opt + 11, 0, &chain_mode)) 8397 goto err; 8398 } 8399 } 8400 return 1; 8401 8402 err: 8403 pr_err("%s: ERROR broken module parameter conversion", __func__); 8404 return 1; 8405 } 8406 8407 __setup("stmmaceth=", stmmac_cmdline_opt); 8408 #endif /* MODULE */ 8409 8410 static int __init stmmac_init(void) 8411 { 8412 #ifdef CONFIG_DEBUG_FS 8413 /* Create debugfs main directory if it doesn't exist yet */ 8414 if (!stmmac_fs_dir) 8415 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 8416 register_netdevice_notifier(&stmmac_notifier); 8417 #endif 8418 8419 return 0; 8420 } 8421 8422 static void __exit stmmac_exit(void) 8423 { 8424 #ifdef CONFIG_DEBUG_FS 8425 unregister_netdevice_notifier(&stmmac_notifier); 8426 debugfs_remove_recursive(stmmac_fs_dir); 8427 #endif 8428 } 8429 8430 module_init(stmmac_init) 8431 module_exit(stmmac_exit) 8432 8433 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 8434 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 8435 MODULE_LICENSE("GPL"); 8436