1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/circ_buf.h> 18 #include <linux/clk.h> 19 #include <linux/kernel.h> 20 #include <linux/interrupt.h> 21 #include <linux/ip.h> 22 #include <linux/tcp.h> 23 #include <linux/skbuff.h> 24 #include <linux/ethtool.h> 25 #include <linux/if_ether.h> 26 #include <linux/crc32.h> 27 #include <linux/mii.h> 28 #include <linux/if.h> 29 #include <linux/if_vlan.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/slab.h> 32 #include <linux/pm_runtime.h> 33 #include <linux/pm_wakeirq.h> 34 #include <linux/prefetch.h> 35 #include <linux/pinctrl/consumer.h> 36 #ifdef CONFIG_DEBUG_FS 37 #include <linux/debugfs.h> 38 #include <linux/seq_file.h> 39 #endif /* CONFIG_DEBUG_FS */ 40 #include <linux/net_tstamp.h> 41 #include <linux/phylink.h> 42 #include <linux/udp.h> 43 #include <linux/bpf_trace.h> 44 #include <net/devlink.h> 45 #include <net/page_pool/helpers.h> 46 #include <net/pkt_cls.h> 47 #include <net/xdp_sock_drv.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac_fpe.h" 50 #include "stmmac.h" 51 #include "stmmac_pcs.h" 52 #include "stmmac_xdp.h" 53 #include <linux/reset.h> 54 #include <linux/of_mdio.h> 55 #include "dwmac1000.h" 56 #include "dwxgmac2.h" 57 #include "hwif.h" 58 59 /* As long as the interface is active, we keep the timestamping counter enabled 60 * with fine resolution and binary rollover. This avoid non-monotonic behavior 61 * (clock jumps) when changing timestamping settings at runtime. 62 */ 63 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCTRLSSR) 64 65 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 66 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 67 68 /* Module parameters */ 69 #define TX_TIMEO 5000 70 static int watchdog = TX_TIMEO; 71 module_param(watchdog, int, 0644); 72 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 73 74 static int debug = -1; 75 module_param(debug, int, 0644); 76 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 77 78 static int phyaddr = -1; 79 module_param(phyaddr, int, 0444); 80 MODULE_PARM_DESC(phyaddr, "Physical device address"); 81 82 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 83 84 /* Limit to make sure XDP TX and slow path can coexist */ 85 #define STMMAC_XSK_TX_BUDGET_MAX 256 86 #define STMMAC_TX_XSK_AVAIL 16 87 #define STMMAC_RX_FILL_BATCH 16 88 89 #define STMMAC_XDP_PASS 0 90 #define STMMAC_XDP_CONSUMED BIT(0) 91 #define STMMAC_XDP_TX BIT(1) 92 #define STMMAC_XDP_REDIRECT BIT(2) 93 #define STMMAC_XSK_CONSUMED BIT(3) 94 95 static int flow_ctrl = 0xdead; 96 module_param(flow_ctrl, int, 0644); 97 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)"); 98 99 static int pause = PAUSE_TIME; 100 module_param(pause, int, 0644); 101 MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)"); 102 103 #define TC_DEFAULT 64 104 static int tc = TC_DEFAULT; 105 module_param(tc, int, 0644); 106 MODULE_PARM_DESC(tc, "DMA threshold control value"); 107 108 /* This is unused */ 109 #define DEFAULT_BUFSIZE 1536 110 static int buf_sz = DEFAULT_BUFSIZE; 111 module_param(buf_sz, int, 0644); 112 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 113 114 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 115 NETIF_MSG_LINK | NETIF_MSG_IFUP | 116 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 117 118 #define STMMAC_DEFAULT_LPI_TIMER 1000 119 static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 120 module_param(eee_timer, uint, 0644); 121 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 122 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 123 124 /* By default the driver will use the ring mode to manage tx and rx descriptors, 125 * but allow user to force to use the chain instead of the ring 126 */ 127 static unsigned int chain_mode; 128 module_param(chain_mode, int, 0444); 129 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 130 131 static const char *stmmac_dwmac_actphyif[8] = { 132 [PHY_INTF_SEL_GMII_MII] = "GMII/MII", 133 [PHY_INTF_SEL_RGMII] = "RGMII", 134 [PHY_INTF_SEL_SGMII] = "SGMII", 135 [PHY_INTF_SEL_TBI] = "TBI", 136 [PHY_INTF_SEL_RMII] = "RMII", 137 [PHY_INTF_SEL_RTBI] = "RTBI", 138 [PHY_INTF_SEL_SMII] = "SMII", 139 [PHY_INTF_SEL_REVMII] = "REVMII", 140 }; 141 142 static const char *stmmac_dwxgmac_phyif[4] = { 143 [PHY_INTF_GMII] = "GMII", 144 [PHY_INTF_RGMII] = "RGMII", 145 }; 146 147 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 148 /* For MSI interrupts handling */ 149 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 150 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 151 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 152 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 153 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 154 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 155 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 156 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 157 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 158 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 159 u32 rxmode, u32 chan); 160 static void stmmac_vlan_restore(struct stmmac_priv *priv); 161 162 #ifdef CONFIG_DEBUG_FS 163 static const struct net_device_ops stmmac_netdev_ops; 164 static void stmmac_init_fs(struct net_device *dev); 165 static void stmmac_exit_fs(struct net_device *dev); 166 #endif 167 168 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 169 170 struct stmmac_devlink_priv { 171 struct stmmac_priv *stmmac_priv; 172 }; 173 174 enum stmmac_dl_param_id { 175 STMMAC_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, 176 STMMAC_DEVLINK_PARAM_ID_TS_COARSE, 177 }; 178 179 /** 180 * stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock 181 * @bsp_priv: BSP private data structure (unused) 182 * @clk_tx_i: the transmit clock 183 * @interface: the selected interface mode 184 * @speed: the speed that the MAC will be operating at 185 * 186 * Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps, 187 * 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least 188 * MII, GMII, RGMII and RMII interface modes. Platforms can hook this into 189 * the plat_data->set_clk_tx_rate method directly, call it via their own 190 * implementation, or implement their own method should they have more 191 * complex requirements. It is intended to only be used in this method. 192 * 193 * plat_data->clk_tx_i must be filled in. 194 */ 195 int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, 196 phy_interface_t interface, int speed) 197 { 198 long rate = rgmii_clock(speed); 199 200 /* Silently ignore unsupported speeds as rgmii_clock() only 201 * supports 10, 100 and 1000Mbps. We do not want to spit 202 * errors for 2500 and higher speeds here. 203 */ 204 if (rate < 0) 205 return 0; 206 207 return clk_set_rate(clk_tx_i, rate); 208 } 209 EXPORT_SYMBOL_GPL(stmmac_set_clk_tx_rate); 210 211 /** 212 * stmmac_axi_blen_to_mask() - convert a burst length array to reg value 213 * @regval: pointer to a u32 for the resulting register value 214 * @blen: pointer to an array of u32 containing the burst length values in bytes 215 * @len: the number of entries in the @blen array 216 */ 217 void stmmac_axi_blen_to_mask(u32 *regval, const u32 *blen, size_t len) 218 { 219 size_t i; 220 u32 val; 221 222 for (val = i = 0; i < len; i++) { 223 u32 burst = blen[i]; 224 225 /* Burst values of zero must be skipped. */ 226 if (!burst) 227 continue; 228 229 /* The valid range for the burst length is 4 to 256 inclusive, 230 * and it must be a power of two. 231 */ 232 if (burst < 4 || burst > 256 || !is_power_of_2(burst)) { 233 pr_err("stmmac: invalid burst length %u at index %zu\n", 234 burst, i); 235 continue; 236 } 237 238 /* Since burst is a power of two, and the register field starts 239 * with burst = 4, shift right by two bits so bit 0 of the field 240 * corresponds with the minimum value. 241 */ 242 val |= burst >> 2; 243 } 244 245 *regval = FIELD_PREP(DMA_AXI_BLEN_MASK, val); 246 } 247 EXPORT_SYMBOL_GPL(stmmac_axi_blen_to_mask); 248 249 /** 250 * stmmac_verify_args - verify the driver parameters. 251 * Description: it checks the driver parameters and set a default in case of 252 * errors. 253 */ 254 static void stmmac_verify_args(void) 255 { 256 if (unlikely(watchdog < 0)) 257 watchdog = TX_TIMEO; 258 if (unlikely((pause < 0) || (pause > 0xffff))) 259 pause = PAUSE_TIME; 260 261 if (flow_ctrl != 0xdead) 262 pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n"); 263 } 264 265 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 266 { 267 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 268 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 269 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 270 u8 queue; 271 272 for (queue = 0; queue < maxq; queue++) { 273 struct stmmac_channel *ch = &priv->channel[queue]; 274 275 if (stmmac_xdp_is_enabled(priv) && 276 test_bit(queue, priv->af_xdp_zc_qps)) { 277 napi_disable(&ch->rxtx_napi); 278 continue; 279 } 280 281 if (queue < rx_queues_cnt) 282 napi_disable(&ch->rx_napi); 283 if (queue < tx_queues_cnt) 284 napi_disable(&ch->tx_napi); 285 } 286 } 287 288 /** 289 * stmmac_disable_all_queues - Disable all queues 290 * @priv: driver private structure 291 */ 292 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 293 { 294 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 295 struct stmmac_rx_queue *rx_q; 296 u8 queue; 297 298 /* synchronize_rcu() needed for pending XDP buffers to drain */ 299 for (queue = 0; queue < rx_queues_cnt; queue++) { 300 rx_q = &priv->dma_conf.rx_queue[queue]; 301 if (rx_q->xsk_pool) { 302 synchronize_rcu(); 303 break; 304 } 305 } 306 307 __stmmac_disable_all_queues(priv); 308 } 309 310 /** 311 * stmmac_enable_all_queues - Enable all queues 312 * @priv: driver private structure 313 */ 314 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 315 { 316 u8 rx_queues_cnt = priv->plat->rx_queues_to_use; 317 u8 tx_queues_cnt = priv->plat->tx_queues_to_use; 318 u8 maxq = max(rx_queues_cnt, tx_queues_cnt); 319 u8 queue; 320 321 for (queue = 0; queue < maxq; queue++) { 322 struct stmmac_channel *ch = &priv->channel[queue]; 323 324 if (stmmac_xdp_is_enabled(priv) && 325 test_bit(queue, priv->af_xdp_zc_qps)) { 326 napi_enable(&ch->rxtx_napi); 327 continue; 328 } 329 330 if (queue < rx_queues_cnt) 331 napi_enable(&ch->rx_napi); 332 if (queue < tx_queues_cnt) 333 napi_enable(&ch->tx_napi); 334 } 335 } 336 337 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 338 { 339 if (!test_bit(STMMAC_DOWN, &priv->state) && 340 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 341 queue_work(priv->wq, &priv->service_task); 342 } 343 344 static void stmmac_global_err(struct stmmac_priv *priv) 345 { 346 netif_carrier_off(priv->dev); 347 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 348 stmmac_service_event_schedule(priv); 349 } 350 351 static void print_pkt(unsigned char *buf, int len) 352 { 353 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 354 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 355 } 356 357 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 358 { 359 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 360 361 return CIRC_SPACE(tx_q->cur_tx, tx_q->dirty_tx, 362 priv->dma_conf.dma_tx_size); 363 } 364 365 static size_t stmmac_get_tx_desc_size(struct stmmac_priv *priv, 366 struct stmmac_tx_queue *tx_q) 367 { 368 if (priv->extend_desc) 369 return sizeof(struct dma_extended_desc); 370 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 371 return sizeof(struct dma_edesc); 372 else 373 return sizeof(struct dma_desc); 374 } 375 376 static struct dma_desc *stmmac_get_tx_desc(struct stmmac_priv *priv, 377 struct stmmac_tx_queue *tx_q, 378 unsigned int index) 379 { 380 if (priv->extend_desc) 381 return &tx_q->dma_etx[index].basic; 382 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 383 return &tx_q->dma_entx[index].basic; 384 else 385 return &tx_q->dma_tx[index]; 386 } 387 388 static void stmmac_set_queue_tx_tail_ptr(struct stmmac_priv *priv, 389 struct stmmac_tx_queue *tx_q, 390 unsigned int chan, unsigned int index) 391 { 392 size_t desc_size; 393 u32 tx_tail_addr; 394 395 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 396 397 tx_tail_addr = tx_q->dma_tx_phy + index * desc_size; 398 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_tail_addr, chan); 399 } 400 401 static size_t stmmac_get_rx_desc_size(struct stmmac_priv *priv) 402 { 403 if (priv->extend_desc) 404 return sizeof(struct dma_extended_desc); 405 else 406 return sizeof(struct dma_desc); 407 } 408 409 static struct dma_desc *stmmac_get_rx_desc(struct stmmac_priv *priv, 410 struct stmmac_rx_queue *rx_q, 411 unsigned int index) 412 { 413 if (priv->extend_desc) 414 return &rx_q->dma_erx[index].basic; 415 else 416 return &rx_q->dma_rx[index]; 417 } 418 419 static void stmmac_set_queue_rx_tail_ptr(struct stmmac_priv *priv, 420 struct stmmac_rx_queue *rx_q, 421 unsigned int chan, unsigned int index) 422 { 423 /* This only needs to deal with normal descriptors as enhanced 424 * descriptiors are only supported with dwmac1000 (<v4.0) which 425 * does not implement .set_rx_tail_ptr 426 */ 427 u32 rx_tail_addr = rx_q->dma_rx_phy + index * sizeof(struct dma_desc); 428 429 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_tail_addr, chan); 430 } 431 432 static void stmmac_set_queue_rx_buf_size(struct stmmac_priv *priv, 433 struct stmmac_rx_queue *rx_q, 434 unsigned int chan) 435 { 436 u32 buf_size; 437 438 if (rx_q->xsk_pool && rx_q->buf_alloc_num) 439 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 440 else 441 buf_size = priv->dma_conf.dma_buf_sz; 442 443 stmmac_set_dma_bfsize(priv, priv->ioaddr, buf_size, chan); 444 } 445 446 /** 447 * stmmac_rx_dirty - Get RX queue dirty 448 * @priv: driver private structure 449 * @queue: RX queue index 450 */ 451 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 452 { 453 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 454 455 return CIRC_CNT(rx_q->cur_rx, rx_q->dirty_rx, 456 priv->dma_conf.dma_rx_size); 457 } 458 459 static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) 460 { 461 u8 tx_cnt = priv->plat->tx_queues_to_use; 462 u8 queue; 463 464 /* check if all TX queues have the work finished */ 465 for (queue = 0; queue < tx_cnt; queue++) { 466 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 467 468 if (tx_q->dirty_tx != tx_q->cur_tx) 469 return true; /* still unfinished work */ 470 } 471 472 return false; 473 } 474 475 static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv) 476 { 477 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 478 } 479 480 /** 481 * stmmac_try_to_start_sw_lpi - check and enter in LPI mode 482 * @priv: driver private structure 483 * Description: this function is to verify and enter in LPI mode in case of 484 * EEE. 485 */ 486 static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv) 487 { 488 if (stmmac_eee_tx_busy(priv)) { 489 stmmac_restart_sw_lpi_timer(priv); 490 return; 491 } 492 493 /* Check and enter in LPI mode */ 494 if (!priv->tx_path_in_lpi_mode) 495 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED, 496 priv->tx_lpi_clk_stop, 0); 497 } 498 499 /** 500 * stmmac_stop_sw_lpi - stop transmitting LPI 501 * @priv: driver private structure 502 * Description: When using software-controlled LPI, stop transmitting LPI state. 503 */ 504 static void stmmac_stop_sw_lpi(struct stmmac_priv *priv) 505 { 506 timer_delete_sync(&priv->eee_ctrl_timer); 507 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 508 priv->tx_path_in_lpi_mode = false; 509 } 510 511 /** 512 * stmmac_eee_ctrl_timer - EEE TX SW timer. 513 * @t: timer_list struct containing private info 514 * Description: 515 * if there is no data transfer and if we are not in LPI state, 516 * then MAC Transmitter can be moved to LPI state. 517 */ 518 static void stmmac_eee_ctrl_timer(struct timer_list *t) 519 { 520 struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer); 521 522 stmmac_try_to_start_sw_lpi(priv); 523 } 524 525 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 526 * @priv: driver private structure 527 * @p : descriptor pointer 528 * @skb : the socket buffer 529 * Description : 530 * This function will read timestamp from the descriptor & pass it to stack. 531 * and also perform some sanity checks. 532 */ 533 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 534 struct dma_desc *p, struct sk_buff *skb) 535 { 536 struct skb_shared_hwtstamps shhwtstamp; 537 bool found = false; 538 u64 ns = 0; 539 540 if (!priv->hwts_tx_en) 541 return; 542 543 /* exit if skb doesn't support hw tstamp */ 544 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 545 return; 546 547 /* check tx tstamp status */ 548 if (stmmac_get_tx_timestamp_status(priv, p)) { 549 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 550 found = true; 551 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 552 found = true; 553 } 554 555 if (found) { 556 ns -= priv->plat->cdc_error_adj; 557 558 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 559 shhwtstamp.hwtstamp = ns_to_ktime(ns); 560 561 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 562 /* pass tstamp to stack */ 563 skb_tstamp_tx(skb, &shhwtstamp); 564 } 565 } 566 567 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 568 * @priv: driver private structure 569 * @p : descriptor pointer 570 * @np : next descriptor pointer 571 * @skb : the socket buffer 572 * Description : 573 * This function will read received packet's timestamp from the descriptor 574 * and pass it to stack. It also perform some sanity checks. 575 */ 576 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 577 struct dma_desc *np, struct sk_buff *skb) 578 { 579 struct skb_shared_hwtstamps *shhwtstamp = NULL; 580 struct dma_desc *desc = p; 581 u64 ns = 0; 582 583 if (!priv->hwts_rx_en) 584 return; 585 /* For GMAC4, the valid timestamp is from CTX next desc. */ 586 if (dwmac_is_xmac(priv->plat->core_type)) 587 desc = np; 588 589 /* Check if timestamp is available */ 590 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 591 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 592 593 ns -= priv->plat->cdc_error_adj; 594 595 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 596 shhwtstamp = skb_hwtstamps(skb); 597 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 598 shhwtstamp->hwtstamp = ns_to_ktime(ns); 599 } else { 600 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 601 } 602 } 603 604 static void stmmac_update_subsecond_increment(struct stmmac_priv *priv) 605 { 606 bool xmac = dwmac_is_xmac(priv->plat->core_type); 607 u32 sec_inc = 0; 608 u64 temp = 0; 609 610 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 611 612 /* program Sub Second Increment reg */ 613 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 614 priv->plat->clk_ptp_rate, 615 xmac, &sec_inc); 616 temp = div_u64(1000000000ULL, sec_inc); 617 618 /* Store sub second increment for later use */ 619 priv->sub_second_inc = sec_inc; 620 621 /* calculate default added value: 622 * formula is : 623 * addend = (2^32)/freq_div_ratio; 624 * where, freq_div_ratio = 1e9ns/sec_inc 625 */ 626 temp = (u64)(temp << 32); 627 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 628 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 629 } 630 631 /** 632 * stmmac_hwtstamp_set - control hardware timestamping. 633 * @dev: device pointer. 634 * @config: the timestamping configuration. 635 * @extack: netlink extended ack structure for error reporting. 636 * Description: 637 * This function configures the MAC to enable/disable both outgoing(TX) 638 * and incoming(RX) packets time stamping based on user input. 639 * Return Value: 640 * 0 on success and an appropriate -ve integer on failure. 641 */ 642 static int stmmac_hwtstamp_set(struct net_device *dev, 643 struct kernel_hwtstamp_config *config, 644 struct netlink_ext_ack *extack) 645 { 646 struct stmmac_priv *priv = netdev_priv(dev); 647 u32 ptp_v2 = 0; 648 u32 tstamp_all = 0; 649 u32 ptp_over_ipv4_udp = 0; 650 u32 ptp_over_ipv6_udp = 0; 651 u32 ptp_over_ethernet = 0; 652 u32 snap_type_sel = 0; 653 u32 ts_master_en = 0; 654 u32 ts_event_en = 0; 655 656 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 657 NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping"); 658 priv->hwts_tx_en = 0; 659 priv->hwts_rx_en = 0; 660 661 return -EOPNOTSUPP; 662 } 663 664 if (!netif_running(dev)) { 665 NL_SET_ERR_MSG_MOD(extack, 666 "Cannot change timestamping configuration while down"); 667 return -ENODEV; 668 } 669 670 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 671 __func__, config->flags, config->tx_type, config->rx_filter); 672 673 if (config->tx_type != HWTSTAMP_TX_OFF && 674 config->tx_type != HWTSTAMP_TX_ON) 675 return -ERANGE; 676 677 if (priv->adv_ts) { 678 switch (config->rx_filter) { 679 case HWTSTAMP_FILTER_NONE: 680 /* time stamp no incoming packet at all */ 681 config->rx_filter = HWTSTAMP_FILTER_NONE; 682 break; 683 684 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 685 /* PTP v1, UDP, any kind of event packet */ 686 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 687 /* 'xmac' hardware can support Sync, Pdelay_Req and 688 * Pdelay_resp by setting bit14 and bits17/16 to 01 689 * This leaves Delay_Req timestamps out. 690 * Enable all events *and* general purpose message 691 * timestamping 692 */ 693 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 694 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 695 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 696 break; 697 698 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 699 /* PTP v1, UDP, Sync packet */ 700 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 701 /* take time stamp for SYNC messages only */ 702 ts_event_en = PTP_TCR_TSEVNTENA; 703 704 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 705 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 706 break; 707 708 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 709 /* PTP v1, UDP, Delay_req packet */ 710 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 711 /* take time stamp for Delay_Req messages only */ 712 ts_master_en = PTP_TCR_TSMSTRENA; 713 ts_event_en = PTP_TCR_TSEVNTENA; 714 715 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 716 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 717 break; 718 719 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 720 /* PTP v2, UDP, any kind of event packet */ 721 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 722 ptp_v2 = PTP_TCR_TSVER2ENA; 723 /* take time stamp for all event messages */ 724 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 725 726 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 727 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 728 break; 729 730 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 731 /* PTP v2, UDP, Sync packet */ 732 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 733 ptp_v2 = PTP_TCR_TSVER2ENA; 734 /* take time stamp for SYNC messages only */ 735 ts_event_en = PTP_TCR_TSEVNTENA; 736 737 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 738 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 739 break; 740 741 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 742 /* PTP v2, UDP, Delay_req packet */ 743 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 744 ptp_v2 = PTP_TCR_TSVER2ENA; 745 /* take time stamp for Delay_Req messages only */ 746 ts_master_en = PTP_TCR_TSMSTRENA; 747 ts_event_en = PTP_TCR_TSEVNTENA; 748 749 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 750 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 751 break; 752 753 case HWTSTAMP_FILTER_PTP_V2_EVENT: 754 /* PTP v2/802.AS1 any layer, any kind of event packet */ 755 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 756 ptp_v2 = PTP_TCR_TSVER2ENA; 757 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 758 if (priv->synopsys_id < DWMAC_CORE_3_70 && 759 priv->plat->core_type != DWMAC_CORE_XGMAC) 760 ts_event_en = PTP_TCR_TSEVNTENA; 761 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 762 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 763 ptp_over_ethernet = PTP_TCR_TSIPENA; 764 break; 765 766 case HWTSTAMP_FILTER_PTP_V2_SYNC: 767 /* PTP v2/802.AS1, any layer, Sync packet */ 768 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 769 ptp_v2 = PTP_TCR_TSVER2ENA; 770 /* take time stamp for SYNC messages only */ 771 ts_event_en = PTP_TCR_TSEVNTENA; 772 773 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 774 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 775 ptp_over_ethernet = PTP_TCR_TSIPENA; 776 break; 777 778 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 779 /* PTP v2/802.AS1, any layer, Delay_req packet */ 780 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 781 ptp_v2 = PTP_TCR_TSVER2ENA; 782 /* take time stamp for Delay_Req messages only */ 783 ts_master_en = PTP_TCR_TSMSTRENA; 784 ts_event_en = PTP_TCR_TSEVNTENA; 785 786 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 787 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 788 ptp_over_ethernet = PTP_TCR_TSIPENA; 789 break; 790 791 case HWTSTAMP_FILTER_NTP_ALL: 792 case HWTSTAMP_FILTER_ALL: 793 /* time stamp any incoming packet */ 794 config->rx_filter = HWTSTAMP_FILTER_ALL; 795 tstamp_all = PTP_TCR_TSENALL; 796 break; 797 798 default: 799 return -ERANGE; 800 } 801 } else { 802 switch (config->rx_filter) { 803 case HWTSTAMP_FILTER_NONE: 804 config->rx_filter = HWTSTAMP_FILTER_NONE; 805 break; 806 default: 807 /* PTP v1, UDP, any kind of event packet */ 808 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 809 break; 810 } 811 } 812 priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE; 813 priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON; 814 815 priv->systime_flags = STMMAC_HWTS_ACTIVE; 816 if (!priv->tsfupdt_coarse) 817 priv->systime_flags |= PTP_TCR_TSCFUPDT; 818 819 if (priv->hwts_tx_en || priv->hwts_rx_en) { 820 priv->systime_flags |= tstamp_all | ptp_v2 | 821 ptp_over_ethernet | ptp_over_ipv6_udp | 822 ptp_over_ipv4_udp | ts_event_en | 823 ts_master_en | snap_type_sel; 824 } 825 826 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 827 828 priv->tstamp_config = *config; 829 830 return 0; 831 } 832 833 /** 834 * stmmac_hwtstamp_get - read hardware timestamping. 835 * @dev: device pointer. 836 * @config: the timestamping configuration. 837 * Description: 838 * This function obtain the current hardware timestamping settings 839 * as requested. 840 */ 841 static int stmmac_hwtstamp_get(struct net_device *dev, 842 struct kernel_hwtstamp_config *config) 843 { 844 struct stmmac_priv *priv = netdev_priv(dev); 845 846 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 847 return -EOPNOTSUPP; 848 849 *config = priv->tstamp_config; 850 851 return 0; 852 } 853 854 /** 855 * stmmac_init_tstamp_counter - init hardware timestamping counter 856 * @priv: driver private structure 857 * @systime_flags: timestamping flags 858 * Description: 859 * Initialize hardware counter for packet timestamping. 860 * This is valid as long as the interface is open and not suspended. 861 * Will be rerun after resuming from suspend, case in which the timestamping 862 * flags updated by stmmac_hwtstamp_set() also need to be restored. 863 */ 864 static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, 865 u32 systime_flags) 866 { 867 struct timespec64 now; 868 869 if (!priv->plat->clk_ptp_rate) { 870 netdev_err(priv->dev, "Invalid PTP clock rate"); 871 return -EINVAL; 872 } 873 874 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 875 priv->systime_flags = systime_flags; 876 877 stmmac_update_subsecond_increment(priv); 878 879 /* initialize system time */ 880 ktime_get_real_ts64(&now); 881 882 /* lower 32 bits of tv_sec are safe until y2106 */ 883 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 884 885 return 0; 886 } 887 888 /** 889 * stmmac_init_timestamping - initialise timestamping 890 * @priv: driver private structure 891 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 892 * This is done by looking at the HW cap. register. 893 * This function also registers the ptp driver. 894 */ 895 static int stmmac_init_timestamping(struct stmmac_priv *priv) 896 { 897 bool xmac = dwmac_is_xmac(priv->plat->core_type); 898 int ret; 899 900 if (priv->plat->ptp_clk_freq_config) 901 priv->plat->ptp_clk_freq_config(priv); 902 903 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) { 904 netdev_info(priv->dev, "PTP not supported by HW\n"); 905 return -EOPNOTSUPP; 906 } 907 908 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE | 909 PTP_TCR_TSCFUPDT); 910 if (ret) { 911 netdev_warn(priv->dev, "PTP init failed\n"); 912 return ret; 913 } 914 915 priv->adv_ts = 0; 916 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 917 if (xmac && priv->dma_cap.atime_stamp) 918 priv->adv_ts = 1; 919 /* Dwmac 3.x core with extend_desc can support adv_ts */ 920 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 921 priv->adv_ts = 1; 922 923 if (priv->dma_cap.time_stamp) 924 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 925 926 if (priv->adv_ts) 927 netdev_info(priv->dev, 928 "IEEE 1588-2008 Advanced Timestamp supported\n"); 929 930 memset(&priv->tstamp_config, 0, sizeof(priv->tstamp_config)); 931 priv->hwts_tx_en = 0; 932 priv->hwts_rx_en = 0; 933 934 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 935 stmmac_hwtstamp_correct_latency(priv, priv); 936 937 return 0; 938 } 939 940 static void stmmac_setup_ptp(struct stmmac_priv *priv) 941 { 942 int ret; 943 944 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 945 if (ret < 0) 946 netdev_warn(priv->dev, 947 "failed to enable PTP reference clock: %pe\n", 948 ERR_PTR(ret)); 949 950 if (stmmac_init_timestamping(priv) == 0) 951 stmmac_ptp_register(priv); 952 } 953 954 static void stmmac_release_ptp(struct stmmac_priv *priv) 955 { 956 stmmac_ptp_unregister(priv); 957 clk_disable_unprepare(priv->plat->clk_ptp_ref); 958 } 959 960 static void stmmac_legacy_serdes_power_down(struct stmmac_priv *priv) 961 { 962 if (priv->plat->serdes_powerdown && priv->legacy_serdes_is_powered) 963 priv->plat->serdes_powerdown(priv->dev, priv->plat->bsp_priv); 964 965 priv->legacy_serdes_is_powered = false; 966 } 967 968 static int stmmac_legacy_serdes_power_up(struct stmmac_priv *priv) 969 { 970 int ret; 971 972 if (!priv->plat->serdes_powerup) 973 return 0; 974 975 ret = priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 976 if (ret < 0) 977 netdev_err(priv->dev, "SerDes powerup failed\n"); 978 else 979 priv->legacy_serdes_is_powered = true; 980 981 return ret; 982 } 983 984 /** 985 * stmmac_mac_flow_ctrl - Configure flow control in all queues 986 * @priv: driver private structure 987 * @duplex: duplex passed to the next function 988 * @flow_ctrl: desired flow control modes 989 * Description: It is used for configuring the flow control in all queues 990 */ 991 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex, 992 unsigned int flow_ctrl) 993 { 994 u8 tx_cnt = priv->plat->tx_queues_to_use; 995 996 stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time, 997 tx_cnt); 998 } 999 1000 static unsigned long stmmac_mac_get_caps(struct phylink_config *config, 1001 phy_interface_t interface) 1002 { 1003 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1004 1005 /* Refresh the MAC-specific capabilities */ 1006 stmmac_mac_update_caps(priv); 1007 1008 if (priv->hw_cap_support && !priv->dma_cap.half_duplex) 1009 priv->hw->link.caps &= ~(MAC_1000HD | MAC_100HD | MAC_10HD); 1010 1011 config->mac_capabilities = priv->hw->link.caps; 1012 1013 if (priv->plat->max_speed) 1014 phylink_limit_mac_speed(config, priv->plat->max_speed); 1015 1016 return config->mac_capabilities; 1017 } 1018 1019 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 1020 phy_interface_t interface) 1021 { 1022 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1023 struct phylink_pcs *pcs; 1024 1025 if (priv->plat->select_pcs) { 1026 pcs = priv->plat->select_pcs(priv, interface); 1027 if (!IS_ERR(pcs)) 1028 return pcs; 1029 } 1030 1031 if (priv->integrated_pcs && 1032 test_bit(interface, priv->integrated_pcs->pcs.supported_interfaces)) 1033 return &priv->integrated_pcs->pcs; 1034 1035 return NULL; 1036 } 1037 1038 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 1039 const struct phylink_link_state *state) 1040 { 1041 /* Nothing to do, xpcs_config() handles everything */ 1042 } 1043 1044 static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode, 1045 phy_interface_t interface) 1046 { 1047 struct net_device *ndev = to_net_dev(config->dev); 1048 struct stmmac_priv *priv = netdev_priv(ndev); 1049 1050 if (priv->plat->mac_finish) 1051 priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, 1052 interface); 1053 1054 return 0; 1055 } 1056 1057 static void stmmac_mac_link_down(struct phylink_config *config, 1058 unsigned int mode, phy_interface_t interface) 1059 { 1060 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1061 1062 stmmac_mac_set(priv, priv->ioaddr, false); 1063 if (priv->dma_cap.eee) 1064 stmmac_set_eee_pls(priv, priv->hw, false); 1065 1066 if (stmmac_fpe_supported(priv)) 1067 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, false); 1068 } 1069 1070 static void stmmac_mac_link_up(struct phylink_config *config, 1071 struct phy_device *phy, 1072 unsigned int mode, phy_interface_t interface, 1073 int speed, int duplex, 1074 bool tx_pause, bool rx_pause) 1075 { 1076 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1077 unsigned int flow_ctrl; 1078 u32 old_ctrl, ctrl; 1079 int ret; 1080 1081 if (priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) 1082 stmmac_legacy_serdes_power_up(priv); 1083 1084 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1085 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1086 1087 switch (speed) { 1088 case SPEED_100000: 1089 ctrl |= priv->hw->link.xlgmii.speed100000; 1090 break; 1091 case SPEED_50000: 1092 ctrl |= priv->hw->link.xlgmii.speed50000; 1093 break; 1094 case SPEED_40000: 1095 ctrl |= priv->hw->link.xlgmii.speed40000; 1096 break; 1097 case SPEED_25000: 1098 ctrl |= priv->hw->link.xlgmii.speed25000; 1099 break; 1100 case SPEED_10000: 1101 ctrl |= priv->hw->link.xgmii.speed10000; 1102 break; 1103 case SPEED_5000: 1104 ctrl |= priv->hw->link.xgmii.speed5000; 1105 break; 1106 case SPEED_2500: 1107 if (interface == PHY_INTERFACE_MODE_USXGMII) 1108 ctrl |= priv->hw->link.xgmii.speed2500; 1109 else 1110 ctrl |= priv->hw->link.speed2500; 1111 break; 1112 case SPEED_1000: 1113 ctrl |= priv->hw->link.speed1000; 1114 break; 1115 case SPEED_100: 1116 ctrl |= priv->hw->link.speed100; 1117 break; 1118 case SPEED_10: 1119 ctrl |= priv->hw->link.speed10; 1120 break; 1121 default: 1122 netdev_err(priv->dev, 1123 "unsupported speed %s on %s, leaving the MAC disabled\n", 1124 phy_speed_to_str(speed), phy_modes(interface)); 1125 return; 1126 } 1127 1128 if (priv->plat->fix_mac_speed) 1129 priv->plat->fix_mac_speed(priv->plat->bsp_priv, interface, 1130 speed, mode); 1131 1132 if (!duplex) 1133 ctrl &= ~priv->hw->link.duplex; 1134 else 1135 ctrl |= priv->hw->link.duplex; 1136 1137 /* Flow Control operation */ 1138 if (rx_pause && tx_pause) 1139 flow_ctrl = FLOW_AUTO; 1140 else if (rx_pause && !tx_pause) 1141 flow_ctrl = FLOW_RX; 1142 else if (!rx_pause && tx_pause) 1143 flow_ctrl = FLOW_TX; 1144 else 1145 flow_ctrl = FLOW_OFF; 1146 1147 stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl); 1148 1149 if (ctrl != old_ctrl) 1150 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1151 1152 if (priv->plat->set_clk_tx_rate) { 1153 ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv, 1154 priv->plat->clk_tx_i, 1155 interface, speed); 1156 if (ret < 0) 1157 netdev_err(priv->dev, 1158 "failed to configure %s transmit clock for %dMbps: %pe\n", 1159 phy_modes(interface), speed, ERR_PTR(ret)); 1160 } 1161 1162 stmmac_mac_set(priv, priv->ioaddr, true); 1163 if (priv->dma_cap.eee) 1164 stmmac_set_eee_pls(priv, priv->hw, true); 1165 1166 if (stmmac_fpe_supported(priv)) 1167 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, true); 1168 1169 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1170 stmmac_hwtstamp_correct_latency(priv, priv); 1171 } 1172 1173 static void stmmac_mac_disable_tx_lpi(struct phylink_config *config) 1174 { 1175 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1176 1177 priv->eee_active = false; 1178 1179 mutex_lock(&priv->lock); 1180 1181 priv->eee_enabled = false; 1182 1183 netdev_dbg(priv->dev, "disable EEE\n"); 1184 priv->eee_sw_timer_en = false; 1185 timer_delete_sync(&priv->eee_ctrl_timer); 1186 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 1187 priv->tx_path_in_lpi_mode = false; 1188 1189 stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS); 1190 mutex_unlock(&priv->lock); 1191 } 1192 1193 static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 1194 bool tx_clk_stop) 1195 { 1196 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1197 int ret; 1198 1199 priv->tx_lpi_timer = timer; 1200 priv->eee_active = true; 1201 1202 mutex_lock(&priv->lock); 1203 1204 priv->eee_enabled = true; 1205 1206 /* Update the transmit clock stop according to PHY capability if 1207 * the platform allows 1208 */ 1209 if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP) 1210 priv->tx_lpi_clk_stop = tx_clk_stop; 1211 1212 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 1213 STMMAC_DEFAULT_TWT_LS); 1214 1215 /* Try to configure the hardware timer. */ 1216 ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER, 1217 priv->tx_lpi_clk_stop, priv->tx_lpi_timer); 1218 1219 if (ret) { 1220 /* Hardware timer mode not supported, or value out of range. 1221 * Fall back to using software LPI mode 1222 */ 1223 priv->eee_sw_timer_en = true; 1224 stmmac_restart_sw_lpi_timer(priv); 1225 } 1226 1227 mutex_unlock(&priv->lock); 1228 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 1229 1230 return 0; 1231 } 1232 1233 static int stmmac_mac_wol_set(struct phylink_config *config, u32 wolopts, 1234 const u8 *sopass) 1235 { 1236 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1237 1238 device_set_wakeup_enable(priv->device, !!wolopts); 1239 1240 mutex_lock(&priv->lock); 1241 priv->wolopts = wolopts; 1242 mutex_unlock(&priv->lock); 1243 1244 return 0; 1245 } 1246 1247 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1248 .mac_get_caps = stmmac_mac_get_caps, 1249 .mac_select_pcs = stmmac_mac_select_pcs, 1250 .mac_config = stmmac_mac_config, 1251 .mac_finish = stmmac_mac_finish, 1252 .mac_link_down = stmmac_mac_link_down, 1253 .mac_link_up = stmmac_mac_link_up, 1254 .mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi, 1255 .mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi, 1256 .mac_wol_set = stmmac_mac_wol_set, 1257 }; 1258 1259 /** 1260 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1261 * @priv: driver private structure 1262 * Description: this is to verify if the HW supports the PCS. 1263 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1264 * configured for the TBI, RTBI, or SGMII PHY interface. 1265 */ 1266 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1267 { 1268 int interface = priv->plat->phy_interface; 1269 int speed = priv->plat->mac_port_sel_speed; 1270 1271 if (priv->dma_cap.pcs && interface == PHY_INTERFACE_MODE_SGMII) { 1272 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1273 1274 switch (speed) { 1275 case SPEED_10: 1276 case SPEED_100: 1277 case SPEED_1000: 1278 priv->hw->reverse_sgmii_enable = true; 1279 break; 1280 1281 default: 1282 dev_warn(priv->device, "invalid port speed\n"); 1283 fallthrough; 1284 case 0: 1285 priv->hw->reverse_sgmii_enable = false; 1286 break; 1287 } 1288 } 1289 } 1290 1291 /** 1292 * stmmac_init_phy - PHY initialization 1293 * @dev: net device structure 1294 * Description: it initializes the driver's PHY state, and attaches the PHY 1295 * to the mac driver. 1296 * Return value: 1297 * 0 on success 1298 */ 1299 static int stmmac_init_phy(struct net_device *dev) 1300 { 1301 struct stmmac_priv *priv = netdev_priv(dev); 1302 int mode = priv->plat->phy_interface; 1303 struct fwnode_handle *phy_fwnode; 1304 struct fwnode_handle *fwnode; 1305 struct ethtool_keee eee; 1306 u32 dev_flags = 0; 1307 int ret; 1308 1309 if (!phylink_expects_phy(priv->phylink)) 1310 return 0; 1311 1312 if (priv->hw->xpcs && 1313 xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73) 1314 return 0; 1315 1316 fwnode = dev_fwnode(priv->device); 1317 if (fwnode) 1318 phy_fwnode = fwnode_get_phy_node(fwnode); 1319 else 1320 phy_fwnode = NULL; 1321 1322 if (priv->plat->flags & STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD) 1323 dev_flags |= PHY_F_KEEP_PREAMBLE_BEFORE_SFD; 1324 1325 /* Some DT bindings do not set-up the PHY handle. Let's try to 1326 * manually parse it 1327 */ 1328 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1329 int addr = priv->plat->phy_addr; 1330 struct phy_device *phydev; 1331 1332 if (addr < 0) { 1333 /* If a custom PCS is in use, no PHY is needed */ 1334 if (priv->hw->phylink_pcs) 1335 return 0; 1336 1337 netdev_err(priv->dev, "no phy found\n"); 1338 return -ENODEV; 1339 } 1340 1341 phydev = mdiobus_get_phy(priv->mii, addr); 1342 if (!phydev) { 1343 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1344 return -ENODEV; 1345 } 1346 1347 phydev->dev_flags |= dev_flags; 1348 1349 ret = phylink_connect_phy(priv->phylink, phydev); 1350 } else { 1351 fwnode_handle_put(phy_fwnode); 1352 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, dev_flags); 1353 } 1354 1355 if (ret) { 1356 netdev_err(priv->dev, "cannot attach to PHY (error: %pe)\n", 1357 ERR_PTR(ret)); 1358 return ret; 1359 } 1360 1361 /* Configure phylib's copy of the LPI timer. Normally, 1362 * phylink_config.lpi_timer_default would do this, but there is a 1363 * chance that userspace could change the eee_timer setting via sysfs 1364 * before the first open. Thus, preserve existing behaviour. 1365 */ 1366 if (!phylink_ethtool_get_eee(priv->phylink, &eee)) { 1367 eee.tx_lpi_timer = priv->tx_lpi_timer; 1368 phylink_ethtool_set_eee(priv->phylink, &eee); 1369 } 1370 1371 return 0; 1372 } 1373 1374 static int stmmac_phylink_setup(struct stmmac_priv *priv) 1375 { 1376 struct phylink_config *config; 1377 struct phylink_pcs *pcs; 1378 struct phylink *phylink; 1379 1380 config = &priv->phylink_config; 1381 1382 config->dev = &priv->dev->dev; 1383 config->type = PHYLINK_NETDEV; 1384 config->mac_managed_pm = true; 1385 1386 /* Stmmac always requires an RX clock for hardware initialization */ 1387 config->mac_requires_rxc = true; 1388 1389 /* Disable EEE RX clock stop to ensure VLAN register access works 1390 * correctly. 1391 */ 1392 if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI) && 1393 !(priv->dev->features & NETIF_F_VLAN_FEATURES)) 1394 config->eee_rx_clk_stop_enable = true; 1395 1396 /* Set the default transmit clock stop bit based on the platform glue */ 1397 priv->tx_lpi_clk_stop = priv->plat->flags & 1398 STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; 1399 1400 /* Get the PHY interface modes (at the PHY end of the link) that 1401 * are supported by the platform. 1402 */ 1403 if (priv->plat->get_interfaces) 1404 priv->plat->get_interfaces(priv, priv->plat->bsp_priv, 1405 config->supported_interfaces); 1406 1407 config->default_an_inband = priv->plat->default_an_inband; 1408 1409 /* Set the platform/firmware specified interface mode if the 1410 * supported interfaces have not already been provided using 1411 * phy_interface as a last resort. 1412 */ 1413 if (phy_interface_empty(config->supported_interfaces)) 1414 __set_bit(priv->plat->phy_interface, 1415 config->supported_interfaces); 1416 1417 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1418 if (priv->hw->xpcs) 1419 pcs = xpcs_to_phylink_pcs(priv->hw->xpcs); 1420 else 1421 pcs = priv->hw->phylink_pcs; 1422 1423 if (pcs) 1424 phy_interface_or(config->supported_interfaces, 1425 config->supported_interfaces, 1426 pcs->supported_interfaces); 1427 1428 /* Some platforms, e.g. iMX8MP, wire lpi_intr_o to the same interrupt 1429 * used for stmmac's main interrupts, which leads to interrupt storms. 1430 * STMMAC_FLAG_EEE_DISABLE allows EEE to be disabled on such platforms. 1431 */ 1432 if (priv->dma_cap.eee && 1433 !(priv->plat->flags & STMMAC_FLAG_EEE_DISABLE)) { 1434 /* The GMAC 3.74a databook states that EEE is only supported 1435 * in MII, GMII, and RGMII interfaces. 1436 */ 1437 __set_bit(PHY_INTERFACE_MODE_MII, config->lpi_interfaces); 1438 __set_bit(PHY_INTERFACE_MODE_GMII, config->lpi_interfaces); 1439 phy_interface_set_rgmii(config->lpi_interfaces); 1440 1441 /* If we have a non-integrated PCS, assume that it is connected 1442 * to the GMAC using GMII or another EEE compatible interface, 1443 * and thus all PCS-supported interfaces support LPI. 1444 */ 1445 if (pcs) 1446 phy_interface_or(config->lpi_interfaces, 1447 config->lpi_interfaces, 1448 pcs->supported_interfaces); 1449 1450 /* All full duplex speeds above 100Mbps are supported */ 1451 config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD; 1452 config->lpi_timer_default = eee_timer * 1000; 1453 config->eee_enabled_default = true; 1454 } 1455 1456 config->wol_phy_speed_ctrl = true; 1457 if (priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL) { 1458 config->wol_phy_legacy = true; 1459 } else { 1460 if (priv->dma_cap.pmt_remote_wake_up) 1461 config->wol_mac_support |= WAKE_UCAST; 1462 if (priv->dma_cap.pmt_magic_frame) 1463 config->wol_mac_support |= WAKE_MAGIC; 1464 } 1465 1466 phylink = phylink_create(config, dev_fwnode(priv->device), 1467 priv->plat->phy_interface, 1468 &stmmac_phylink_mac_ops); 1469 if (IS_ERR(phylink)) 1470 return PTR_ERR(phylink); 1471 1472 priv->phylink = phylink; 1473 return 0; 1474 } 1475 1476 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1477 struct stmmac_dma_conf *dma_conf) 1478 { 1479 u8 rx_cnt = priv->plat->rx_queues_to_use; 1480 unsigned int desc_size; 1481 void *head_rx; 1482 u8 queue; 1483 1484 /* Display RX rings */ 1485 for (queue = 0; queue < rx_cnt; queue++) { 1486 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1487 1488 pr_info("\tRX Queue %u rings\n", queue); 1489 1490 head_rx = stmmac_get_rx_desc(priv, rx_q, 0); 1491 desc_size = stmmac_get_rx_desc_size(priv); 1492 1493 /* Display RX ring */ 1494 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1495 rx_q->dma_rx_phy, desc_size); 1496 } 1497 } 1498 1499 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1500 struct stmmac_dma_conf *dma_conf) 1501 { 1502 u8 tx_cnt = priv->plat->tx_queues_to_use; 1503 unsigned int desc_size; 1504 void *head_tx; 1505 u8 queue; 1506 1507 /* Display TX rings */ 1508 for (queue = 0; queue < tx_cnt; queue++) { 1509 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1510 1511 pr_info("\tTX Queue %d rings\n", queue); 1512 1513 head_tx = stmmac_get_tx_desc(priv, tx_q, 0); 1514 desc_size = stmmac_get_tx_desc_size(priv, tx_q); 1515 1516 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1517 tx_q->dma_tx_phy, desc_size); 1518 } 1519 } 1520 1521 static void stmmac_display_rings(struct stmmac_priv *priv, 1522 struct stmmac_dma_conf *dma_conf) 1523 { 1524 /* Display RX ring */ 1525 stmmac_display_rx_rings(priv, dma_conf); 1526 1527 /* Display TX ring */ 1528 stmmac_display_tx_rings(priv, dma_conf); 1529 } 1530 1531 static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) 1532 { 1533 if (stmmac_xdp_is_enabled(priv)) 1534 return XDP_PACKET_HEADROOM + NET_IP_ALIGN; 1535 1536 return NET_SKB_PAD + NET_IP_ALIGN; 1537 } 1538 1539 static int stmmac_set_bfsize(int len) 1540 { 1541 int ret; 1542 1543 if (len > BUF_SIZE_8KiB) 1544 ret = BUF_SIZE_16KiB; 1545 else if (len > BUF_SIZE_4KiB) 1546 ret = BUF_SIZE_8KiB; 1547 else if (len > BUF_SIZE_2KiB) 1548 ret = BUF_SIZE_4KiB; 1549 else if (len > DEFAULT_BUFSIZE) 1550 ret = BUF_SIZE_2KiB; 1551 else 1552 ret = DEFAULT_BUFSIZE; 1553 1554 return ret; 1555 } 1556 1557 /** 1558 * stmmac_clear_rx_descriptors - clear RX descriptors 1559 * @priv: driver private structure 1560 * @dma_conf: structure to take the dma data 1561 * @queue: RX queue index 1562 * Description: this function is called to clear the RX descriptors 1563 * in case of both basic and extended descriptors are used. 1564 */ 1565 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1566 struct stmmac_dma_conf *dma_conf, 1567 u32 queue) 1568 { 1569 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1570 struct dma_desc *desc; 1571 int i; 1572 1573 /* Clear the RX descriptors */ 1574 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1575 desc = stmmac_get_rx_desc(priv, rx_q, i); 1576 1577 stmmac_init_rx_desc(priv, desc, priv->use_riwt, 1578 priv->descriptor_mode, 1579 (i == dma_conf->dma_rx_size - 1), 1580 dma_conf->dma_buf_sz); 1581 } 1582 } 1583 1584 /** 1585 * stmmac_clear_tx_descriptors - clear tx descriptors 1586 * @priv: driver private structure 1587 * @dma_conf: structure to take the dma data 1588 * @queue: TX queue index. 1589 * Description: this function is called to clear the TX descriptors 1590 * in case of both basic and extended descriptors are used. 1591 */ 1592 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1593 struct stmmac_dma_conf *dma_conf, 1594 u32 queue) 1595 { 1596 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1597 int i; 1598 1599 /* Clear the TX descriptors */ 1600 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1601 int last = (i == (dma_conf->dma_tx_size - 1)); 1602 struct dma_desc *p; 1603 1604 p = stmmac_get_tx_desc(priv, tx_q, i); 1605 stmmac_init_tx_desc(priv, p, priv->descriptor_mode, last); 1606 } 1607 } 1608 1609 /** 1610 * stmmac_clear_descriptors - clear descriptors 1611 * @priv: driver private structure 1612 * @dma_conf: structure to take the dma data 1613 * Description: this function is called to clear the TX and RX descriptors 1614 * in case of both basic and extended descriptors are used. 1615 */ 1616 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1617 struct stmmac_dma_conf *dma_conf) 1618 { 1619 u8 rx_queue_cnt = priv->plat->rx_queues_to_use; 1620 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 1621 u8 queue; 1622 1623 /* Clear the RX descriptors */ 1624 for (queue = 0; queue < rx_queue_cnt; queue++) 1625 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1626 1627 /* Clear the TX descriptors */ 1628 for (queue = 0; queue < tx_queue_cnt; queue++) 1629 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1630 } 1631 1632 /** 1633 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1634 * @priv: driver private structure 1635 * @dma_conf: structure to take the dma data 1636 * @p: descriptor pointer 1637 * @i: descriptor index 1638 * @flags: gfp flag 1639 * @queue: RX queue index 1640 * Description: this function is called to allocate a receive buffer, perform 1641 * the DMA mapping and init the descriptor. 1642 */ 1643 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1644 struct stmmac_dma_conf *dma_conf, 1645 struct dma_desc *p, 1646 int i, gfp_t flags, u32 queue) 1647 { 1648 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1649 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1650 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1651 1652 if (priv->dma_cap.host_dma_width <= 32) 1653 gfp |= GFP_DMA32; 1654 1655 if (!buf->page) { 1656 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1657 if (!buf->page) 1658 return -ENOMEM; 1659 buf->page_offset = stmmac_rx_offset(priv); 1660 } 1661 1662 if (priv->sph_active && !buf->sec_page) { 1663 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1664 if (!buf->sec_page) 1665 return -ENOMEM; 1666 1667 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1668 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1669 } else { 1670 buf->sec_page = NULL; 1671 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1672 } 1673 1674 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1675 1676 stmmac_set_desc_addr(priv, p, buf->addr); 1677 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1678 stmmac_init_desc3(priv, p); 1679 1680 return 0; 1681 } 1682 1683 /** 1684 * stmmac_free_rx_buffer - free RX dma buffers 1685 * @priv: private structure 1686 * @rx_q: RX queue 1687 * @i: buffer index. 1688 */ 1689 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1690 struct stmmac_rx_queue *rx_q, 1691 int i) 1692 { 1693 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1694 1695 if (buf->page) 1696 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1697 buf->page = NULL; 1698 1699 if (buf->sec_page) 1700 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1701 buf->sec_page = NULL; 1702 } 1703 1704 /** 1705 * stmmac_free_tx_buffer - free RX dma buffers 1706 * @priv: private structure 1707 * @dma_conf: structure to take the dma data 1708 * @queue: RX queue index 1709 * @i: buffer index. 1710 */ 1711 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1712 struct stmmac_dma_conf *dma_conf, 1713 u32 queue, int i) 1714 { 1715 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1716 1717 if (tx_q->tx_skbuff_dma[i].buf && 1718 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1719 if (tx_q->tx_skbuff_dma[i].map_as_page) 1720 dma_unmap_page(priv->device, 1721 tx_q->tx_skbuff_dma[i].buf, 1722 tx_q->tx_skbuff_dma[i].len, 1723 DMA_TO_DEVICE); 1724 else 1725 dma_unmap_single(priv->device, 1726 tx_q->tx_skbuff_dma[i].buf, 1727 tx_q->tx_skbuff_dma[i].len, 1728 DMA_TO_DEVICE); 1729 } 1730 1731 if (tx_q->xdpf[i] && 1732 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1733 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1734 xdp_return_frame(tx_q->xdpf[i]); 1735 tx_q->xdpf[i] = NULL; 1736 } 1737 1738 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1739 tx_q->xsk_frames_done++; 1740 1741 if (tx_q->tx_skbuff[i] && 1742 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1743 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1744 tx_q->tx_skbuff[i] = NULL; 1745 } 1746 1747 tx_q->tx_skbuff_dma[i].buf = 0; 1748 tx_q->tx_skbuff_dma[i].map_as_page = false; 1749 } 1750 1751 /** 1752 * dma_free_rx_skbufs - free RX dma buffers 1753 * @priv: private structure 1754 * @dma_conf: structure to take the dma data 1755 * @queue: RX queue index 1756 */ 1757 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1758 struct stmmac_dma_conf *dma_conf, 1759 u32 queue) 1760 { 1761 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1762 int i; 1763 1764 for (i = 0; i < dma_conf->dma_rx_size; i++) 1765 stmmac_free_rx_buffer(priv, rx_q, i); 1766 } 1767 1768 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1769 struct stmmac_dma_conf *dma_conf, 1770 u32 queue, gfp_t flags) 1771 { 1772 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1773 int i; 1774 1775 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1776 struct dma_desc *p; 1777 int ret; 1778 1779 p = stmmac_get_rx_desc(priv, rx_q, i); 1780 1781 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1782 queue); 1783 if (ret) 1784 return ret; 1785 1786 rx_q->buf_alloc_num++; 1787 } 1788 1789 return 0; 1790 } 1791 1792 /** 1793 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1794 * @priv: private structure 1795 * @dma_conf: structure to take the dma data 1796 * @queue: RX queue index 1797 */ 1798 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1799 struct stmmac_dma_conf *dma_conf, 1800 u32 queue) 1801 { 1802 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1803 int i; 1804 1805 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1806 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1807 1808 if (!buf->xdp) 1809 continue; 1810 1811 xsk_buff_free(buf->xdp); 1812 buf->xdp = NULL; 1813 } 1814 } 1815 1816 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1817 struct stmmac_dma_conf *dma_conf, 1818 u32 queue) 1819 { 1820 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1821 int i; 1822 1823 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1824 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1825 * use this macro to make sure no size violations. 1826 */ 1827 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1828 1829 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1830 struct stmmac_rx_buffer *buf; 1831 dma_addr_t dma_addr; 1832 struct dma_desc *p; 1833 1834 p = stmmac_get_rx_desc(priv, rx_q, i); 1835 1836 buf = &rx_q->buf_pool[i]; 1837 1838 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1839 if (!buf->xdp) 1840 return -ENOMEM; 1841 1842 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1843 stmmac_set_desc_addr(priv, p, dma_addr); 1844 rx_q->buf_alloc_num++; 1845 } 1846 1847 return 0; 1848 } 1849 1850 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1851 { 1852 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1853 return NULL; 1854 1855 return xsk_get_pool_from_qid(priv->dev, queue); 1856 } 1857 1858 /** 1859 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1860 * @priv: driver private structure 1861 * @dma_conf: structure to take the dma data 1862 * @queue: RX queue index 1863 * @flags: gfp flag. 1864 * Description: this function initializes the DMA RX descriptors 1865 * and allocates the socket buffers. It supports the chained and ring 1866 * modes. 1867 */ 1868 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1869 struct stmmac_dma_conf *dma_conf, 1870 u32 queue, gfp_t flags) 1871 { 1872 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1873 void *des; 1874 int ret; 1875 1876 netif_dbg(priv, probe, priv->dev, 1877 "(%s) dma_rx_phy=0x%08x\n", __func__, 1878 (u32)rx_q->dma_rx_phy); 1879 1880 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1881 1882 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1883 1884 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1885 1886 if (rx_q->xsk_pool) { 1887 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1888 MEM_TYPE_XSK_BUFF_POOL, 1889 NULL)); 1890 netdev_info(priv->dev, 1891 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1892 queue); 1893 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1894 } else { 1895 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1896 MEM_TYPE_PAGE_POOL, 1897 rx_q->page_pool)); 1898 netdev_info(priv->dev, 1899 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1900 queue); 1901 } 1902 1903 if (rx_q->xsk_pool) { 1904 /* RX XDP ZC buffer pool may not be populated, e.g. 1905 * xdpsock TX-only. 1906 */ 1907 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1908 } else { 1909 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1910 if (ret < 0) 1911 return -ENOMEM; 1912 } 1913 1914 /* Setup the chained descriptor addresses */ 1915 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 1916 if (priv->extend_desc) 1917 des = rx_q->dma_erx; 1918 else 1919 des = rx_q->dma_rx; 1920 1921 stmmac_mode_init(priv, des, rx_q->dma_rx_phy, 1922 dma_conf->dma_rx_size, priv->extend_desc); 1923 } 1924 1925 return 0; 1926 } 1927 1928 static int init_dma_rx_desc_rings(struct net_device *dev, 1929 struct stmmac_dma_conf *dma_conf, 1930 gfp_t flags) 1931 { 1932 struct stmmac_priv *priv = netdev_priv(dev); 1933 u8 rx_count = priv->plat->rx_queues_to_use; 1934 int queue; 1935 int ret; 1936 1937 /* RX INITIALIZATION */ 1938 netif_dbg(priv, probe, priv->dev, 1939 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1940 1941 for (queue = 0; queue < rx_count; queue++) { 1942 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1943 if (ret) 1944 goto err_init_rx_buffers; 1945 } 1946 1947 return 0; 1948 1949 err_init_rx_buffers: 1950 while (queue >= 0) { 1951 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1952 1953 if (rx_q->xsk_pool) 1954 dma_free_rx_xskbufs(priv, dma_conf, queue); 1955 else 1956 dma_free_rx_skbufs(priv, dma_conf, queue); 1957 1958 rx_q->buf_alloc_num = 0; 1959 rx_q->xsk_pool = NULL; 1960 1961 queue--; 1962 } 1963 1964 return ret; 1965 } 1966 1967 static void stmmac_set_tx_dma_entry(struct stmmac_tx_queue *tx_q, 1968 unsigned int entry, 1969 enum stmmac_txbuf_type type, 1970 dma_addr_t addr, size_t len, 1971 bool map_as_page) 1972 { 1973 tx_q->tx_skbuff_dma[entry].buf = addr; 1974 tx_q->tx_skbuff_dma[entry].len = len; 1975 tx_q->tx_skbuff_dma[entry].buf_type = type; 1976 tx_q->tx_skbuff_dma[entry].map_as_page = map_as_page; 1977 tx_q->tx_skbuff_dma[entry].last_segment = false; 1978 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1979 } 1980 1981 static void stmmac_set_tx_skb_dma_entry(struct stmmac_tx_queue *tx_q, 1982 unsigned int entry, dma_addr_t addr, 1983 size_t len, bool map_as_page) 1984 { 1985 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_SKB, addr, len, 1986 map_as_page); 1987 } 1988 1989 static void stmmac_set_tx_dma_last_segment(struct stmmac_tx_queue *tx_q, 1990 unsigned int entry) 1991 { 1992 tx_q->tx_skbuff_dma[entry].last_segment = true; 1993 } 1994 1995 /** 1996 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1997 * @priv: driver private structure 1998 * @dma_conf: structure to take the dma data 1999 * @queue: TX queue index 2000 * Description: this function initializes the DMA TX descriptors 2001 * and allocates the socket buffers. It supports the chained and ring 2002 * modes. 2003 */ 2004 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 2005 struct stmmac_dma_conf *dma_conf, 2006 u32 queue) 2007 { 2008 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2009 int i; 2010 2011 netif_dbg(priv, probe, priv->dev, 2012 "(%s) dma_tx_phy=0x%08x\n", __func__, 2013 (u32)tx_q->dma_tx_phy); 2014 2015 /* Setup the chained descriptor addresses */ 2016 if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { 2017 if (priv->extend_desc) 2018 stmmac_mode_init(priv, tx_q->dma_etx, 2019 tx_q->dma_tx_phy, 2020 dma_conf->dma_tx_size, 1); 2021 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 2022 stmmac_mode_init(priv, tx_q->dma_tx, 2023 tx_q->dma_tx_phy, 2024 dma_conf->dma_tx_size, 0); 2025 } 2026 2027 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 2028 2029 for (i = 0; i < dma_conf->dma_tx_size; i++) { 2030 struct dma_desc *p; 2031 2032 p = stmmac_get_tx_desc(priv, tx_q, i); 2033 stmmac_clear_desc(priv, p); 2034 stmmac_set_tx_skb_dma_entry(tx_q, i, 0, 0, false); 2035 2036 tx_q->tx_skbuff[i] = NULL; 2037 } 2038 2039 return 0; 2040 } 2041 2042 static int init_dma_tx_desc_rings(struct net_device *dev, 2043 struct stmmac_dma_conf *dma_conf) 2044 { 2045 struct stmmac_priv *priv = netdev_priv(dev); 2046 u8 tx_queue_cnt; 2047 u8 queue; 2048 2049 tx_queue_cnt = priv->plat->tx_queues_to_use; 2050 2051 for (queue = 0; queue < tx_queue_cnt; queue++) 2052 __init_dma_tx_desc_rings(priv, dma_conf, queue); 2053 2054 return 0; 2055 } 2056 2057 /** 2058 * init_dma_desc_rings - init the RX/TX descriptor rings 2059 * @dev: net device structure 2060 * @dma_conf: structure to take the dma data 2061 * @flags: gfp flag. 2062 * Description: this function initializes the DMA RX/TX descriptors 2063 * and allocates the socket buffers. It supports the chained and ring 2064 * modes. 2065 */ 2066 static int init_dma_desc_rings(struct net_device *dev, 2067 struct stmmac_dma_conf *dma_conf, 2068 gfp_t flags) 2069 { 2070 struct stmmac_priv *priv = netdev_priv(dev); 2071 int ret; 2072 2073 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 2074 if (ret) 2075 return ret; 2076 2077 ret = init_dma_tx_desc_rings(dev, dma_conf); 2078 2079 stmmac_clear_descriptors(priv, dma_conf); 2080 2081 if (netif_msg_hw(priv)) 2082 stmmac_display_rings(priv, dma_conf); 2083 2084 return ret; 2085 } 2086 2087 /** 2088 * dma_free_tx_skbufs - free TX dma buffers 2089 * @priv: private structure 2090 * @dma_conf: structure to take the dma data 2091 * @queue: TX queue index 2092 */ 2093 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 2094 struct stmmac_dma_conf *dma_conf, 2095 u32 queue) 2096 { 2097 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2098 int i; 2099 2100 tx_q->xsk_frames_done = 0; 2101 2102 for (i = 0; i < dma_conf->dma_tx_size; i++) 2103 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 2104 2105 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 2106 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2107 tx_q->xsk_frames_done = 0; 2108 tx_q->xsk_pool = NULL; 2109 } 2110 } 2111 2112 /** 2113 * stmmac_free_tx_skbufs - free TX skb buffers 2114 * @priv: private structure 2115 */ 2116 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 2117 { 2118 u8 tx_queue_cnt = priv->plat->tx_queues_to_use; 2119 u8 queue; 2120 2121 for (queue = 0; queue < tx_queue_cnt; queue++) 2122 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 2123 } 2124 2125 /** 2126 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 2127 * @priv: private structure 2128 * @dma_conf: structure to take the dma data 2129 * @queue: RX queue index 2130 */ 2131 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 2132 struct stmmac_dma_conf *dma_conf, 2133 u32 queue) 2134 { 2135 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2136 size_t size; 2137 void *addr; 2138 2139 /* Release the DMA RX socket buffers */ 2140 if (rx_q->xsk_pool) 2141 dma_free_rx_xskbufs(priv, dma_conf, queue); 2142 else 2143 dma_free_rx_skbufs(priv, dma_conf, queue); 2144 2145 rx_q->buf_alloc_num = 0; 2146 rx_q->xsk_pool = NULL; 2147 2148 /* Free DMA regions of consistent memory previously allocated */ 2149 if (priv->extend_desc) 2150 addr = rx_q->dma_erx; 2151 else 2152 addr = rx_q->dma_rx; 2153 2154 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2155 2156 dma_free_coherent(priv->device, size, addr, rx_q->dma_rx_phy); 2157 2158 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 2159 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 2160 2161 kfree(rx_q->buf_pool); 2162 if (rx_q->page_pool) 2163 page_pool_destroy(rx_q->page_pool); 2164 } 2165 2166 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 2167 struct stmmac_dma_conf *dma_conf) 2168 { 2169 u8 rx_count = priv->plat->rx_queues_to_use; 2170 u8 queue; 2171 2172 /* Free RX queue resources */ 2173 for (queue = 0; queue < rx_count; queue++) 2174 __free_dma_rx_desc_resources(priv, dma_conf, queue); 2175 } 2176 2177 /** 2178 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 2179 * @priv: private structure 2180 * @dma_conf: structure to take the dma data 2181 * @queue: TX queue index 2182 */ 2183 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 2184 struct stmmac_dma_conf *dma_conf, 2185 u32 queue) 2186 { 2187 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2188 size_t size; 2189 void *addr; 2190 2191 /* Release the DMA TX socket buffers */ 2192 dma_free_tx_skbufs(priv, dma_conf, queue); 2193 2194 if (priv->extend_desc) { 2195 addr = tx_q->dma_etx; 2196 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 2197 addr = tx_q->dma_entx; 2198 } else { 2199 addr = tx_q->dma_tx; 2200 } 2201 2202 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2203 2204 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 2205 2206 kfree(tx_q->tx_skbuff_dma); 2207 kfree(tx_q->tx_skbuff); 2208 } 2209 2210 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2211 struct stmmac_dma_conf *dma_conf) 2212 { 2213 u8 tx_count = priv->plat->tx_queues_to_use; 2214 u8 queue; 2215 2216 /* Free TX queue resources */ 2217 for (queue = 0; queue < tx_count; queue++) 2218 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2219 } 2220 2221 /** 2222 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2223 * @priv: private structure 2224 * @dma_conf: structure to take the dma data 2225 * @queue: RX queue index 2226 * Description: according to which descriptor can be used (extend or basic) 2227 * this function allocates the resources for TX and RX paths. In case of 2228 * reception, for example, it pre-allocated the RX socket buffer in order to 2229 * allow zero-copy mechanism. 2230 */ 2231 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2232 struct stmmac_dma_conf *dma_conf, 2233 u32 queue) 2234 { 2235 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2236 struct stmmac_channel *ch = &priv->channel[queue]; 2237 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2238 struct page_pool_params pp_params = { 0 }; 2239 unsigned int dma_buf_sz_pad, num_pages; 2240 unsigned int napi_id; 2241 size_t size; 2242 void *addr; 2243 int ret; 2244 2245 dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + 2246 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2247 num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE); 2248 2249 rx_q->queue_index = queue; 2250 rx_q->priv_data = priv; 2251 rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE; 2252 2253 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2254 pp_params.pool_size = dma_conf->dma_rx_size; 2255 pp_params.order = order_base_2(num_pages); 2256 pp_params.nid = dev_to_node(priv->device); 2257 pp_params.dev = priv->device; 2258 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2259 pp_params.offset = stmmac_rx_offset(priv); 2260 pp_params.max_len = dma_conf->dma_buf_sz; 2261 2262 if (priv->sph_active) { 2263 pp_params.offset = 0; 2264 pp_params.max_len += stmmac_rx_offset(priv); 2265 } 2266 2267 rx_q->page_pool = page_pool_create(&pp_params); 2268 if (IS_ERR(rx_q->page_pool)) { 2269 ret = PTR_ERR(rx_q->page_pool); 2270 rx_q->page_pool = NULL; 2271 return ret; 2272 } 2273 2274 rx_q->buf_pool = kzalloc_objs(*rx_q->buf_pool, dma_conf->dma_rx_size); 2275 if (!rx_q->buf_pool) 2276 return -ENOMEM; 2277 2278 size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; 2279 2280 addr = dma_alloc_coherent(priv->device, size, &rx_q->dma_rx_phy, 2281 GFP_KERNEL); 2282 if (!addr) 2283 return -ENOMEM; 2284 2285 if (priv->extend_desc) 2286 rx_q->dma_erx = addr; 2287 else 2288 rx_q->dma_rx = addr; 2289 2290 if (stmmac_xdp_is_enabled(priv) && 2291 test_bit(queue, priv->af_xdp_zc_qps)) 2292 napi_id = ch->rxtx_napi.napi_id; 2293 else 2294 napi_id = ch->rx_napi.napi_id; 2295 2296 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, queue, napi_id); 2297 if (ret) { 2298 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2299 return -EINVAL; 2300 } 2301 2302 return 0; 2303 } 2304 2305 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2306 struct stmmac_dma_conf *dma_conf) 2307 { 2308 u8 rx_count = priv->plat->rx_queues_to_use; 2309 u8 queue; 2310 int ret; 2311 2312 /* RX queues buffers and DMA */ 2313 for (queue = 0; queue < rx_count; queue++) { 2314 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2315 if (ret) 2316 goto err_dma; 2317 } 2318 2319 return 0; 2320 2321 err_dma: 2322 free_dma_rx_desc_resources(priv, dma_conf); 2323 2324 return ret; 2325 } 2326 2327 /** 2328 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2329 * @priv: private structure 2330 * @dma_conf: structure to take the dma data 2331 * @queue: TX queue index 2332 * Description: according to which descriptor can be used (extend or basic) 2333 * this function allocates the resources for TX and RX paths. In case of 2334 * reception, for example, it pre-allocated the RX socket buffer in order to 2335 * allow zero-copy mechanism. 2336 */ 2337 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2338 struct stmmac_dma_conf *dma_conf, 2339 u32 queue) 2340 { 2341 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2342 size_t size; 2343 void *addr; 2344 2345 tx_q->queue_index = queue; 2346 tx_q->priv_data = priv; 2347 2348 tx_q->tx_skbuff_dma = kzalloc_objs(*tx_q->tx_skbuff_dma, 2349 dma_conf->dma_tx_size); 2350 if (!tx_q->tx_skbuff_dma) 2351 return -ENOMEM; 2352 2353 tx_q->tx_skbuff = kzalloc_objs(struct sk_buff *, dma_conf->dma_tx_size); 2354 if (!tx_q->tx_skbuff) 2355 return -ENOMEM; 2356 2357 size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; 2358 2359 addr = dma_alloc_coherent(priv->device, size, 2360 &tx_q->dma_tx_phy, GFP_KERNEL); 2361 if (!addr) 2362 return -ENOMEM; 2363 2364 if (priv->extend_desc) 2365 tx_q->dma_etx = addr; 2366 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2367 tx_q->dma_entx = addr; 2368 else 2369 tx_q->dma_tx = addr; 2370 2371 return 0; 2372 } 2373 2374 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2375 struct stmmac_dma_conf *dma_conf) 2376 { 2377 u8 tx_count = priv->plat->tx_queues_to_use; 2378 u8 queue; 2379 int ret; 2380 2381 /* TX queues buffers and DMA */ 2382 for (queue = 0; queue < tx_count; queue++) { 2383 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2384 if (ret) 2385 goto err_dma; 2386 } 2387 2388 return 0; 2389 2390 err_dma: 2391 free_dma_tx_desc_resources(priv, dma_conf); 2392 return ret; 2393 } 2394 2395 /** 2396 * alloc_dma_desc_resources - alloc TX/RX resources. 2397 * @priv: private structure 2398 * @dma_conf: structure to take the dma data 2399 * Description: according to which descriptor can be used (extend or basic) 2400 * this function allocates the resources for TX and RX paths. In case of 2401 * reception, for example, it pre-allocated the RX socket buffer in order to 2402 * allow zero-copy mechanism. 2403 */ 2404 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2405 struct stmmac_dma_conf *dma_conf) 2406 { 2407 /* RX Allocation */ 2408 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2409 2410 if (ret) 2411 return ret; 2412 2413 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2414 2415 return ret; 2416 } 2417 2418 /** 2419 * free_dma_desc_resources - free dma desc resources 2420 * @priv: private structure 2421 * @dma_conf: structure to take the dma data 2422 */ 2423 static void free_dma_desc_resources(struct stmmac_priv *priv, 2424 struct stmmac_dma_conf *dma_conf) 2425 { 2426 /* Release the DMA TX socket buffers */ 2427 free_dma_tx_desc_resources(priv, dma_conf); 2428 2429 /* Release the DMA RX socket buffers later 2430 * to ensure all pending XDP_TX buffers are returned. 2431 */ 2432 free_dma_rx_desc_resources(priv, dma_conf); 2433 } 2434 2435 /** 2436 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2437 * @priv: driver private structure 2438 * Description: It is used for enabling the rx queues in the MAC 2439 */ 2440 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2441 { 2442 u8 rx_queues_count = priv->plat->rx_queues_to_use; 2443 u8 queue; 2444 u8 mode; 2445 2446 for (queue = 0; queue < rx_queues_count; queue++) { 2447 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2448 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2449 } 2450 } 2451 2452 /** 2453 * stmmac_start_rx_dma - start RX DMA channel 2454 * @priv: driver private structure 2455 * @chan: RX channel index 2456 * Description: 2457 * This starts a RX DMA channel 2458 */ 2459 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2460 { 2461 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2462 stmmac_start_rx(priv, priv->ioaddr, chan); 2463 } 2464 2465 /** 2466 * stmmac_start_tx_dma - start TX DMA channel 2467 * @priv: driver private structure 2468 * @chan: TX channel index 2469 * Description: 2470 * This starts a TX DMA channel 2471 */ 2472 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2473 { 2474 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2475 stmmac_start_tx(priv, priv->ioaddr, chan); 2476 } 2477 2478 /** 2479 * stmmac_stop_rx_dma - stop RX DMA channel 2480 * @priv: driver private structure 2481 * @chan: RX channel index 2482 * Description: 2483 * This stops a RX DMA channel 2484 */ 2485 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2486 { 2487 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2488 stmmac_stop_rx(priv, priv->ioaddr, chan); 2489 } 2490 2491 /** 2492 * stmmac_stop_tx_dma - stop TX DMA channel 2493 * @priv: driver private structure 2494 * @chan: TX channel index 2495 * Description: 2496 * This stops a TX DMA channel 2497 */ 2498 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2499 { 2500 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2501 stmmac_stop_tx(priv, priv->ioaddr, chan); 2502 } 2503 2504 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2505 { 2506 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2507 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2508 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2509 u8 chan; 2510 2511 for (chan = 0; chan < dma_csr_ch; chan++) { 2512 struct stmmac_channel *ch = &priv->channel[chan]; 2513 unsigned long flags; 2514 2515 spin_lock_irqsave(&ch->lock, flags); 2516 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2517 spin_unlock_irqrestore(&ch->lock, flags); 2518 } 2519 } 2520 2521 /** 2522 * stmmac_start_all_dma - start all RX and TX DMA channels 2523 * @priv: driver private structure 2524 * Description: 2525 * This starts all the RX and TX DMA channels 2526 */ 2527 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2528 { 2529 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2530 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2531 u8 chan; 2532 2533 for (chan = 0; chan < rx_channels_count; chan++) 2534 stmmac_start_rx_dma(priv, chan); 2535 2536 for (chan = 0; chan < tx_channels_count; chan++) 2537 stmmac_start_tx_dma(priv, chan); 2538 } 2539 2540 /** 2541 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2542 * @priv: driver private structure 2543 * Description: 2544 * This stops the RX and TX DMA channels 2545 */ 2546 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2547 { 2548 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2549 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2550 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2551 u8 chan; 2552 2553 for (chan = 0; chan < rx_channels_count; chan++) 2554 stmmac_stop_rx_dma(priv, chan); 2555 2556 for (chan = 0; chan < tx_channels_count; chan++) 2557 stmmac_stop_tx_dma(priv, chan); 2558 2559 for (chan = 0; chan < dma_csr_ch; chan++) 2560 stmmac_deinit_chan(priv, priv->ioaddr, chan); 2561 } 2562 2563 /** 2564 * stmmac_dma_operation_mode - HW DMA operation mode 2565 * @priv: driver private structure 2566 * Description: it is used for configuring the DMA operation mode register in 2567 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2568 */ 2569 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2570 { 2571 u8 rx_channels_count = priv->plat->rx_queues_to_use; 2572 u8 tx_channels_count = priv->plat->tx_queues_to_use; 2573 int rxfifosz = priv->plat->rx_fifo_size; 2574 int txfifosz = priv->plat->tx_fifo_size; 2575 u32 txmode = 0; 2576 u32 rxmode = 0; 2577 u8 qmode = 0; 2578 u8 chan; 2579 2580 if (rxfifosz == 0) 2581 rxfifosz = priv->dma_cap.rx_fifo_size; 2582 if (txfifosz == 0) 2583 txfifosz = priv->dma_cap.tx_fifo_size; 2584 2585 /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ 2586 if (dwmac_is_xmac(priv->plat->core_type)) { 2587 rxfifosz /= rx_channels_count; 2588 txfifosz /= tx_channels_count; 2589 } 2590 2591 if (priv->plat->force_thresh_dma_mode) { 2592 txmode = tc; 2593 rxmode = tc; 2594 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2595 /* 2596 * In case of GMAC, SF mode can be enabled 2597 * to perform the TX COE in HW. This depends on: 2598 * 1) TX COE if actually supported 2599 * 2) There is no bugged Jumbo frame support 2600 * that needs to not insert csum in the TDES. 2601 */ 2602 txmode = SF_DMA_MODE; 2603 rxmode = SF_DMA_MODE; 2604 priv->xstats.threshold = SF_DMA_MODE; 2605 } else { 2606 txmode = tc; 2607 rxmode = SF_DMA_MODE; 2608 } 2609 2610 /* configure all channels */ 2611 for (chan = 0; chan < rx_channels_count; chan++) { 2612 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2613 2614 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2615 2616 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2617 rxfifosz, qmode); 2618 2619 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 2620 } 2621 2622 for (chan = 0; chan < tx_channels_count; chan++) { 2623 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2624 2625 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2626 txfifosz, qmode); 2627 } 2628 } 2629 2630 static void stmmac_xsk_request_timestamp(void *_priv) 2631 { 2632 struct stmmac_metadata_request *meta_req = _priv; 2633 2634 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2635 *meta_req->set_ic = true; 2636 } 2637 2638 static u64 stmmac_xsk_fill_timestamp(void *_priv) 2639 { 2640 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2641 struct stmmac_priv *priv = tx_compl->priv; 2642 struct dma_desc *desc = tx_compl->desc; 2643 bool found = false; 2644 u64 ns = 0; 2645 2646 if (!priv->hwts_tx_en) 2647 return 0; 2648 2649 /* check tx tstamp status */ 2650 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2651 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2652 found = true; 2653 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2654 found = true; 2655 } 2656 2657 if (found) { 2658 ns -= priv->plat->cdc_error_adj; 2659 return ns_to_ktime(ns); 2660 } 2661 2662 return 0; 2663 } 2664 2665 static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv) 2666 { 2667 struct timespec64 ts = ns_to_timespec64(launch_time); 2668 struct stmmac_metadata_request *meta_req = _priv; 2669 2670 if (meta_req->tbs & STMMAC_TBS_EN) 2671 stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec, 2672 ts.tv_nsec); 2673 } 2674 2675 static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2676 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2677 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2678 .tmo_request_launch_time = stmmac_xsk_request_launch_time, 2679 }; 2680 2681 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2682 { 2683 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2684 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2685 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2686 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 2687 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2688 unsigned int entry = tx_q->cur_tx; 2689 struct dma_desc *tx_desc = NULL; 2690 struct xdp_desc xdp_desc; 2691 bool work_done = true; 2692 u32 tx_set_ic_bit = 0; 2693 2694 /* Avoids TX time-out as we are sharing with slow path */ 2695 txq_trans_cond_update(nq); 2696 2697 budget = min(budget, stmmac_tx_avail(priv, queue)); 2698 2699 for (; budget > 0; budget--) { 2700 struct stmmac_metadata_request meta_req; 2701 struct xsk_tx_metadata *meta = NULL; 2702 dma_addr_t dma_addr; 2703 bool set_ic; 2704 2705 /* We are sharing with slow path and stop XSK TX desc submission when 2706 * available TX ring is less than threshold. 2707 */ 2708 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2709 !netif_carrier_ok(priv->dev)) { 2710 work_done = false; 2711 break; 2712 } 2713 2714 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2715 break; 2716 2717 if (priv->est && priv->est->enable && 2718 priv->est->max_sdu[queue] && 2719 xdp_desc.len > priv->est->max_sdu[queue]) { 2720 priv->xstats.max_sdu_txq_drop[queue]++; 2721 continue; 2722 } 2723 2724 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 2725 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2726 meta = xsk_buff_get_metadata(pool, xdp_desc.addr, 2727 xdp_desc.options); 2728 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2729 2730 /* To return XDP buffer to XSK pool, we simple call 2731 * xsk_tx_completed(), so we don't need to fill up 2732 * 'buf' and 'xdpf'. 2733 */ 2734 stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_XSK_TX, 2735 0, xdp_desc.len, false); 2736 stmmac_set_tx_dma_last_segment(tx_q, entry); 2737 2738 tx_q->xdpf[entry] = NULL; 2739 2740 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2741 2742 tx_q->tx_count_frames++; 2743 2744 if (!priv->tx_coal_frames[queue]) 2745 set_ic = false; 2746 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2747 set_ic = true; 2748 else 2749 set_ic = false; 2750 2751 meta_req.priv = priv; 2752 meta_req.tx_desc = tx_desc; 2753 meta_req.set_ic = &set_ic; 2754 meta_req.tbs = tx_q->tbs; 2755 meta_req.edesc = &tx_q->dma_entx[entry]; 2756 xsk_tx_metadata_request(pool, &meta, 2757 &stmmac_xsk_tx_metadata_ops, &meta_req); 2758 if (set_ic) { 2759 tx_q->tx_count_frames = 0; 2760 stmmac_set_tx_ic(priv, tx_desc); 2761 tx_set_ic_bit++; 2762 } 2763 2764 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2765 csum, priv->descriptor_mode, true, true, 2766 xdp_desc.len); 2767 2768 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 2769 2770 xsk_tx_metadata_to_compl(meta, 2771 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2772 2773 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2774 entry = tx_q->cur_tx; 2775 } 2776 u64_stats_update_begin(&txq_stats->napi_syncp); 2777 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2778 u64_stats_update_end(&txq_stats->napi_syncp); 2779 2780 if (tx_desc) { 2781 stmmac_flush_tx_descriptors(priv, queue); 2782 xsk_tx_release(pool); 2783 } 2784 2785 /* Return true if all of the 3 conditions are met 2786 * a) TX Budget is still available 2787 * b) work_done = true when XSK TX desc peek is empty (no more 2788 * pending XSK TX for transmission) 2789 */ 2790 return !!budget && work_done; 2791 } 2792 2793 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2794 { 2795 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2796 tc += 64; 2797 2798 if (priv->plat->force_thresh_dma_mode) 2799 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2800 else 2801 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2802 chan); 2803 2804 priv->xstats.threshold = tc; 2805 } 2806 } 2807 2808 /** 2809 * stmmac_tx_clean - to manage the transmission completion 2810 * @priv: driver private structure 2811 * @budget: napi budget limiting this functions packet handling 2812 * @queue: TX queue index 2813 * @pending_packets: signal to arm the TX coal timer 2814 * Description: it reclaims the transmit resources after transmission completes. 2815 * If some packets still needs to be handled, due to TX coalesce, set 2816 * pending_packets to true to make NAPI arm the TX coal timer. 2817 */ 2818 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2819 bool *pending_packets) 2820 { 2821 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2822 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2823 unsigned int bytes_compl = 0, pkts_compl = 0; 2824 unsigned int entry, xmits = 0, count = 0; 2825 u32 tx_packets = 0, tx_errors = 0; 2826 2827 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2828 2829 tx_q->xsk_frames_done = 0; 2830 2831 entry = tx_q->dirty_tx; 2832 2833 /* Try to clean all TX complete frame in 1 shot */ 2834 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2835 struct xdp_frame *xdpf; 2836 struct sk_buff *skb; 2837 struct dma_desc *p; 2838 int status; 2839 2840 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2841 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2842 xdpf = tx_q->xdpf[entry]; 2843 skb = NULL; 2844 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2845 xdpf = NULL; 2846 skb = tx_q->tx_skbuff[entry]; 2847 } else { 2848 xdpf = NULL; 2849 skb = NULL; 2850 } 2851 2852 p = stmmac_get_tx_desc(priv, tx_q, entry); 2853 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2854 /* Check if the descriptor is owned by the DMA */ 2855 if (unlikely(status & tx_dma_own)) 2856 break; 2857 2858 count++; 2859 2860 /* Make sure descriptor fields are read after reading 2861 * the own bit. 2862 */ 2863 dma_rmb(); 2864 2865 /* Just consider the last segment and ...*/ 2866 if (likely(!(status & tx_not_ls))) { 2867 /* ... verify the status error condition */ 2868 if (unlikely(status & tx_err)) { 2869 tx_errors++; 2870 if (unlikely(status & tx_err_bump_tc)) 2871 stmmac_bump_dma_threshold(priv, queue); 2872 } else { 2873 tx_packets++; 2874 } 2875 if (skb) { 2876 stmmac_get_tx_hwtstamp(priv, p, skb); 2877 } else if (tx_q->xsk_pool && 2878 xp_tx_metadata_enabled(tx_q->xsk_pool)) { 2879 struct stmmac_xsk_tx_complete tx_compl = { 2880 .priv = priv, 2881 .desc = p, 2882 }; 2883 2884 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2885 &stmmac_xsk_tx_metadata_ops, 2886 &tx_compl); 2887 } 2888 } 2889 2890 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2891 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2892 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2893 dma_unmap_page(priv->device, 2894 tx_q->tx_skbuff_dma[entry].buf, 2895 tx_q->tx_skbuff_dma[entry].len, 2896 DMA_TO_DEVICE); 2897 else 2898 dma_unmap_single(priv->device, 2899 tx_q->tx_skbuff_dma[entry].buf, 2900 tx_q->tx_skbuff_dma[entry].len, 2901 DMA_TO_DEVICE); 2902 tx_q->tx_skbuff_dma[entry].buf = 0; 2903 tx_q->tx_skbuff_dma[entry].len = 0; 2904 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2905 } 2906 2907 /* This looks at tx_q->tx_skbuff_dma[tx_q->dirty_tx].is_jumbo 2908 * and tx_q->tx_skbuff_dma[tx_q->dirty_tx].last_segment 2909 */ 2910 stmmac_clean_desc3(priv, tx_q, p); 2911 2912 tx_q->tx_skbuff_dma[entry].last_segment = false; 2913 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2914 2915 if (xdpf && 2916 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2917 xdp_return_frame_rx_napi(xdpf); 2918 tx_q->xdpf[entry] = NULL; 2919 } 2920 2921 if (xdpf && 2922 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2923 xdp_return_frame(xdpf); 2924 tx_q->xdpf[entry] = NULL; 2925 } 2926 2927 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2928 tx_q->xsk_frames_done++; 2929 2930 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2931 if (likely(skb)) { 2932 pkts_compl++; 2933 bytes_compl += skb->len; 2934 dev_consume_skb_any(skb); 2935 tx_q->tx_skbuff[entry] = NULL; 2936 } 2937 } 2938 2939 stmmac_release_tx_desc(priv, p, priv->descriptor_mode); 2940 2941 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 2942 } 2943 tx_q->dirty_tx = entry; 2944 2945 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2946 pkts_compl, bytes_compl); 2947 2948 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2949 queue))) && 2950 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2951 2952 netif_dbg(priv, tx_done, priv->dev, 2953 "%s: restart transmit\n", __func__); 2954 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2955 } 2956 2957 if (tx_q->xsk_pool) { 2958 bool work_done; 2959 2960 if (tx_q->xsk_frames_done) 2961 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2962 2963 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2964 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2965 2966 /* For XSK TX, we try to send as many as possible. 2967 * If XSK work done (XSK TX desc empty and budget still 2968 * available), return "budget - 1" to reenable TX IRQ. 2969 * Else, return "budget" to make NAPI continue polling. 2970 */ 2971 work_done = stmmac_xdp_xmit_zc(priv, queue, 2972 STMMAC_XSK_TX_BUDGET_MAX); 2973 if (work_done) 2974 xmits = budget - 1; 2975 else 2976 xmits = budget; 2977 } 2978 2979 if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode) 2980 stmmac_restart_sw_lpi_timer(priv); 2981 2982 /* We still have pending packets, let's call for a new scheduling */ 2983 if (tx_q->dirty_tx != tx_q->cur_tx) 2984 *pending_packets = true; 2985 2986 u64_stats_update_begin(&txq_stats->napi_syncp); 2987 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2988 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2989 u64_stats_inc(&txq_stats->napi.tx_clean); 2990 u64_stats_update_end(&txq_stats->napi_syncp); 2991 2992 priv->xstats.tx_errors += tx_errors; 2993 2994 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2995 2996 /* Combine decisions from TX clean and XSK TX */ 2997 return max(count, xmits); 2998 } 2999 3000 /** 3001 * stmmac_tx_err - to manage the tx error 3002 * @priv: driver private structure 3003 * @chan: channel index 3004 * Description: it cleans the descriptors and restarts the transmission 3005 * in case of transmission errors. 3006 */ 3007 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 3008 { 3009 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3010 3011 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 3012 3013 stmmac_stop_tx_dma(priv, chan); 3014 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 3015 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 3016 stmmac_reset_tx_queue(priv, chan); 3017 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3018 tx_q->dma_tx_phy, chan); 3019 stmmac_start_tx_dma(priv, chan); 3020 3021 priv->xstats.tx_errors++; 3022 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 3023 } 3024 3025 /** 3026 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 3027 * @priv: driver private structure 3028 * @txmode: TX operating mode 3029 * @rxmode: RX operating mode 3030 * @chan: channel index 3031 * Description: it is used for configuring of the DMA operation mode in 3032 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 3033 * mode. 3034 */ 3035 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 3036 u32 rxmode, u32 chan) 3037 { 3038 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 3039 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 3040 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3041 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3042 int rxfifosz = priv->plat->rx_fifo_size; 3043 int txfifosz = priv->plat->tx_fifo_size; 3044 3045 if (rxfifosz == 0) 3046 rxfifosz = priv->dma_cap.rx_fifo_size; 3047 if (txfifosz == 0) 3048 txfifosz = priv->dma_cap.tx_fifo_size; 3049 3050 /* Adjust for real per queue fifo size */ 3051 rxfifosz /= rx_channels_count; 3052 txfifosz /= tx_channels_count; 3053 3054 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 3055 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 3056 } 3057 3058 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 3059 { 3060 int ret; 3061 3062 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 3063 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 3064 if (ret && (ret != -EINVAL)) { 3065 stmmac_global_err(priv); 3066 return true; 3067 } 3068 3069 return false; 3070 } 3071 3072 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 3073 { 3074 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 3075 &priv->xstats, chan, dir); 3076 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 3077 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3078 struct stmmac_channel *ch = &priv->channel[chan]; 3079 struct napi_struct *rx_napi; 3080 struct napi_struct *tx_napi; 3081 unsigned long flags; 3082 3083 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 3084 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3085 3086 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 3087 if (napi_schedule_prep(rx_napi)) { 3088 spin_lock_irqsave(&ch->lock, flags); 3089 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 3090 spin_unlock_irqrestore(&ch->lock, flags); 3091 __napi_schedule(rx_napi); 3092 } 3093 } 3094 3095 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 3096 if (napi_schedule_prep(tx_napi)) { 3097 spin_lock_irqsave(&ch->lock, flags); 3098 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 3099 spin_unlock_irqrestore(&ch->lock, flags); 3100 __napi_schedule(tx_napi); 3101 } 3102 } 3103 3104 return status; 3105 } 3106 3107 /** 3108 * stmmac_dma_interrupt - DMA ISR 3109 * @priv: driver private structure 3110 * Description: this is the DMA ISR. It is called by the main ISR. 3111 * It calls the dwmac dma routine and schedule poll method in case of some 3112 * work can be done. 3113 */ 3114 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 3115 { 3116 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3117 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3118 u8 channels_to_check = tx_channel_count > rx_channel_count ? 3119 tx_channel_count : rx_channel_count; 3120 int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 3121 u8 chan; 3122 3123 /* Make sure we never check beyond our status buffer. */ 3124 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 3125 channels_to_check = ARRAY_SIZE(status); 3126 3127 for (chan = 0; chan < channels_to_check; chan++) 3128 status[chan] = stmmac_napi_check(priv, chan, 3129 DMA_DIR_RXTX); 3130 3131 for (chan = 0; chan < tx_channel_count; chan++) { 3132 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 3133 /* Try to bump up the dma threshold on this failure */ 3134 stmmac_bump_dma_threshold(priv, chan); 3135 } else if (unlikely(status[chan] == tx_hard_error)) { 3136 stmmac_tx_err(priv, chan); 3137 } 3138 } 3139 } 3140 3141 /** 3142 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 3143 * @priv: driver private structure 3144 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 3145 */ 3146 static void stmmac_mmc_setup(struct stmmac_priv *priv) 3147 { 3148 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 3149 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 3150 3151 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 3152 3153 if (priv->dma_cap.rmon) { 3154 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 3155 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 3156 } else 3157 netdev_info(priv->dev, "No MAC Management Counters available\n"); 3158 } 3159 3160 /** 3161 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 3162 * @priv: driver private structure 3163 * Description: 3164 * new GMAC chip generations have a new register to indicate the 3165 * presence of the optional feature/functions. 3166 * This can be also used to override the value passed through the 3167 * platform and necessary for old MAC10/100 and GMAC chips. 3168 */ 3169 static int stmmac_get_hw_features(struct stmmac_priv *priv) 3170 { 3171 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 3172 } 3173 3174 /** 3175 * stmmac_check_ether_addr - check if the MAC addr is valid 3176 * @priv: driver private structure 3177 * Description: 3178 * it is to verify if the MAC address is valid, in case of failures it 3179 * generates a random MAC address 3180 */ 3181 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 3182 { 3183 u8 addr[ETH_ALEN]; 3184 3185 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 3186 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 3187 if (is_valid_ether_addr(addr)) 3188 eth_hw_addr_set(priv->dev, addr); 3189 else 3190 eth_hw_addr_random(priv->dev); 3191 dev_info(priv->device, "device MAC address %pM\n", 3192 priv->dev->dev_addr); 3193 } 3194 } 3195 3196 int stmmac_get_phy_intf_sel(phy_interface_t interface) 3197 { 3198 int phy_intf_sel = -EINVAL; 3199 3200 if (interface == PHY_INTERFACE_MODE_MII || 3201 interface == PHY_INTERFACE_MODE_GMII) 3202 phy_intf_sel = PHY_INTF_SEL_GMII_MII; 3203 else if (phy_interface_mode_is_rgmii(interface)) 3204 phy_intf_sel = PHY_INTF_SEL_RGMII; 3205 else if (interface == PHY_INTERFACE_MODE_RMII) 3206 phy_intf_sel = PHY_INTF_SEL_RMII; 3207 else if (interface == PHY_INTERFACE_MODE_REVMII) 3208 phy_intf_sel = PHY_INTF_SEL_REVMII; 3209 3210 return phy_intf_sel; 3211 } 3212 EXPORT_SYMBOL_GPL(stmmac_get_phy_intf_sel); 3213 3214 static int stmmac_prereset_configure(struct stmmac_priv *priv) 3215 { 3216 struct plat_stmmacenet_data *plat_dat = priv->plat; 3217 phy_interface_t interface; 3218 struct phylink_pcs *pcs; 3219 int phy_intf_sel, ret; 3220 3221 if (!plat_dat->set_phy_intf_sel) 3222 return 0; 3223 3224 interface = plat_dat->phy_interface; 3225 3226 /* Check whether this mode uses a PCS */ 3227 pcs = stmmac_mac_select_pcs(&priv->phylink_config, interface); 3228 if (priv->integrated_pcs && pcs == &priv->integrated_pcs->pcs) { 3229 /* Request the phy_intf_sel from the integrated PCS */ 3230 phy_intf_sel = stmmac_integrated_pcs_get_phy_intf_sel(pcs, 3231 interface); 3232 } else { 3233 phy_intf_sel = stmmac_get_phy_intf_sel(interface); 3234 } 3235 3236 if (phy_intf_sel < 0) { 3237 netdev_err(priv->dev, 3238 "failed to get phy_intf_sel for %s: %pe\n", 3239 phy_modes(interface), ERR_PTR(phy_intf_sel)); 3240 return phy_intf_sel; 3241 } 3242 3243 ret = plat_dat->set_phy_intf_sel(plat_dat->bsp_priv, phy_intf_sel); 3244 if (ret == -EINVAL) 3245 netdev_err(priv->dev, "platform does not support %s\n", 3246 phy_modes(interface)); 3247 else if (ret < 0) 3248 netdev_err(priv->dev, 3249 "platform failed to set interface %s: %pe\n", 3250 phy_modes(interface), ERR_PTR(ret)); 3251 3252 return ret; 3253 } 3254 3255 /** 3256 * stmmac_init_dma_engine - DMA init. 3257 * @priv: driver private structure 3258 * Description: 3259 * It inits the DMA invoking the specific MAC/GMAC callback. 3260 * Some DMA parameters can be passed from the platform; 3261 * in case of these are not passed a default is kept for the MAC or GMAC. 3262 */ 3263 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 3264 { 3265 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3266 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3267 u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); 3268 struct stmmac_rx_queue *rx_q; 3269 struct stmmac_tx_queue *tx_q; 3270 int ret = 0; 3271 u8 chan; 3272 3273 ret = stmmac_prereset_configure(priv); 3274 if (ret) 3275 return ret; 3276 3277 ret = stmmac_reset(priv); 3278 if (ret) { 3279 netdev_err(priv->dev, "Failed to reset the dma\n"); 3280 return ret; 3281 } 3282 3283 /* DMA Configuration */ 3284 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); 3285 3286 if (priv->plat->axi) 3287 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3288 3289 /* DMA CSR Channel configuration */ 3290 for (chan = 0; chan < dma_csr_ch; chan++) { 3291 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3292 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3293 } 3294 3295 /* DMA RX Channel Configuration */ 3296 for (chan = 0; chan < rx_channels_count; chan++) { 3297 rx_q = &priv->dma_conf.rx_queue[chan]; 3298 3299 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3300 rx_q->dma_rx_phy, chan); 3301 3302 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 3303 rx_q->buf_alloc_num); 3304 } 3305 3306 /* DMA TX Channel Configuration */ 3307 for (chan = 0; chan < tx_channels_count; chan++) { 3308 tx_q = &priv->dma_conf.tx_queue[chan]; 3309 3310 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3311 tx_q->dma_tx_phy, chan); 3312 3313 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 3314 } 3315 3316 return ret; 3317 } 3318 3319 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3320 { 3321 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3322 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3323 struct stmmac_channel *ch; 3324 struct napi_struct *napi; 3325 3326 if (!tx_coal_timer) 3327 return; 3328 3329 ch = &priv->channel[queue]; 3330 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3331 3332 /* Arm timer only if napi is not already scheduled. 3333 * Try to cancel any timer if napi is scheduled, timer will be armed 3334 * again in the next scheduled napi. 3335 */ 3336 if (unlikely(!napi_is_scheduled(napi))) { 3337 if (unlikely(!(hrtimer_active(&tx_q->txtimer)))) 3338 hrtimer_start(&tx_q->txtimer, 3339 STMMAC_COAL_TIMER(tx_coal_timer), 3340 HRTIMER_MODE_REL); 3341 } else { 3342 hrtimer_try_to_cancel(&tx_q->txtimer); 3343 } 3344 } 3345 3346 /** 3347 * stmmac_tx_timer - mitigation sw timer for tx. 3348 * @t: data pointer 3349 * Description: 3350 * This is the timer handler to directly invoke the stmmac_tx_clean. 3351 */ 3352 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3353 { 3354 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3355 struct stmmac_priv *priv = tx_q->priv_data; 3356 struct stmmac_channel *ch; 3357 struct napi_struct *napi; 3358 3359 ch = &priv->channel[tx_q->queue_index]; 3360 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3361 3362 if (likely(napi_schedule_prep(napi))) { 3363 unsigned long flags; 3364 3365 spin_lock_irqsave(&ch->lock, flags); 3366 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3367 spin_unlock_irqrestore(&ch->lock, flags); 3368 __napi_schedule(napi); 3369 } 3370 3371 return HRTIMER_NORESTART; 3372 } 3373 3374 /** 3375 * stmmac_init_coalesce - init mitigation options. 3376 * @priv: driver private structure 3377 * Description: 3378 * This inits the coalesce parameters: i.e. timer rate, 3379 * timer handler and default threshold used for enabling the 3380 * interrupt on completion bit. 3381 */ 3382 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3383 { 3384 u8 tx_channel_count = priv->plat->tx_queues_to_use; 3385 u8 rx_channel_count = priv->plat->rx_queues_to_use; 3386 u8 chan; 3387 3388 for (chan = 0; chan < tx_channel_count; chan++) { 3389 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3390 3391 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3392 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3393 3394 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3395 } 3396 3397 for (chan = 0; chan < rx_channel_count; chan++) 3398 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3399 } 3400 3401 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3402 { 3403 u8 rx_channels_count = priv->plat->rx_queues_to_use; 3404 u8 tx_channels_count = priv->plat->tx_queues_to_use; 3405 u8 chan; 3406 3407 /* set TX ring length */ 3408 for (chan = 0; chan < tx_channels_count; chan++) 3409 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3410 (priv->dma_conf.dma_tx_size - 1), chan); 3411 3412 /* set RX ring length */ 3413 for (chan = 0; chan < rx_channels_count; chan++) 3414 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3415 (priv->dma_conf.dma_rx_size - 1), chan); 3416 } 3417 3418 /** 3419 * stmmac_set_tx_queue_weight - Set TX queue weight 3420 * @priv: driver private structure 3421 * Description: It is used for setting TX queues weight 3422 */ 3423 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3424 { 3425 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3426 u32 weight; 3427 u8 queue; 3428 3429 for (queue = 0; queue < tx_queues_count; queue++) { 3430 weight = priv->plat->tx_queues_cfg[queue].weight; 3431 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3432 } 3433 } 3434 3435 /** 3436 * stmmac_configure_cbs - Configure CBS in TX queue 3437 * @priv: driver private structure 3438 * Description: It is used for configuring CBS in AVB TX queues 3439 */ 3440 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3441 { 3442 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3443 u32 mode_to_use; 3444 u8 queue; 3445 3446 /* queue 0 is reserved for legacy traffic */ 3447 for (queue = 1; queue < tx_queues_count; queue++) { 3448 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3449 if (mode_to_use == MTL_QUEUE_DCB) 3450 continue; 3451 3452 stmmac_config_cbs(priv, priv->hw, 3453 priv->plat->tx_queues_cfg[queue].send_slope, 3454 priv->plat->tx_queues_cfg[queue].idle_slope, 3455 priv->plat->tx_queues_cfg[queue].high_credit, 3456 priv->plat->tx_queues_cfg[queue].low_credit, 3457 queue); 3458 } 3459 } 3460 3461 /** 3462 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3463 * @priv: driver private structure 3464 * Description: It is used for mapping RX queues to RX dma channels 3465 */ 3466 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3467 { 3468 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3469 u8 queue; 3470 u32 chan; 3471 3472 for (queue = 0; queue < rx_queues_count; queue++) { 3473 chan = priv->plat->rx_queues_cfg[queue].chan; 3474 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3475 } 3476 } 3477 3478 /** 3479 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3480 * @priv: driver private structure 3481 * Description: It is used for configuring the RX Queue Priority 3482 */ 3483 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3484 { 3485 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3486 u8 queue; 3487 u32 prio; 3488 3489 for (queue = 0; queue < rx_queues_count; queue++) { 3490 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3491 continue; 3492 3493 prio = priv->plat->rx_queues_cfg[queue].prio; 3494 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3495 } 3496 } 3497 3498 /** 3499 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3500 * @priv: driver private structure 3501 * Description: It is used for configuring the TX Queue Priority 3502 */ 3503 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3504 { 3505 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3506 u8 queue; 3507 u32 prio; 3508 3509 for (queue = 0; queue < tx_queues_count; queue++) { 3510 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3511 continue; 3512 3513 prio = priv->plat->tx_queues_cfg[queue].prio; 3514 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3515 } 3516 } 3517 3518 /** 3519 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3520 * @priv: driver private structure 3521 * Description: It is used for configuring the RX queue routing 3522 */ 3523 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3524 { 3525 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3526 u8 packet; 3527 u8 queue; 3528 3529 for (queue = 0; queue < rx_queues_count; queue++) { 3530 /* no specific packet type routing specified for the queue */ 3531 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3532 continue; 3533 3534 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3535 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3536 } 3537 } 3538 3539 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3540 { 3541 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3542 priv->rss.enable = false; 3543 return; 3544 } 3545 3546 if (priv->dev->features & NETIF_F_RXHASH) 3547 priv->rss.enable = true; 3548 else 3549 priv->rss.enable = false; 3550 3551 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3552 priv->plat->rx_queues_to_use); 3553 } 3554 3555 /** 3556 * stmmac_mtl_configuration - Configure MTL 3557 * @priv: driver private structure 3558 * Description: It is used for configuring MTL 3559 */ 3560 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3561 { 3562 u8 rx_queues_count = priv->plat->rx_queues_to_use; 3563 u8 tx_queues_count = priv->plat->tx_queues_to_use; 3564 3565 if (tx_queues_count > 1) 3566 stmmac_set_tx_queue_weight(priv); 3567 3568 /* Configure MTL RX algorithms */ 3569 if (rx_queues_count > 1) 3570 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3571 priv->plat->rx_sched_algorithm); 3572 3573 /* Configure MTL TX algorithms */ 3574 if (tx_queues_count > 1) 3575 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3576 priv->plat->tx_sched_algorithm); 3577 3578 /* Configure CBS in AVB TX queues */ 3579 if (tx_queues_count > 1) 3580 stmmac_configure_cbs(priv); 3581 3582 /* Map RX MTL to DMA channels */ 3583 stmmac_rx_queue_dma_chan_map(priv); 3584 3585 /* Enable MAC RX Queues */ 3586 stmmac_mac_enable_rx_queues(priv); 3587 3588 /* Set RX priorities */ 3589 if (rx_queues_count > 1) 3590 stmmac_mac_config_rx_queues_prio(priv); 3591 3592 /* Set TX priorities */ 3593 if (tx_queues_count > 1) 3594 stmmac_mac_config_tx_queues_prio(priv); 3595 3596 /* Set RX routing */ 3597 if (rx_queues_count > 1) 3598 stmmac_mac_config_rx_queues_routing(priv); 3599 3600 /* Receive Side Scaling */ 3601 if (rx_queues_count > 1) 3602 stmmac_mac_config_rss(priv); 3603 } 3604 3605 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3606 { 3607 if (priv->dma_cap.asp) { 3608 netdev_info(priv->dev, "Enabling Safety Features\n"); 3609 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3610 priv->plat->safety_feat_cfg); 3611 } else { 3612 netdev_info(priv->dev, "No Safety Features support found\n"); 3613 } 3614 } 3615 3616 /* STM32MP25xx (dwmac v5.3) states "Do not enable time-based scheduling for 3617 * channels on which the TSO feature is enabled." If we have a skb for a 3618 * channel which has TBS enabled, fall back to software GSO. 3619 */ 3620 static bool stmmac_tso_channel_permitted(struct stmmac_priv *priv, 3621 unsigned int chan) 3622 { 3623 /* TSO and TBS cannot co-exist */ 3624 return !(priv->dma_conf.tx_queue[chan].tbs & STMMAC_TBS_AVAIL); 3625 } 3626 3627 /** 3628 * stmmac_hw_setup - setup mac in a usable state. 3629 * @dev : pointer to the device structure. 3630 * Description: 3631 * this is the main function to setup the HW in a usable state because the 3632 * dma engine is reset, the core registers are configured (e.g. AXI, 3633 * Checksum features, timers). The DMA is ready to start receiving and 3634 * transmitting. 3635 * Return value: 3636 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3637 * file on failure. 3638 */ 3639 static int stmmac_hw_setup(struct net_device *dev) 3640 { 3641 struct stmmac_priv *priv = netdev_priv(dev); 3642 u8 rx_cnt = priv->plat->rx_queues_to_use; 3643 u8 tx_cnt = priv->plat->tx_queues_to_use; 3644 bool sph_en; 3645 u8 chan; 3646 int ret; 3647 3648 /* Make sure RX clock is enabled */ 3649 if (priv->hw->phylink_pcs) 3650 phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs); 3651 3652 /* Note that clk_rx_i must be running for reset to complete. This 3653 * clock may also be required when setting the MAC address. 3654 * 3655 * Block the receive clock stop for LPI mode at the PHY in case 3656 * the link is established with EEE mode active. 3657 */ 3658 phylink_rx_clk_stop_block(priv->phylink); 3659 3660 /* DMA initialization and SW reset */ 3661 ret = stmmac_init_dma_engine(priv); 3662 if (ret < 0) { 3663 phylink_rx_clk_stop_unblock(priv->phylink); 3664 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3665 __func__); 3666 return ret; 3667 } 3668 3669 /* Copy the MAC addr into the HW */ 3670 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3671 phylink_rx_clk_stop_unblock(priv->phylink); 3672 3673 /* Initialize the MAC Core */ 3674 stmmac_core_init(priv, priv->hw, dev); 3675 3676 /* Initialize MTL*/ 3677 stmmac_mtl_configuration(priv); 3678 3679 /* Apply the RX packet parser table */ 3680 if (priv->tc_entries) { 3681 ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries, 3682 priv->tc_entries_max); 3683 if (ret) 3684 return ret; 3685 } 3686 3687 /* Initialize Safety Features */ 3688 stmmac_safety_feat_configuration(priv); 3689 3690 ret = stmmac_rx_ipc(priv, priv->hw); 3691 if (!ret) { 3692 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3693 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3694 priv->hw->rx_csum = 0; 3695 } 3696 3697 /* Enable the MAC Rx/Tx */ 3698 stmmac_mac_set(priv, priv->ioaddr, true); 3699 3700 /* Set the HW DMA mode and the COE */ 3701 stmmac_dma_operation_mode(priv); 3702 3703 stmmac_mmc_setup(priv); 3704 3705 if (priv->use_riwt) { 3706 u32 queue; 3707 3708 for (queue = 0; queue < rx_cnt; queue++) { 3709 if (!priv->rx_riwt[queue]) 3710 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3711 3712 stmmac_rx_watchdog(priv, priv->ioaddr, 3713 priv->rx_riwt[queue], queue); 3714 } 3715 } 3716 3717 /* set TX and RX rings length */ 3718 stmmac_set_rings_length(priv); 3719 3720 /* Enable TSO */ 3721 if (priv->dma_cap.tsoen && priv->plat->flags & STMMAC_FLAG_TSO_EN) { 3722 for (chan = 0; chan < tx_cnt; chan++) { 3723 if (!stmmac_tso_channel_permitted(priv, chan)) 3724 continue; 3725 3726 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3727 } 3728 } 3729 3730 /* Enable Split Header */ 3731 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 3732 for (chan = 0; chan < rx_cnt; chan++) 3733 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3734 3735 3736 /* VLAN Tag Insertion */ 3737 if (priv->dma_cap.vlins) 3738 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3739 3740 /* TBS */ 3741 for (chan = 0; chan < tx_cnt; chan++) { 3742 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3743 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3744 3745 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3746 } 3747 3748 /* Configure real RX and TX queues */ 3749 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3750 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3751 3752 /* Start the ball rolling... */ 3753 stmmac_start_all_dma(priv); 3754 3755 phylink_rx_clk_stop_block(priv->phylink); 3756 stmmac_set_hw_vlan_mode(priv, priv->hw); 3757 phylink_rx_clk_stop_unblock(priv->phylink); 3758 3759 return 0; 3760 } 3761 3762 static void stmmac_free_irq(struct net_device *dev, 3763 enum request_irq_err irq_err, int irq_idx) 3764 { 3765 struct stmmac_priv *priv = netdev_priv(dev); 3766 struct stmmac_msi *msi = priv->msi; 3767 int j; 3768 3769 switch (irq_err) { 3770 case REQ_IRQ_ERR_ALL: 3771 irq_idx = priv->plat->tx_queues_to_use; 3772 fallthrough; 3773 case REQ_IRQ_ERR_TX: 3774 for (j = irq_idx - 1; msi && j >= 0; j--) { 3775 if (msi->tx_irq[j] > 0) { 3776 irq_set_affinity_hint(msi->tx_irq[j], NULL); 3777 free_irq(msi->tx_irq[j], 3778 &priv->dma_conf.tx_queue[j]); 3779 } 3780 } 3781 irq_idx = priv->plat->rx_queues_to_use; 3782 fallthrough; 3783 case REQ_IRQ_ERR_RX: 3784 for (j = irq_idx - 1; msi && j >= 0; j--) { 3785 if (msi->rx_irq[j] > 0) { 3786 irq_set_affinity_hint(msi->rx_irq[j], NULL); 3787 free_irq(msi->rx_irq[j], 3788 &priv->dma_conf.rx_queue[j]); 3789 } 3790 } 3791 3792 if (msi && msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) 3793 free_irq(msi->sfty_ue_irq, dev); 3794 fallthrough; 3795 case REQ_IRQ_ERR_SFTY_UE: 3796 if (msi && msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) 3797 free_irq(msi->sfty_ce_irq, dev); 3798 fallthrough; 3799 case REQ_IRQ_ERR_SFTY_CE: 3800 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3801 free_irq(priv->wol_irq, dev); 3802 fallthrough; 3803 case REQ_IRQ_ERR_SFTY: 3804 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) 3805 free_irq(priv->sfty_irq, dev); 3806 fallthrough; 3807 case REQ_IRQ_ERR_WOL: 3808 free_irq(dev->irq, dev); 3809 fallthrough; 3810 case REQ_IRQ_ERR_MAC: 3811 case REQ_IRQ_ERR_NO: 3812 /* If MAC IRQ request error, no more IRQ to free */ 3813 break; 3814 } 3815 } 3816 3817 static int stmmac_msi_init(struct stmmac_priv *priv, 3818 struct stmmac_resources *res) 3819 { 3820 int i; 3821 3822 priv->msi = devm_kmalloc(priv->device, sizeof(*priv->msi), GFP_KERNEL); 3823 if (!priv->msi) 3824 return -ENOMEM; 3825 3826 priv->msi->sfty_ce_irq = res->sfty_ce_irq; 3827 priv->msi->sfty_ue_irq = res->sfty_ue_irq; 3828 3829 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 3830 priv->msi->rx_irq[i] = res->rx_irq[i]; 3831 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 3832 priv->msi->tx_irq[i] = res->tx_irq[i]; 3833 3834 return 0; 3835 } 3836 3837 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3838 { 3839 struct stmmac_priv *priv = netdev_priv(dev); 3840 struct stmmac_msi *msi = priv->msi; 3841 enum request_irq_err irq_err; 3842 int irq_idx = 0; 3843 char *int_name; 3844 int ret; 3845 int i; 3846 3847 /* For common interrupt */ 3848 int_name = msi->int_name_mac; 3849 sprintf(int_name, "%s:%s", dev->name, "mac"); 3850 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3851 0, int_name, dev); 3852 if (unlikely(ret < 0)) { 3853 netdev_err(priv->dev, 3854 "%s: alloc mac MSI %d (error: %d)\n", 3855 __func__, dev->irq, ret); 3856 irq_err = REQ_IRQ_ERR_MAC; 3857 goto irq_error; 3858 } 3859 3860 /* Request the Wake IRQ in case of another line 3861 * is used for WoL 3862 */ 3863 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3864 int_name = msi->int_name_wol; 3865 sprintf(int_name, "%s:%s", dev->name, "wol"); 3866 ret = request_irq(priv->wol_irq, 3867 stmmac_mac_interrupt, 3868 0, int_name, dev); 3869 if (unlikely(ret < 0)) { 3870 netdev_err(priv->dev, 3871 "%s: alloc wol MSI %d (error: %d)\n", 3872 __func__, priv->wol_irq, ret); 3873 irq_err = REQ_IRQ_ERR_WOL; 3874 goto irq_error; 3875 } 3876 } 3877 3878 /* Request the common Safety Feature Correctible/Uncorrectible 3879 * Error line in case of another line is used 3880 */ 3881 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3882 int_name = msi->int_name_sfty; 3883 sprintf(int_name, "%s:%s", dev->name, "safety"); 3884 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3885 0, int_name, dev); 3886 if (unlikely(ret < 0)) { 3887 netdev_err(priv->dev, 3888 "%s: alloc sfty MSI %d (error: %d)\n", 3889 __func__, priv->sfty_irq, ret); 3890 irq_err = REQ_IRQ_ERR_SFTY; 3891 goto irq_error; 3892 } 3893 } 3894 3895 /* Request the Safety Feature Correctible Error line in 3896 * case of another line is used 3897 */ 3898 if (msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) { 3899 int_name = msi->int_name_sfty_ce; 3900 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3901 ret = request_irq(msi->sfty_ce_irq, 3902 stmmac_safety_interrupt, 3903 0, int_name, dev); 3904 if (unlikely(ret < 0)) { 3905 netdev_err(priv->dev, 3906 "%s: alloc sfty ce MSI %d (error: %d)\n", 3907 __func__, msi->sfty_ce_irq, ret); 3908 irq_err = REQ_IRQ_ERR_SFTY_CE; 3909 goto irq_error; 3910 } 3911 } 3912 3913 /* Request the Safety Feature Uncorrectible Error line in 3914 * case of another line is used 3915 */ 3916 if (msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) { 3917 int_name = msi->int_name_sfty_ue; 3918 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3919 ret = request_irq(msi->sfty_ue_irq, 3920 stmmac_safety_interrupt, 3921 0, int_name, dev); 3922 if (unlikely(ret < 0)) { 3923 netdev_err(priv->dev, 3924 "%s: alloc sfty ue MSI %d (error: %d)\n", 3925 __func__, msi->sfty_ue_irq, ret); 3926 irq_err = REQ_IRQ_ERR_SFTY_UE; 3927 goto irq_error; 3928 } 3929 } 3930 3931 /* Request Rx MSI irq */ 3932 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3933 if (i >= MTL_MAX_RX_QUEUES) 3934 break; 3935 if (msi->rx_irq[i] == 0) 3936 continue; 3937 3938 int_name = msi->int_name_rx_irq[i]; 3939 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3940 ret = request_irq(msi->rx_irq[i], 3941 stmmac_msi_intr_rx, 3942 0, int_name, &priv->dma_conf.rx_queue[i]); 3943 if (unlikely(ret < 0)) { 3944 netdev_err(priv->dev, 3945 "%s: alloc rx-%d MSI %d (error: %d)\n", 3946 __func__, i, msi->rx_irq[i], ret); 3947 irq_err = REQ_IRQ_ERR_RX; 3948 irq_idx = i; 3949 goto irq_error; 3950 } 3951 irq_set_affinity_hint(msi->rx_irq[i], 3952 cpumask_of(i % num_online_cpus())); 3953 } 3954 3955 /* Request Tx MSI irq */ 3956 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3957 if (i >= MTL_MAX_TX_QUEUES) 3958 break; 3959 if (msi->tx_irq[i] == 0) 3960 continue; 3961 3962 int_name = msi->int_name_tx_irq[i]; 3963 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3964 ret = request_irq(msi->tx_irq[i], 3965 stmmac_msi_intr_tx, 3966 0, int_name, &priv->dma_conf.tx_queue[i]); 3967 if (unlikely(ret < 0)) { 3968 netdev_err(priv->dev, 3969 "%s: alloc tx-%d MSI %d (error: %d)\n", 3970 __func__, i, msi->tx_irq[i], ret); 3971 irq_err = REQ_IRQ_ERR_TX; 3972 irq_idx = i; 3973 goto irq_error; 3974 } 3975 irq_set_affinity_hint(msi->tx_irq[i], 3976 cpumask_of(i % num_online_cpus())); 3977 } 3978 3979 return 0; 3980 3981 irq_error: 3982 stmmac_free_irq(dev, irq_err, irq_idx); 3983 return ret; 3984 } 3985 3986 static int stmmac_request_irq_single(struct net_device *dev) 3987 { 3988 struct stmmac_priv *priv = netdev_priv(dev); 3989 enum request_irq_err irq_err; 3990 int ret; 3991 3992 ret = request_irq(dev->irq, stmmac_interrupt, 3993 IRQF_SHARED, dev->name, dev); 3994 if (unlikely(ret < 0)) { 3995 netdev_err(priv->dev, 3996 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3997 __func__, dev->irq, ret); 3998 irq_err = REQ_IRQ_ERR_MAC; 3999 goto irq_error; 4000 } 4001 4002 /* Request the Wake IRQ in case of another line 4003 * is used for WoL 4004 */ 4005 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 4006 ret = request_irq(priv->wol_irq, stmmac_interrupt, 4007 IRQF_SHARED, dev->name, dev); 4008 if (unlikely(ret < 0)) { 4009 netdev_err(priv->dev, 4010 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 4011 __func__, priv->wol_irq, ret); 4012 irq_err = REQ_IRQ_ERR_WOL; 4013 goto irq_error; 4014 } 4015 } 4016 4017 /* Request the common Safety Feature Correctible/Uncorrectible 4018 * Error line in case of another line is used 4019 */ 4020 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 4021 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 4022 IRQF_SHARED, dev->name, dev); 4023 if (unlikely(ret < 0)) { 4024 netdev_err(priv->dev, 4025 "%s: ERROR: allocating the sfty IRQ %d (%d)\n", 4026 __func__, priv->sfty_irq, ret); 4027 irq_err = REQ_IRQ_ERR_SFTY; 4028 goto irq_error; 4029 } 4030 } 4031 4032 return 0; 4033 4034 irq_error: 4035 stmmac_free_irq(dev, irq_err, 0); 4036 return ret; 4037 } 4038 4039 static int stmmac_request_irq(struct net_device *dev) 4040 { 4041 struct stmmac_priv *priv = netdev_priv(dev); 4042 int ret; 4043 4044 /* Request the IRQ lines */ 4045 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 4046 ret = stmmac_request_irq_multi_msi(dev); 4047 else 4048 ret = stmmac_request_irq_single(dev); 4049 4050 return ret; 4051 } 4052 4053 /** 4054 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 4055 * @priv: driver private structure 4056 * @mtu: MTU to setup the dma queue and buf with 4057 * Description: Allocate and generate a dma_conf based on the provided MTU. 4058 * Allocate the Tx/Rx DMA queue and init them. 4059 * Return value: 4060 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 4061 */ 4062 static struct stmmac_dma_conf * 4063 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 4064 { 4065 struct stmmac_dma_conf *dma_conf; 4066 int bfsize, len, ret; 4067 u8 chan; 4068 4069 dma_conf = kzalloc_obj(*dma_conf); 4070 if (!dma_conf) { 4071 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 4072 __func__); 4073 return ERR_PTR(-ENOMEM); 4074 } 4075 4076 len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; 4077 4078 /* Returns 0 or BUF_SIZE_16KiB if len > 8KiB and dwmac4 or ring mode */ 4079 bfsize = stmmac_set_16kib_bfsize(priv, len); 4080 if (bfsize < 0) 4081 bfsize = 0; 4082 4083 if (bfsize < BUF_SIZE_16KiB) 4084 bfsize = stmmac_set_bfsize(len); 4085 4086 dma_conf->dma_buf_sz = bfsize; 4087 /* Chose the tx/rx size from the already defined one in the 4088 * priv struct. (if defined) 4089 */ 4090 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 4091 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 4092 4093 if (!dma_conf->dma_tx_size) 4094 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 4095 if (!dma_conf->dma_rx_size) 4096 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 4097 4098 /* Earlier check for TBS */ 4099 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 4100 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 4101 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 4102 4103 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 4104 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 4105 } 4106 4107 ret = alloc_dma_desc_resources(priv, dma_conf); 4108 if (ret < 0) { 4109 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 4110 __func__); 4111 goto alloc_error; 4112 } 4113 4114 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 4115 if (ret < 0) { 4116 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 4117 __func__); 4118 goto init_error; 4119 } 4120 4121 return dma_conf; 4122 4123 init_error: 4124 free_dma_desc_resources(priv, dma_conf); 4125 alloc_error: 4126 kfree(dma_conf); 4127 return ERR_PTR(ret); 4128 } 4129 4130 /** 4131 * __stmmac_open - open entry point of the driver 4132 * @dev : pointer to the device structure. 4133 * @dma_conf : structure to take the dma data 4134 * Description: 4135 * This function is the open entry point of the driver. 4136 * Return value: 4137 * 0 on success and an appropriate (-)ve integer as defined in errno.h 4138 * file on failure. 4139 */ 4140 static int __stmmac_open(struct net_device *dev, 4141 struct stmmac_dma_conf *dma_conf) 4142 { 4143 struct stmmac_priv *priv = netdev_priv(dev); 4144 u8 chan; 4145 int ret; 4146 4147 for (int i = 0; i < priv->plat->tx_queues_to_use; i++) 4148 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 4149 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 4150 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 4151 4152 /* The PHY is suspended when the interface is reopened without 4153 * disconnecting the PHY, e.g. on MTU change. IEEE 802.3 allows PHYs 4154 * to stop their receive clock while powered down, but the DMA 4155 * software reset in stmmac_hw_setup() requires a running receive 4156 * clock, and phylink_start() below resumes the PHY only after the 4157 * hardware setup. Resume a suspended PHY here first. 4158 */ 4159 phylink_prepare_resume(priv->phylink); 4160 4161 stmmac_reset_queues_param(priv); 4162 4163 ret = stmmac_hw_setup(dev); 4164 if (ret < 0) { 4165 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 4166 goto init_error; 4167 } 4168 4169 stmmac_setup_ptp(priv); 4170 4171 stmmac_init_coalesce(priv); 4172 4173 phylink_start(priv->phylink); 4174 4175 stmmac_vlan_restore(priv); 4176 4177 ret = stmmac_request_irq(dev); 4178 if (ret) 4179 goto irq_error; 4180 4181 stmmac_enable_all_queues(priv); 4182 netif_tx_start_all_queues(priv->dev); 4183 stmmac_enable_all_dma_irq(priv); 4184 4185 return 0; 4186 4187 irq_error: 4188 phylink_stop(priv->phylink); 4189 4190 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4191 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4192 4193 stmmac_release_ptp(priv); 4194 init_error: 4195 return ret; 4196 } 4197 4198 static int stmmac_open(struct net_device *dev) 4199 { 4200 struct stmmac_priv *priv = netdev_priv(dev); 4201 struct stmmac_dma_conf *dma_conf; 4202 int ret; 4203 4204 /* Initialise the tx lpi timer, converting from msec to usec */ 4205 if (!priv->tx_lpi_timer) 4206 priv->tx_lpi_timer = eee_timer * 1000; 4207 4208 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 4209 if (IS_ERR(dma_conf)) 4210 return PTR_ERR(dma_conf); 4211 4212 ret = pm_runtime_resume_and_get(priv->device); 4213 if (ret < 0) 4214 goto err_dma_resources; 4215 4216 ret = stmmac_init_phy(dev); 4217 if (ret) 4218 goto err_runtime_pm; 4219 4220 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 4221 ret = stmmac_legacy_serdes_power_up(priv); 4222 if (ret < 0) 4223 goto err_disconnect_phy; 4224 } 4225 4226 ret = __stmmac_open(dev, dma_conf); 4227 if (ret) 4228 goto err_serdes; 4229 4230 kfree(dma_conf); 4231 4232 /* We may have called phylink_speed_down before */ 4233 phylink_speed_up(priv->phylink); 4234 4235 return ret; 4236 4237 err_serdes: 4238 stmmac_legacy_serdes_power_down(priv); 4239 err_disconnect_phy: 4240 phylink_disconnect_phy(priv->phylink); 4241 err_runtime_pm: 4242 pm_runtime_put(priv->device); 4243 err_dma_resources: 4244 free_dma_desc_resources(priv, dma_conf); 4245 kfree(dma_conf); 4246 return ret; 4247 } 4248 4249 static void __stmmac_release(struct net_device *dev) 4250 { 4251 struct stmmac_priv *priv = netdev_priv(dev); 4252 u8 chan; 4253 4254 /* Stop and disconnect the PHY */ 4255 phylink_stop(priv->phylink); 4256 4257 stmmac_disable_all_queues(priv); 4258 4259 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4260 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4261 4262 netif_tx_disable(dev); 4263 4264 /* Free the IRQ lines */ 4265 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4266 4267 /* Stop TX/RX DMA and clear the descriptors */ 4268 stmmac_stop_all_dma(priv); 4269 4270 /* Release and free the Rx/Tx resources */ 4271 free_dma_desc_resources(priv, &priv->dma_conf); 4272 4273 stmmac_release_ptp(priv); 4274 4275 if (stmmac_fpe_supported(priv)) 4276 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 4277 } 4278 4279 /** 4280 * stmmac_release - close entry point of the driver 4281 * @dev : device pointer. 4282 * Description: 4283 * This is the stop entry point of the driver. 4284 */ 4285 static int stmmac_release(struct net_device *dev) 4286 { 4287 struct stmmac_priv *priv = netdev_priv(dev); 4288 4289 /* If the PHY or MAC has WoL enabled, then the PHY will not be 4290 * suspended when phylink_stop() is called below. Set the PHY 4291 * to its slowest speed to save power. 4292 */ 4293 if (device_may_wakeup(priv->device)) 4294 phylink_speed_down(priv->phylink, false); 4295 4296 __stmmac_release(dev); 4297 4298 stmmac_legacy_serdes_power_down(priv); 4299 phylink_disconnect_phy(priv->phylink); 4300 pm_runtime_put(priv->device); 4301 4302 return 0; 4303 } 4304 4305 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4306 struct stmmac_tx_queue *tx_q) 4307 { 4308 struct dma_desc *p; 4309 u16 tag = 0x0; 4310 4311 if (!priv->dma_cap.vlins || !skb_vlan_tag_present(skb)) 4312 return false; 4313 4314 tag = skb_vlan_tag_get(skb); 4315 4316 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4317 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4318 else 4319 p = &tx_q->dma_tx[tx_q->cur_tx]; 4320 4321 if (stmmac_set_desc_vlan_tag(priv, p, tag, 0x0, 0x0)) 4322 return false; 4323 4324 stmmac_set_tx_owner(priv, p); 4325 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4326 return true; 4327 } 4328 4329 /** 4330 * stmmac_tso_allocator - close entry point of the driver 4331 * @priv: driver private structure 4332 * @entry: TX queue buffer index 4333 * @des: buffer start address 4334 * @total_len: total length to fill in descriptors 4335 * @last_segment: condition for the last descriptor 4336 * @queue: TX queue index 4337 * Description: 4338 * This function fills descriptor and request new descriptors according to 4339 * buffer length to fill 4340 */ 4341 static void stmmac_tso_allocator(struct stmmac_priv *priv, u32 *entry, 4342 dma_addr_t des, int total_len, 4343 bool last_segment, u32 queue) 4344 { 4345 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4346 struct dma_desc *desc; 4347 u32 buff_size; 4348 int tmp_len; 4349 4350 tmp_len = total_len; 4351 4352 while (tmp_len > 0) { 4353 dma_addr_t curr_addr; 4354 4355 *entry = STMMAC_NEXT_ENTRY(*entry, priv->dma_conf.dma_tx_size); 4356 WARN_ON(tx_q->tx_skbuff[*entry]); 4357 4358 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4359 desc = &tx_q->dma_entx[*entry].basic; 4360 else 4361 desc = &tx_q->dma_tx[*entry]; 4362 4363 curr_addr = des + (total_len - tmp_len); 4364 stmmac_set_desc_addr(priv, desc, curr_addr); 4365 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4366 TSO_MAX_BUFF_SIZE : tmp_len; 4367 4368 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4369 0, 1, 4370 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4371 0, 0); 4372 4373 tmp_len -= TSO_MAX_BUFF_SIZE; 4374 } 4375 } 4376 4377 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4378 { 4379 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4380 4381 /* The own bit must be the latest setting done when prepare the 4382 * descriptor and then barrier is needed to make sure that 4383 * all is coherent before granting the DMA engine. 4384 */ 4385 wmb(); 4386 4387 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx); 4388 } 4389 4390 static void stmmac_set_gso_features(struct net_device *ndev) 4391 { 4392 struct stmmac_priv *priv = netdev_priv(ndev); 4393 const struct stmmac_dma_cfg *dma_cfg; 4394 int txpbl; 4395 4396 if (priv->dma_cap.tsoen) 4397 dev_info(priv->device, "TSO supported\n"); 4398 4399 if (!(priv->plat->flags & STMMAC_FLAG_TSO_EN)) 4400 return; 4401 4402 if (!priv->dma_cap.tsoen) { 4403 dev_warn(priv->device, "platform requests unsupported TSO\n"); 4404 return; 4405 } 4406 4407 /* FIXME: 4408 * STM32MP151 (v4.2 userver v4.0) states that TxPBL must be >= 4. It 4409 * is not clear whether PBLx8 (which multiplies the PBL value by 8) 4410 * influences this. 4411 */ 4412 dma_cfg = priv->plat->dma_cfg; 4413 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl; 4414 if (txpbl < 4) { 4415 dev_warn(priv->device, "txpbl(%d) is too low for TSO\n", txpbl); 4416 return; 4417 } 4418 4419 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4420 if (priv->plat->core_type == DWMAC_CORE_GMAC4) 4421 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 4422 4423 dev_info(priv->device, "TSO feature enabled\n"); 4424 } 4425 4426 static size_t stmmac_tso_header_size(struct sk_buff *skb) 4427 { 4428 size_t size; 4429 4430 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4431 size = skb_transport_offset(skb) + sizeof(struct udphdr); 4432 else 4433 size = skb_tcp_all_headers(skb); 4434 4435 return size; 4436 } 4437 4438 /* STM32MP151 (dwmac v4.2) and STM32MP25xx (dwmac v5.3) states for TDES2 normal 4439 * (read format) descriptor that the maximum header length supported for the 4440 * TSO feature is 1023 bytes. 4441 * 4442 * While IPv4 is limited to MAC+VLAN+IPv4+ext+TCP+ext = 138 bytes, the IPv6 4443 * extension headers aren't similarly limited. 4444 * 4445 * Fall back to software GSO for these skbs. Also check that the MSS is >= 4446 * the recommended 64 bytes (documented in ETH_DMACxCR register description), 4447 * and that a the header plus MSS is not larger than 16383 (documented in 4448 * "Building the Descriptor and the packet for the TSO feature"). 4449 */ 4450 static bool stmmac_tso_valid_packet(struct sk_buff *skb) 4451 { 4452 size_t header_len = stmmac_tso_header_size(skb); 4453 unsigned int gso_size = skb_shinfo(skb)->gso_size; 4454 4455 return header_len <= 1023 && gso_size >= 64 && 4456 header_len + gso_size < 16383; 4457 } 4458 4459 static int stmmac_tso_get_num_desc(struct stmmac_tx_queue *tx_q, 4460 struct sk_buff *skb, u32 pay_len) 4461 { 4462 int i, ndesc = 1; 4463 4464 /* head payload */ 4465 ndesc += DIV_ROUND_UP(pay_len, TSO_MAX_BUFF_SIZE); 4466 /* frag payload */ 4467 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4468 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4469 4470 ndesc += DIV_ROUND_UP(skb_frag_size(frag), 4471 TSO_MAX_BUFF_SIZE); 4472 } 4473 /* MSS update requires a new descriptor */ 4474 ndesc += !!(skb_shinfo(skb)->gso_size != tx_q->mss); 4475 4476 return ndesc; 4477 } 4478 4479 /** 4480 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4481 * @skb : the socket buffer 4482 * @dev : device pointer 4483 * Description: this is the transmit function that is called on TSO frames 4484 * (support available on GMAC4 and newer chips). 4485 * Diagram below show the ring programming in case of TSO frames: 4486 * 4487 * First Descriptor 4488 * -------- 4489 * | DES0 |---> buffer1 = L2/L3/L4 header 4490 * | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address 4491 * | | width is 32-bit, but we never use it. 4492 * | | Also can be used as the most-significant 8-bits or 16-bits of 4493 * | | buffer1 address pointer if the DMA AXI address width is 40-bit 4494 * | | or 48-bit, and we always use it. 4495 * | DES2 |---> buffer1 len 4496 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4497 * -------- 4498 * -------- 4499 * | DES0 |---> buffer1 = TCP Payload (can continue on next descr...) 4500 * | DES1 |---> same as the First Descriptor 4501 * | DES2 |---> buffer1 len 4502 * | DES3 | 4503 * -------- 4504 * | 4505 * ... 4506 * | 4507 * -------- 4508 * | DES0 |---> buffer1 = Split TCP Payload 4509 * | DES1 |---> same as the First Descriptor 4510 * | DES2 |---> buffer1 len 4511 * | DES3 | 4512 * -------- 4513 * 4514 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4515 */ 4516 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4517 { 4518 unsigned int first_entry, entry, tx_packets, proto_hdr_len; 4519 struct dma_desc *desc, *first, *mss_desc = NULL; 4520 struct stmmac_priv *priv = netdev_priv(dev); 4521 struct stmmac_txq_stats *txq_stats; 4522 int i, first_tx, nfrags, ndesc; 4523 struct stmmac_tx_queue *tx_q; 4524 bool set_ic, is_last_segment; 4525 u32 pay_len, mss, queue; 4526 dma_addr_t des; 4527 u8 hdr; 4528 4529 nfrags = skb_shinfo(skb)->nr_frags; 4530 queue = skb_get_queue_mapping(skb); 4531 4532 tx_q = &priv->dma_conf.tx_queue[queue]; 4533 txq_stats = &priv->xstats.txq_stats[queue]; 4534 first_tx = tx_q->cur_tx; 4535 4536 /* Compute header lengths */ 4537 proto_hdr_len = stmmac_tso_header_size(skb); 4538 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4539 4540 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) 4541 hdr = sizeof(struct udphdr); 4542 else 4543 hdr = tcp_hdrlen(skb); 4544 4545 ndesc = stmmac_tso_get_num_desc(tx_q, skb, pay_len); 4546 if (unlikely(stmmac_tx_avail(priv, queue) < ndesc)) { 4547 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4548 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4549 queue)); 4550 /* This is a hard error, log it. */ 4551 netdev_err(priv->dev, 4552 "%s: Tx Ring full when queue awake\n", 4553 __func__); 4554 } 4555 return NETDEV_TX_BUSY; 4556 } 4557 4558 mss = skb_shinfo(skb)->gso_size; 4559 4560 /* set new MSS value if needed */ 4561 if (mss != tx_q->mss) { 4562 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4563 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4564 else 4565 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4566 4567 stmmac_set_mss(priv, mss_desc, mss); 4568 tx_q->mss = mss; 4569 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, 4570 priv->dma_conf.dma_tx_size); 4571 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4572 } 4573 4574 if (netif_msg_tx_queued(priv)) { 4575 pr_info("%s: hdrlen %d, hdr_len %u, pay_len %d, mss %d\n", 4576 __func__, hdr, proto_hdr_len, pay_len, mss); 4577 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4578 skb->data_len); 4579 } 4580 4581 first_entry = tx_q->cur_tx; 4582 entry = first_entry; 4583 4584 WARN_ON(tx_q->tx_skbuff[entry]); 4585 4586 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4587 desc = &tx_q->dma_entx[entry].basic; 4588 else 4589 desc = &tx_q->dma_tx[entry]; 4590 first = desc; 4591 4592 /* first descriptor: fill Headers on Buf1 */ 4593 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4594 DMA_TO_DEVICE); 4595 if (dma_mapping_error(priv->device, des)) 4596 goto error; 4597 4598 stmmac_set_desc_addr(priv, first, des); 4599 stmmac_tso_allocator(priv, &entry, des + proto_hdr_len, pay_len, 4600 (nfrags == 0), queue); 4601 4602 /* In case two or more DMA transmit descriptors are allocated for this 4603 * non-paged SKB data, the DMA buffer address should be saved to 4604 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4605 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4606 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4607 * since the tail areas of the DMA buffer can be accessed by DMA engine 4608 * sooner or later. 4609 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4610 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4611 * this DMA buffer right after the DMA engine completely finishes the 4612 * full buffer transmission. 4613 */ 4614 stmmac_set_tx_skb_dma_entry(tx_q, entry, des, skb_headlen(skb), false); 4615 4616 /* Prepare fragments */ 4617 for (i = 0; i < nfrags; i++) { 4618 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4619 4620 des = skb_frag_dma_map(priv->device, frag, 0, 4621 skb_frag_size(frag), 4622 DMA_TO_DEVICE); 4623 if (dma_mapping_error(priv->device, des)) 4624 goto error_dma_unmap; 4625 4626 stmmac_tso_allocator(priv, &entry, des, skb_frag_size(frag), 4627 (i == nfrags - 1), queue); 4628 4629 stmmac_set_tx_skb_dma_entry(tx_q, entry, des, 4630 skb_frag_size(frag), true); 4631 } 4632 tx_q->cur_tx = entry; 4633 4634 stmmac_set_tx_dma_last_segment(tx_q, tx_q->cur_tx); 4635 4636 /* Only the last descriptor gets to point to the skb. */ 4637 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4638 4639 /* Manage tx mitigation */ 4640 tx_packets = CIRC_CNT(tx_q->cur_tx + 1, first_tx, 4641 priv->dma_conf.dma_tx_size); 4642 tx_q->tx_count_frames += tx_packets; 4643 4644 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4645 set_ic = true; 4646 else if (!priv->tx_coal_frames[queue]) 4647 set_ic = false; 4648 else if (!netdev_xmit_more()) 4649 set_ic = true; 4650 else if (tx_packets > priv->tx_coal_frames[queue]) 4651 set_ic = true; 4652 else if ((tx_q->tx_count_frames % 4653 priv->tx_coal_frames[queue]) < tx_packets) 4654 set_ic = true; 4655 else 4656 set_ic = false; 4657 4658 if (set_ic) { 4659 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4660 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4661 else 4662 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4663 4664 tx_q->tx_count_frames = 0; 4665 stmmac_set_tx_ic(priv, desc); 4666 } 4667 4668 /* We've used all descriptors we need for this skb, however, 4669 * advance cur_tx so that it references a fresh descriptor. 4670 * ndo_start_xmit will fill this descriptor the next time it's 4671 * called and stmmac_tx_clean may clean up to this descriptor. 4672 */ 4673 tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4674 4675 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4676 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4677 __func__); 4678 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4679 } 4680 4681 u64_stats_update_begin(&txq_stats->q_syncp); 4682 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4683 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4684 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4685 if (set_ic) 4686 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4687 u64_stats_update_end(&txq_stats->q_syncp); 4688 4689 if (priv->sarc_type) 4690 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4691 4692 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4693 priv->hwts_tx_en)) { 4694 /* declare that device is doing timestamping */ 4695 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4696 stmmac_enable_tx_timestamp(priv, first); 4697 } 4698 4699 /* If we only have one entry used, then the first entry is the last 4700 * segment. 4701 */ 4702 is_last_segment = CIRC_CNT(tx_q->cur_tx, first_entry, 4703 priv->dma_conf.dma_tx_size) == 1; 4704 4705 /* Complete the first descriptor before granting the DMA */ 4706 stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, 4707 is_last_segment, hdr / 4, 4708 skb->len - proto_hdr_len); 4709 4710 /* If context desc is used to change MSS */ 4711 if (mss_desc) { 4712 /* Make sure that first descriptor has been completely 4713 * written, including its own bit. This is because MSS is 4714 * actually before first descriptor, so we need to make 4715 * sure that MSS's own bit is the last thing written. 4716 */ 4717 dma_wmb(); 4718 stmmac_set_tx_owner(priv, mss_desc); 4719 } 4720 4721 if (netif_msg_pktdata(priv)) { 4722 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4723 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4724 tx_q->cur_tx, first, nfrags); 4725 pr_info(">>> frame to be transmitted: "); 4726 print_pkt(skb->data, skb_headlen(skb)); 4727 } 4728 4729 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4730 skb_tx_timestamp(skb); 4731 4732 stmmac_flush_tx_descriptors(priv, queue); 4733 stmmac_tx_timer_arm(priv, queue); 4734 4735 return NETDEV_TX_OK; 4736 4737 error_dma_unmap: 4738 for (;;) { 4739 desc = stmmac_get_tx_desc(priv, tx_q, first_entry); 4740 stmmac_release_tx_desc(priv, desc, priv->descriptor_mode); 4741 stmmac_free_tx_buffer(priv, &priv->dma_conf, queue, 4742 first_entry); 4743 if (first_entry == entry) 4744 break; 4745 4746 first_entry = STMMAC_NEXT_ENTRY(first_entry, 4747 priv->dma_conf.dma_tx_size); 4748 } 4749 error: 4750 dev_err(priv->device, "Tx dma map failed\n"); 4751 dev_kfree_skb(skb); 4752 priv->xstats.tx_dropped++; 4753 return NETDEV_TX_OK; 4754 } 4755 4756 /** 4757 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4758 * @skb: socket buffer to check 4759 * 4760 * Check if a packet has an ethertype that will trigger the IP header checks 4761 * and IP/TCP checksum engine of the stmmac core. 4762 * 4763 * Return: true if the ethertype can trigger the checksum engine, false 4764 * otherwise 4765 */ 4766 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4767 { 4768 int depth = 0; 4769 __be16 proto; 4770 4771 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4772 &depth); 4773 4774 return (depth <= ETH_HLEN) && 4775 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4776 } 4777 4778 /** 4779 * stmmac_xmit - Tx entry point of the driver 4780 * @skb : the socket buffer 4781 * @dev : device pointer 4782 * Description : this is the tx entry point of the driver. 4783 * It programs the chain or the ring and supports oversized frames 4784 * and SG feature. 4785 */ 4786 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4787 { 4788 bool enh_desc, has_vlan, set_ic, is_jumbo = false; 4789 struct stmmac_priv *priv = netdev_priv(dev); 4790 unsigned int nopaged_len = skb_headlen(skb); 4791 u32 queue = skb_get_queue_mapping(skb); 4792 int nfrags = skb_shinfo(skb)->nr_frags; 4793 unsigned int first_entry, tx_packets; 4794 struct stmmac_txq_stats *txq_stats; 4795 struct dma_desc *desc, *first_desc; 4796 struct stmmac_tx_queue *tx_q; 4797 int i, csum_insertion = 0; 4798 int entry, first_tx; 4799 dma_addr_t dma_addr; 4800 u32 sdu_len; 4801 4802 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4803 stmmac_stop_sw_lpi(priv); 4804 4805 if (skb_is_gso(skb)) 4806 return stmmac_tso_xmit(skb, dev); 4807 4808 if (priv->est && priv->est->enable && 4809 priv->est->max_sdu[queue]) { 4810 sdu_len = skb->len; 4811 /* Add VLAN tag length if VLAN tag insertion offload is requested */ 4812 if (priv->dma_cap.vlins && skb_vlan_tag_present(skb)) 4813 sdu_len += VLAN_HLEN; 4814 if (sdu_len > priv->est->max_sdu[queue]) { 4815 priv->xstats.max_sdu_txq_drop[queue]++; 4816 goto max_sdu_err; 4817 } 4818 } 4819 4820 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4821 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4822 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4823 queue)); 4824 /* This is a hard error, log it. */ 4825 netdev_err(priv->dev, 4826 "%s: Tx Ring full when queue awake\n", 4827 __func__); 4828 } 4829 return NETDEV_TX_BUSY; 4830 } 4831 4832 tx_q = &priv->dma_conf.tx_queue[queue]; 4833 first_tx = tx_q->cur_tx; 4834 4835 /* Check if VLAN can be inserted by HW */ 4836 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4837 4838 entry = tx_q->cur_tx; 4839 first_entry = entry; 4840 WARN_ON(tx_q->tx_skbuff[first_entry]); 4841 4842 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4843 first_desc = desc; 4844 4845 if (has_vlan) 4846 stmmac_set_desc_vlan(priv, first_desc, STMMAC_VLAN_INSERT); 4847 4848 enh_desc = priv->plat->enh_desc; 4849 /* To program the descriptors according to the size of the frame */ 4850 if (enh_desc) 4851 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4852 4853 csum_insertion = skb->ip_summed == CHECKSUM_PARTIAL; 4854 4855 if (unlikely(is_jumbo)) { 4856 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4857 if (unlikely(entry < 0) && (entry != -EINVAL)) 4858 goto dma_map_err; 4859 } else { 4860 bool last_segment = (nfrags == 0); 4861 4862 dma_addr = dma_map_single(priv->device, skb->data, 4863 nopaged_len, DMA_TO_DEVICE); 4864 if (dma_mapping_error(priv->device, dma_addr)) 4865 goto dma_map_err; 4866 4867 stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr, 4868 nopaged_len, false); 4869 4870 stmmac_set_desc_addr(priv, first_desc, dma_addr); 4871 4872 if (last_segment) 4873 stmmac_set_tx_dma_last_segment(tx_q, first_entry); 4874 4875 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4876 priv->hwts_tx_en)) { 4877 /* declare that device is doing timestamping */ 4878 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4879 stmmac_enable_tx_timestamp(priv, first_desc); 4880 } 4881 4882 /* Prepare the first descriptor without setting the OWN bit */ 4883 stmmac_prepare_tx_desc(priv, first_desc, 1, nopaged_len, 4884 csum_insertion, priv->descriptor_mode, 4885 0, last_segment, skb->len); 4886 } 4887 4888 if (priv->sarc_type) 4889 stmmac_set_desc_sarc(priv, first_desc, priv->sarc_type); 4890 4891 /* STMMAC_TBS_EN can only be set if STMMAC_TBS_AVAIL has already 4892 * been set, which means the underlying type of the descriptors 4893 * will be struct stmmac_edesc. Therefore, it is safe to convert 4894 * the basic descriptor to the enhanced descriptor here. 4895 */ 4896 if (tx_q->tbs & STMMAC_TBS_EN) { 4897 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4898 4899 stmmac_set_desc_tbs(priv, dma_desc_to_edesc(first_desc), 4900 ts.tv_sec, ts.tv_nsec); 4901 } 4902 4903 for (i = 0; i < nfrags; i++) { 4904 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4905 unsigned int frag_size = skb_frag_size(frag); 4906 bool last_segment = (i == (nfrags - 1)); 4907 4908 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4909 WARN_ON(tx_q->tx_skbuff[entry]); 4910 4911 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4912 4913 dma_addr = skb_frag_dma_map(priv->device, frag, 0, frag_size, 4914 DMA_TO_DEVICE); 4915 if (dma_mapping_error(priv->device, dma_addr)) 4916 goto dma_map_err; /* should reuse desc w/o issues */ 4917 4918 stmmac_set_tx_skb_dma_entry(tx_q, entry, dma_addr, frag_size, 4919 true); 4920 stmmac_set_desc_addr(priv, desc, dma_addr); 4921 4922 /* Prepare the descriptor and set the own bit too */ 4923 stmmac_prepare_tx_desc(priv, desc, 0, frag_size, csum_insertion, 4924 priv->descriptor_mode, 1, last_segment, 4925 skb->len); 4926 } 4927 4928 stmmac_set_tx_dma_last_segment(tx_q, entry); 4929 4930 /* Only the last descriptor gets to point to the skb. */ 4931 tx_q->tx_skbuff[entry] = skb; 4932 4933 /* According to the coalesce parameter the IC bit for the latest 4934 * segment is reset and the timer re-started to clean the tx status. 4935 * This approach takes care about the fragments: desc is the first 4936 * element in case of no SG. 4937 */ 4938 tx_packets = CIRC_CNT(entry + 1, first_tx, priv->dma_conf.dma_tx_size); 4939 tx_q->tx_count_frames += tx_packets; 4940 4941 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4942 set_ic = true; 4943 else if (!priv->tx_coal_frames[queue]) 4944 set_ic = false; 4945 else if (!netdev_xmit_more()) 4946 set_ic = true; 4947 else if (tx_packets > priv->tx_coal_frames[queue]) 4948 set_ic = true; 4949 else if ((tx_q->tx_count_frames % 4950 priv->tx_coal_frames[queue]) < tx_packets) 4951 set_ic = true; 4952 else 4953 set_ic = false; 4954 4955 if (set_ic) { 4956 desc = stmmac_get_tx_desc(priv, tx_q, entry); 4957 tx_q->tx_count_frames = 0; 4958 stmmac_set_tx_ic(priv, desc); 4959 } 4960 4961 /* We've used all descriptors we need for this skb, however, 4962 * advance cur_tx so that it references a fresh descriptor. 4963 * ndo_start_xmit will fill this descriptor the next time it's 4964 * called and stmmac_tx_clean may clean up to this descriptor. 4965 */ 4966 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 4967 tx_q->cur_tx = entry; 4968 4969 if (netif_msg_pktdata(priv)) { 4970 netdev_dbg(priv->dev, 4971 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4972 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4973 entry, first_desc, nfrags); 4974 4975 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4976 print_pkt(skb->data, skb->len); 4977 } 4978 4979 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4980 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4981 __func__); 4982 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4983 } 4984 4985 txq_stats = &priv->xstats.txq_stats[queue]; 4986 u64_stats_update_begin(&txq_stats->q_syncp); 4987 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4988 if (set_ic) 4989 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4990 u64_stats_update_end(&txq_stats->q_syncp); 4991 4992 /* Set the OWN bit on the first descriptor now that all descriptors 4993 * for this skb are populated. 4994 */ 4995 stmmac_set_tx_owner(priv, first_desc); 4996 4997 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4998 4999 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5000 skb_tx_timestamp(skb); 5001 stmmac_flush_tx_descriptors(priv, queue); 5002 stmmac_tx_timer_arm(priv, queue); 5003 5004 return NETDEV_TX_OK; 5005 5006 dma_map_err: 5007 netdev_err(priv->dev, "Tx DMA map failed\n"); 5008 max_sdu_err: 5009 dev_kfree_skb(skb); 5010 priv->xstats.tx_dropped++; 5011 return NETDEV_TX_OK; 5012 } 5013 5014 static netdev_features_t stmmac_features_check(struct sk_buff *skb, 5015 struct net_device *dev, 5016 netdev_features_t features) 5017 { 5018 struct stmmac_priv *priv = netdev_priv(dev); 5019 u16 queue = skb_get_queue_mapping(skb); 5020 5021 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 5022 * queues. In that case, checksum offloading for those queues that don't 5023 * support tx coe needs to fallback to software checksum calculation. 5024 * 5025 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 5026 * also have to be checksummed in software. 5027 * 5028 * Note that disabling hardware checksumming also disables TSO. See 5029 * harmonize_features() in net/core/dev.c 5030 */ 5031 if (priv->plat->tx_queues_cfg[queue].coe_unsupported || 5032 !stmmac_has_ip_ethertype(skb)) 5033 features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 5034 5035 if (skb_is_gso(skb)) { 5036 if (!stmmac_tso_channel_permitted(priv, queue) || 5037 !stmmac_tso_valid_packet(skb)) 5038 features &= ~NETIF_F_GSO_MASK; 5039 5040 /* If we are going to be using hardware TSO, always insert 5041 * VLAN tag to SKB payload for TSO frames. 5042 * 5043 * Never insert VLAN tag by HW, since segments split by 5044 * TSO engine will be un-tagged by mistake. 5045 */ 5046 if (features & NETIF_F_GSO_MASK) 5047 features &= ~(NETIF_F_HW_VLAN_STAG_TX | 5048 NETIF_F_HW_VLAN_CTAG_TX); 5049 } 5050 5051 return vlan_features_check(skb, features); 5052 } 5053 5054 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 5055 { 5056 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 5057 __be16 vlan_proto = veth->h_vlan_proto; 5058 u16 vlanid; 5059 5060 if ((vlan_proto == htons(ETH_P_8021Q) && 5061 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 5062 (vlan_proto == htons(ETH_P_8021AD) && 5063 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 5064 /* pop the vlan tag */ 5065 vlanid = ntohs(veth->h_vlan_TCI); 5066 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 5067 skb_pull(skb, VLAN_HLEN); 5068 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 5069 } 5070 } 5071 5072 /** 5073 * stmmac_rx_refill - refill used skb preallocated buffers 5074 * @priv: driver private structure 5075 * @queue: RX queue index 5076 * Description : this is to reallocate the skb for the reception process 5077 * that is based on zero-copy. 5078 */ 5079 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 5080 { 5081 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5082 int dirty = stmmac_rx_dirty(priv, queue); 5083 unsigned int entry = rx_q->dirty_rx; 5084 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 5085 5086 if (priv->dma_cap.host_dma_width <= 32) 5087 gfp |= GFP_DMA32; 5088 5089 while (dirty-- > 0) { 5090 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5091 struct dma_desc *p; 5092 bool use_rx_wd; 5093 5094 p = stmmac_get_rx_desc(priv, rx_q, entry); 5095 5096 if (!buf->page) { 5097 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5098 if (!buf->page) 5099 break; 5100 } 5101 5102 if (priv->sph_active && !buf->sec_page) { 5103 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 5104 if (!buf->sec_page) 5105 break; 5106 5107 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 5108 } 5109 5110 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 5111 5112 stmmac_set_desc_addr(priv, p, buf->addr); 5113 if (priv->sph_active) 5114 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 5115 else 5116 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 5117 stmmac_refill_desc3(priv, rx_q, p); 5118 5119 rx_q->rx_count_frames++; 5120 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5121 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5122 rx_q->rx_count_frames = 0; 5123 5124 use_rx_wd = !priv->rx_coal_frames[queue]; 5125 use_rx_wd |= rx_q->rx_count_frames > 0; 5126 if (!priv->use_riwt) 5127 use_rx_wd = false; 5128 5129 dma_wmb(); 5130 stmmac_set_rx_owner(priv, p, use_rx_wd); 5131 5132 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5133 } 5134 rx_q->dirty_rx = entry; 5135 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5136 /* Wake up Rx DMA from the suspend state if required */ 5137 stmmac_enable_dma_reception(priv, priv->ioaddr, queue); 5138 } 5139 5140 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 5141 struct dma_desc *p, 5142 int status, unsigned int len) 5143 { 5144 unsigned int plen = 0, hlen = 0; 5145 int coe = priv->hw->rx_csum; 5146 5147 /* Not first descriptor, buffer is always zero */ 5148 if (priv->sph_active && len) 5149 return 0; 5150 5151 /* First descriptor, get split header length */ 5152 stmmac_get_rx_header_len(priv, p, &hlen); 5153 if (priv->sph_active && hlen) { 5154 priv->xstats.rx_split_hdr_pkt_n++; 5155 return hlen; 5156 } 5157 5158 /* First descriptor, not last descriptor and not split header */ 5159 if (status & rx_not_ls) 5160 return priv->dma_conf.dma_buf_sz; 5161 5162 plen = stmmac_get_rx_frame_len(priv, p, coe); 5163 5164 /* First descriptor and last descriptor and not split header */ 5165 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 5166 } 5167 5168 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 5169 struct dma_desc *p, 5170 int status, unsigned int len) 5171 { 5172 int coe = priv->hw->rx_csum; 5173 unsigned int plen = 0; 5174 5175 /* Not split header, buffer is not available */ 5176 if (!priv->sph_active) 5177 return 0; 5178 5179 /* For GMAC4, when split header is enabled, in some rare cases, the 5180 * hardware does not fill buf2 of the first descriptor with payload. 5181 * Thus we cannot assume buf2 is always fully filled if it is not 5182 * the last descriptor. Otherwise, the length of buf2 of the second 5183 * descriptor will be calculated wrong and cause an oops. 5184 * 5185 * If this is the last descriptor, 'plen' is the length of the 5186 * received packet that was transferred to system memory. 5187 * Otherwise, it is the accumulated number of bytes that have been 5188 * transferred for the current packet. 5189 * 5190 * Thus 'plen - len' always gives the correct length of buf2. 5191 */ 5192 5193 /* Not GMAC4 and not last descriptor */ 5194 if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) 5195 return priv->dma_conf.dma_buf_sz; 5196 5197 /* GMAC4 or last descriptor */ 5198 plen = stmmac_get_rx_frame_len(priv, p, coe); 5199 5200 return plen - len; 5201 } 5202 5203 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 5204 struct xdp_frame *xdpf, bool dma_map) 5205 { 5206 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 5207 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 5208 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 5209 unsigned int entry = tx_q->cur_tx; 5210 enum stmmac_txbuf_type buf_type; 5211 struct dma_desc *tx_desc; 5212 dma_addr_t dma_addr; 5213 bool set_ic; 5214 5215 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 5216 return STMMAC_XDP_CONSUMED; 5217 5218 if (priv->est && priv->est->enable && 5219 priv->est->max_sdu[queue] && 5220 xdpf->len > priv->est->max_sdu[queue]) { 5221 priv->xstats.max_sdu_txq_drop[queue]++; 5222 return STMMAC_XDP_CONSUMED; 5223 } 5224 5225 tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); 5226 if (dma_map) { 5227 dma_addr = dma_map_single(priv->device, xdpf->data, 5228 xdpf->len, DMA_TO_DEVICE); 5229 if (dma_mapping_error(priv->device, dma_addr)) 5230 return STMMAC_XDP_CONSUMED; 5231 5232 buf_type = STMMAC_TXBUF_T_XDP_NDO; 5233 } else { 5234 struct page *page = virt_to_page(xdpf->data); 5235 5236 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 5237 xdpf->headroom; 5238 dma_sync_single_for_device(priv->device, dma_addr, 5239 xdpf->len, DMA_BIDIRECTIONAL); 5240 5241 buf_type = STMMAC_TXBUF_T_XDP_TX; 5242 } 5243 5244 stmmac_set_tx_dma_entry(tx_q, entry, buf_type, dma_addr, xdpf->len, 5245 false); 5246 stmmac_set_tx_dma_last_segment(tx_q, entry); 5247 5248 tx_q->xdpf[entry] = xdpf; 5249 5250 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 5251 5252 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 5253 csum, priv->descriptor_mode, true, true, 5254 xdpf->len); 5255 5256 tx_q->tx_count_frames++; 5257 5258 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 5259 set_ic = true; 5260 else 5261 set_ic = false; 5262 5263 if (set_ic) { 5264 tx_q->tx_count_frames = 0; 5265 stmmac_set_tx_ic(priv, tx_desc); 5266 u64_stats_update_begin(&txq_stats->q_syncp); 5267 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 5268 u64_stats_update_end(&txq_stats->q_syncp); 5269 } 5270 5271 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5272 5273 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); 5274 tx_q->cur_tx = entry; 5275 5276 return STMMAC_XDP_TX; 5277 } 5278 5279 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 5280 int cpu) 5281 { 5282 int index = cpu; 5283 5284 if (unlikely(index < 0)) 5285 index = 0; 5286 5287 while (index >= priv->plat->tx_queues_to_use) 5288 index -= priv->plat->tx_queues_to_use; 5289 5290 return index; 5291 } 5292 5293 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 5294 struct xdp_buff *xdp) 5295 { 5296 bool zc = !!(xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL); 5297 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 5298 int cpu = smp_processor_id(); 5299 struct netdev_queue *nq; 5300 int queue; 5301 int res; 5302 5303 if (unlikely(!xdpf)) 5304 return STMMAC_XDP_CONSUMED; 5305 5306 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5307 nq = netdev_get_tx_queue(priv->dev, queue); 5308 5309 __netif_tx_lock(nq, cpu); 5310 /* Avoids TX time-out as we are sharing with slow path */ 5311 txq_trans_cond_update(nq); 5312 5313 /* For zero copy XDP_TX action, dma_map is true */ 5314 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, zc); 5315 if (res == STMMAC_XDP_TX) { 5316 stmmac_flush_tx_descriptors(priv, queue); 5317 } else if (res == STMMAC_XDP_CONSUMED && zc) { 5318 /* xdp has been freed by xdp_convert_buff_to_frame(), 5319 * no need to call xsk_buff_free() again, so return 5320 * STMMAC_XSK_CONSUMED. 5321 */ 5322 res = STMMAC_XSK_CONSUMED; 5323 xdp_return_frame(xdpf); 5324 } 5325 5326 __netif_tx_unlock(nq); 5327 5328 return res; 5329 } 5330 5331 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 5332 struct bpf_prog *prog, 5333 struct xdp_buff *xdp) 5334 { 5335 u32 act; 5336 int res; 5337 5338 act = bpf_prog_run_xdp(prog, xdp); 5339 switch (act) { 5340 case XDP_PASS: 5341 res = STMMAC_XDP_PASS; 5342 break; 5343 case XDP_TX: 5344 res = stmmac_xdp_xmit_back(priv, xdp); 5345 break; 5346 case XDP_REDIRECT: 5347 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 5348 res = STMMAC_XDP_CONSUMED; 5349 else 5350 res = STMMAC_XDP_REDIRECT; 5351 break; 5352 default: 5353 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5354 fallthrough; 5355 case XDP_ABORTED: 5356 trace_xdp_exception(priv->dev, prog, act); 5357 fallthrough; 5358 case XDP_DROP: 5359 res = STMMAC_XDP_CONSUMED; 5360 break; 5361 } 5362 5363 return res; 5364 } 5365 5366 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5367 struct xdp_buff *xdp) 5368 { 5369 struct bpf_prog *prog; 5370 int res; 5371 5372 prog = READ_ONCE(priv->xdp_prog); 5373 if (!prog) { 5374 res = STMMAC_XDP_PASS; 5375 goto out; 5376 } 5377 5378 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5379 out: 5380 return ERR_PTR(-res); 5381 } 5382 5383 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5384 int xdp_status) 5385 { 5386 int cpu = smp_processor_id(); 5387 int queue; 5388 5389 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5390 5391 if (xdp_status & STMMAC_XDP_TX) 5392 stmmac_tx_timer_arm(priv, queue); 5393 5394 if (xdp_status & STMMAC_XDP_REDIRECT) 5395 xdp_do_flush(); 5396 } 5397 5398 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5399 struct xdp_buff *xdp) 5400 { 5401 unsigned int metasize = xdp->data - xdp->data_meta; 5402 unsigned int datasize = xdp->data_end - xdp->data; 5403 struct sk_buff *skb; 5404 5405 skb = napi_alloc_skb(&ch->rxtx_napi, 5406 xdp->data_end - xdp->data_hard_start); 5407 if (unlikely(!skb)) 5408 return NULL; 5409 5410 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5411 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5412 if (metasize) 5413 skb_metadata_set(skb, metasize); 5414 5415 return skb; 5416 } 5417 5418 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5419 struct dma_desc *p, struct dma_desc *np, 5420 struct xdp_buff *xdp) 5421 { 5422 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5423 struct stmmac_channel *ch = &priv->channel[queue]; 5424 unsigned int len = xdp->data_end - xdp->data; 5425 enum pkt_hash_types hash_type; 5426 int coe = priv->hw->rx_csum; 5427 struct sk_buff *skb; 5428 u32 hash; 5429 5430 skb = stmmac_construct_skb_zc(ch, xdp); 5431 if (!skb) { 5432 priv->xstats.rx_dropped++; 5433 return; 5434 } 5435 5436 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5437 if (priv->hw->hw_vlan_en) 5438 /* MAC level stripping. */ 5439 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5440 else 5441 /* Driver level stripping. */ 5442 stmmac_rx_vlan(priv->dev, skb); 5443 skb->protocol = eth_type_trans(skb, priv->dev); 5444 5445 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5446 skb_checksum_none_assert(skb); 5447 else 5448 skb->ip_summed = CHECKSUM_UNNECESSARY; 5449 5450 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5451 skb_set_hash(skb, hash, hash_type); 5452 5453 skb_record_rx_queue(skb, queue); 5454 napi_gro_receive(&ch->rxtx_napi, skb); 5455 5456 u64_stats_update_begin(&rxq_stats->napi_syncp); 5457 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5458 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5459 u64_stats_update_end(&rxq_stats->napi_syncp); 5460 } 5461 5462 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5463 { 5464 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5465 unsigned int entry = rx_q->dirty_rx; 5466 struct dma_desc *rx_desc = NULL; 5467 bool ret = true; 5468 5469 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5470 5471 while (budget-- > 0 && entry != rx_q->cur_rx) { 5472 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5473 dma_addr_t dma_addr; 5474 bool use_rx_wd; 5475 5476 if (!buf->xdp) { 5477 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5478 if (!buf->xdp) { 5479 ret = false; 5480 break; 5481 } 5482 } 5483 5484 rx_desc = stmmac_get_rx_desc(priv, rx_q, entry); 5485 5486 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5487 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5488 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5489 stmmac_refill_desc3(priv, rx_q, rx_desc); 5490 5491 rx_q->rx_count_frames++; 5492 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5493 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5494 rx_q->rx_count_frames = 0; 5495 5496 use_rx_wd = !priv->rx_coal_frames[queue]; 5497 use_rx_wd |= rx_q->rx_count_frames > 0; 5498 if (!priv->use_riwt) 5499 use_rx_wd = false; 5500 5501 dma_wmb(); 5502 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5503 5504 entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); 5505 } 5506 5507 if (rx_desc) { 5508 rx_q->dirty_rx = entry; 5509 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); 5510 } 5511 5512 return ret; 5513 } 5514 5515 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5516 { 5517 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5518 * to represent incoming packet, whereas cb field in the same structure 5519 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5520 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5521 */ 5522 return (struct stmmac_xdp_buff *)xdp; 5523 } 5524 5525 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5526 { 5527 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5528 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5529 unsigned int count = 0, error = 0, len = 0; 5530 int dirty = stmmac_rx_dirty(priv, queue); 5531 unsigned int next_entry = rx_q->cur_rx; 5532 u32 rx_errors = 0, rx_dropped = 0; 5533 unsigned int desc_size; 5534 struct bpf_prog *prog; 5535 bool failure = false; 5536 int xdp_status = 0; 5537 int status = 0; 5538 5539 if (netif_msg_rx_status(priv)) { 5540 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5541 5542 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5543 desc_size = stmmac_get_rx_desc_size(priv); 5544 5545 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5546 rx_q->dma_rx_phy, desc_size); 5547 } 5548 while (count < limit) { 5549 struct stmmac_rx_buffer *buf; 5550 struct stmmac_xdp_buff *ctx; 5551 unsigned int buf1_len = 0; 5552 struct dma_desc *np, *p; 5553 int entry; 5554 int res; 5555 5556 if (!count && rx_q->state_saved) { 5557 error = rx_q->state.error; 5558 len = rx_q->state.len; 5559 } else { 5560 rx_q->state_saved = false; 5561 error = 0; 5562 len = 0; 5563 } 5564 5565 read_again: 5566 if (count >= limit) 5567 break; 5568 5569 buf1_len = 0; 5570 entry = next_entry; 5571 buf = &rx_q->buf_pool[entry]; 5572 5573 if (dirty >= STMMAC_RX_FILL_BATCH) { 5574 failure = failure || 5575 !stmmac_rx_refill_zc(priv, queue, dirty); 5576 dirty = 0; 5577 } 5578 5579 p = stmmac_get_rx_desc(priv, rx_q, entry); 5580 5581 /* read the status of the incoming frame */ 5582 status = stmmac_rx_status(priv, &priv->xstats, p); 5583 /* check if managed by the DMA otherwise go ahead */ 5584 if (unlikely(status & dma_own)) 5585 break; 5586 5587 /* Prefetch the next RX descriptor */ 5588 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5589 priv->dma_conf.dma_rx_size); 5590 if (unlikely(next_entry == rx_q->dirty_rx)) 5591 break; 5592 5593 rx_q->cur_rx = next_entry; 5594 5595 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5596 5597 prefetch(np); 5598 5599 /* Ensure a valid XSK buffer before proceed */ 5600 if (!buf->xdp) 5601 break; 5602 5603 if (priv->extend_desc) 5604 stmmac_rx_extended_status(priv, &priv->xstats, 5605 rx_q->dma_erx + entry); 5606 if (unlikely(status == discard_frame)) { 5607 xsk_buff_free(buf->xdp); 5608 buf->xdp = NULL; 5609 dirty++; 5610 error = 1; 5611 if (!priv->hwts_rx_en) 5612 rx_errors++; 5613 } 5614 5615 if (unlikely(error && (status & rx_not_ls))) 5616 goto read_again; 5617 if (unlikely(error)) { 5618 count++; 5619 continue; 5620 } 5621 5622 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5623 if (likely(status & rx_not_ls)) { 5624 xsk_buff_free(buf->xdp); 5625 buf->xdp = NULL; 5626 dirty++; 5627 count++; 5628 goto read_again; 5629 } 5630 5631 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5632 ctx->priv = priv; 5633 ctx->desc = p; 5634 ctx->ndesc = np; 5635 5636 /* XDP ZC Frame only support primary buffers for now */ 5637 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5638 len += buf1_len; 5639 5640 /* ACS is disabled; strip manually. */ 5641 if (likely(!(status & rx_not_ls))) { 5642 buf1_len -= ETH_FCS_LEN; 5643 len -= ETH_FCS_LEN; 5644 } 5645 5646 /* RX buffer is good and fit into a XSK pool buffer */ 5647 buf->xdp->data_end = buf->xdp->data + buf1_len; 5648 xsk_buff_dma_sync_for_cpu(buf->xdp); 5649 5650 prog = READ_ONCE(priv->xdp_prog); 5651 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5652 5653 switch (res) { 5654 case STMMAC_XDP_PASS: 5655 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5656 xsk_buff_free(buf->xdp); 5657 break; 5658 case STMMAC_XDP_CONSUMED: 5659 xsk_buff_free(buf->xdp); 5660 fallthrough; 5661 case STMMAC_XSK_CONSUMED: 5662 rx_dropped++; 5663 break; 5664 case STMMAC_XDP_TX: 5665 case STMMAC_XDP_REDIRECT: 5666 xdp_status |= res; 5667 break; 5668 } 5669 5670 buf->xdp = NULL; 5671 dirty++; 5672 count++; 5673 } 5674 5675 if (status & rx_not_ls) { 5676 rx_q->state_saved = true; 5677 rx_q->state.error = error; 5678 rx_q->state.len = len; 5679 } 5680 5681 stmmac_finalize_xdp_rx(priv, xdp_status); 5682 5683 u64_stats_update_begin(&rxq_stats->napi_syncp); 5684 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5685 u64_stats_update_end(&rxq_stats->napi_syncp); 5686 5687 priv->xstats.rx_dropped += rx_dropped; 5688 priv->xstats.rx_errors += rx_errors; 5689 5690 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5691 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5692 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5693 else 5694 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5695 5696 return (int)count; 5697 } 5698 5699 return failure ? limit : (int)count; 5700 } 5701 5702 /** 5703 * stmmac_rx - manage the receive process 5704 * @priv: driver private structure 5705 * @limit: napi bugget 5706 * @queue: RX queue index. 5707 * Description : this the function called by the napi poll method. 5708 * It gets all the frames inside the ring. 5709 */ 5710 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5711 { 5712 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5713 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5714 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5715 struct stmmac_channel *ch = &priv->channel[queue]; 5716 unsigned int count = 0, error = 0, len = 0; 5717 int status = 0, coe = priv->hw->rx_csum; 5718 unsigned int next_entry = rx_q->cur_rx; 5719 enum dma_data_direction dma_dir; 5720 unsigned int desc_size; 5721 struct sk_buff *skb = NULL; 5722 struct stmmac_xdp_buff ctx; 5723 int xdp_status = 0; 5724 int bufsz; 5725 5726 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5727 bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5728 5729 if (netif_msg_rx_status(priv)) { 5730 void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); 5731 5732 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5733 desc_size = stmmac_get_rx_desc_size(priv); 5734 5735 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5736 rx_q->dma_rx_phy, desc_size); 5737 } 5738 while (count < limit) { 5739 unsigned int buf1_len = 0, buf2_len = 0; 5740 enum pkt_hash_types hash_type; 5741 struct stmmac_rx_buffer *buf; 5742 struct dma_desc *np, *p; 5743 int entry; 5744 u32 hash; 5745 5746 if (!count && rx_q->state_saved) { 5747 skb = rx_q->state.skb; 5748 error = rx_q->state.error; 5749 len = rx_q->state.len; 5750 } else { 5751 rx_q->state_saved = false; 5752 skb = NULL; 5753 error = 0; 5754 len = 0; 5755 } 5756 5757 read_again: 5758 if (count >= limit) 5759 break; 5760 5761 buf1_len = 0; 5762 buf2_len = 0; 5763 entry = next_entry; 5764 buf = &rx_q->buf_pool[entry]; 5765 5766 p = stmmac_get_rx_desc(priv, rx_q, entry); 5767 5768 /* read the status of the incoming frame */ 5769 status = stmmac_rx_status(priv, &priv->xstats, p); 5770 /* check if managed by the DMA otherwise go ahead */ 5771 if (unlikely(status & dma_own)) 5772 break; 5773 5774 next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, 5775 priv->dma_conf.dma_rx_size); 5776 if (unlikely(next_entry == rx_q->dirty_rx)) 5777 break; 5778 5779 rx_q->cur_rx = next_entry; 5780 5781 np = stmmac_get_rx_desc(priv, rx_q, next_entry); 5782 5783 prefetch(np); 5784 5785 if (priv->extend_desc) 5786 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5787 if (unlikely(status == discard_frame)) { 5788 page_pool_put_page(rx_q->page_pool, buf->page, 0, true); 5789 buf->page = NULL; 5790 error = 1; 5791 if (!priv->hwts_rx_en) 5792 rx_errors++; 5793 } 5794 5795 if (unlikely(error && (status & rx_not_ls))) 5796 goto read_again; 5797 if (unlikely(error)) { 5798 dev_kfree_skb(skb); 5799 skb = NULL; 5800 count++; 5801 continue; 5802 } 5803 5804 /* Buffer is good. Go on. */ 5805 5806 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5807 len += buf1_len; 5808 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5809 len += buf2_len; 5810 5811 /* ACS is disabled; strip manually. */ 5812 if (likely(!(status & rx_not_ls))) { 5813 if (buf2_len) { 5814 buf2_len -= ETH_FCS_LEN; 5815 len -= ETH_FCS_LEN; 5816 } else if (buf1_len) { 5817 buf1_len -= ETH_FCS_LEN; 5818 len -= ETH_FCS_LEN; 5819 } 5820 } 5821 5822 if (!skb) { 5823 unsigned int pre_len, sync_len; 5824 5825 dma_sync_single_for_cpu(priv->device, buf->addr, 5826 buf1_len, dma_dir); 5827 net_prefetch(page_address(buf->page) + 5828 buf->page_offset); 5829 5830 xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq); 5831 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5832 buf->page_offset, buf1_len, true); 5833 5834 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5835 buf->page_offset; 5836 5837 ctx.priv = priv; 5838 ctx.desc = p; 5839 ctx.ndesc = np; 5840 5841 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5842 /* Due xdp_adjust_tail: DMA sync for_device 5843 * cover max len CPU touch 5844 */ 5845 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5846 buf->page_offset; 5847 sync_len = max(sync_len, pre_len); 5848 5849 /* For Not XDP_PASS verdict */ 5850 if (IS_ERR(skb)) { 5851 unsigned int xdp_res = -PTR_ERR(skb); 5852 5853 if (xdp_res & STMMAC_XDP_CONSUMED) { 5854 page_pool_put_page(rx_q->page_pool, 5855 virt_to_head_page(ctx.xdp.data), 5856 sync_len, true); 5857 buf->page = NULL; 5858 rx_dropped++; 5859 5860 /* Clear skb as it was set as 5861 * status by XDP program. 5862 */ 5863 skb = NULL; 5864 5865 if (unlikely((status & rx_not_ls))) 5866 goto read_again; 5867 5868 count++; 5869 continue; 5870 } else if (xdp_res & (STMMAC_XDP_TX | 5871 STMMAC_XDP_REDIRECT)) { 5872 xdp_status |= xdp_res; 5873 buf->page = NULL; 5874 skb = NULL; 5875 count++; 5876 continue; 5877 } 5878 } 5879 } 5880 5881 if (!skb) { 5882 unsigned int head_pad_len; 5883 5884 /* XDP program may expand or reduce tail */ 5885 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5886 5887 skb = napi_build_skb(page_address(buf->page), 5888 rx_q->napi_skb_frag_size); 5889 if (!skb) { 5890 page_pool_recycle_direct(rx_q->page_pool, 5891 buf->page); 5892 buf->page = NULL; 5893 rx_dropped++; 5894 count++; 5895 goto drain_data; 5896 } 5897 5898 /* XDP program may adjust header */ 5899 head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; 5900 skb_reserve(skb, head_pad_len); 5901 skb_put(skb, buf1_len); 5902 skb_mark_for_recycle(skb); 5903 buf->page = NULL; 5904 } else if (buf1_len) { 5905 dma_sync_single_for_cpu(priv->device, buf->addr, 5906 buf1_len, dma_dir); 5907 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5908 buf->page, buf->page_offset, buf1_len, 5909 priv->dma_conf.dma_buf_sz); 5910 buf->page = NULL; 5911 } 5912 5913 if (buf2_len) { 5914 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5915 buf2_len, dma_dir); 5916 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5917 buf->sec_page, 0, buf2_len, 5918 priv->dma_conf.dma_buf_sz); 5919 buf->sec_page = NULL; 5920 } 5921 5922 drain_data: 5923 if (likely(status & rx_not_ls)) 5924 goto read_again; 5925 if (!skb) 5926 continue; 5927 5928 /* Got entire packet into SKB. Finish it. */ 5929 5930 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5931 5932 if (priv->hw->hw_vlan_en) 5933 /* MAC level stripping. */ 5934 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5935 else 5936 /* Driver level stripping. */ 5937 stmmac_rx_vlan(priv->dev, skb); 5938 5939 skb->protocol = eth_type_trans(skb, priv->dev); 5940 5941 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) || 5942 (status & csum_none)) 5943 skb_checksum_none_assert(skb); 5944 else 5945 skb->ip_summed = CHECKSUM_UNNECESSARY; 5946 5947 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5948 skb_set_hash(skb, hash, hash_type); 5949 5950 skb_record_rx_queue(skb, queue); 5951 napi_gro_receive(&ch->rx_napi, skb); 5952 skb = NULL; 5953 5954 rx_packets++; 5955 rx_bytes += len; 5956 count++; 5957 } 5958 5959 if (status & rx_not_ls || skb) { 5960 rx_q->state_saved = true; 5961 rx_q->state.skb = skb; 5962 rx_q->state.error = error; 5963 rx_q->state.len = len; 5964 } 5965 5966 stmmac_finalize_xdp_rx(priv, xdp_status); 5967 5968 stmmac_rx_refill(priv, queue); 5969 5970 u64_stats_update_begin(&rxq_stats->napi_syncp); 5971 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5972 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5973 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5974 u64_stats_update_end(&rxq_stats->napi_syncp); 5975 5976 priv->xstats.rx_dropped += rx_dropped; 5977 priv->xstats.rx_errors += rx_errors; 5978 5979 return count; 5980 } 5981 5982 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5983 { 5984 struct stmmac_channel *ch = 5985 container_of(napi, struct stmmac_channel, rx_napi); 5986 struct stmmac_priv *priv = ch->priv_data; 5987 struct stmmac_rxq_stats *rxq_stats; 5988 u32 chan = ch->index; 5989 int work_done; 5990 5991 rxq_stats = &priv->xstats.rxq_stats[chan]; 5992 u64_stats_update_begin(&rxq_stats->napi_syncp); 5993 u64_stats_inc(&rxq_stats->napi.poll); 5994 u64_stats_update_end(&rxq_stats->napi_syncp); 5995 5996 work_done = stmmac_rx(priv, budget, chan); 5997 if (work_done < budget && napi_complete_done(napi, work_done)) { 5998 unsigned long flags; 5999 6000 spin_lock_irqsave(&ch->lock, flags); 6001 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 6002 spin_unlock_irqrestore(&ch->lock, flags); 6003 } 6004 6005 return work_done; 6006 } 6007 6008 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 6009 { 6010 struct stmmac_channel *ch = 6011 container_of(napi, struct stmmac_channel, tx_napi); 6012 struct stmmac_priv *priv = ch->priv_data; 6013 struct stmmac_txq_stats *txq_stats; 6014 bool pending_packets = false; 6015 u32 chan = ch->index; 6016 int work_done; 6017 6018 txq_stats = &priv->xstats.txq_stats[chan]; 6019 u64_stats_update_begin(&txq_stats->napi_syncp); 6020 u64_stats_inc(&txq_stats->napi.poll); 6021 u64_stats_update_end(&txq_stats->napi_syncp); 6022 6023 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 6024 work_done = min(work_done, budget); 6025 6026 if (work_done < budget && napi_complete_done(napi, work_done)) { 6027 unsigned long flags; 6028 6029 spin_lock_irqsave(&ch->lock, flags); 6030 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 6031 spin_unlock_irqrestore(&ch->lock, flags); 6032 } 6033 6034 /* TX still have packet to handle, check if we need to arm tx timer */ 6035 if (pending_packets) 6036 stmmac_tx_timer_arm(priv, chan); 6037 6038 return work_done; 6039 } 6040 6041 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 6042 { 6043 struct stmmac_channel *ch = 6044 container_of(napi, struct stmmac_channel, rxtx_napi); 6045 struct stmmac_priv *priv = ch->priv_data; 6046 bool tx_pending_packets = false; 6047 int rx_done, tx_done, rxtx_done; 6048 struct stmmac_rxq_stats *rxq_stats; 6049 struct stmmac_txq_stats *txq_stats; 6050 u32 chan = ch->index; 6051 6052 rxq_stats = &priv->xstats.rxq_stats[chan]; 6053 u64_stats_update_begin(&rxq_stats->napi_syncp); 6054 u64_stats_inc(&rxq_stats->napi.poll); 6055 u64_stats_update_end(&rxq_stats->napi_syncp); 6056 6057 txq_stats = &priv->xstats.txq_stats[chan]; 6058 u64_stats_update_begin(&txq_stats->napi_syncp); 6059 u64_stats_inc(&txq_stats->napi.poll); 6060 u64_stats_update_end(&txq_stats->napi_syncp); 6061 6062 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 6063 tx_done = min(tx_done, budget); 6064 6065 rx_done = stmmac_rx_zc(priv, budget, chan); 6066 6067 rxtx_done = max(tx_done, rx_done); 6068 6069 /* If either TX or RX work is not complete, return budget 6070 * and keep pooling 6071 */ 6072 if (rxtx_done >= budget) 6073 return budget; 6074 6075 /* all work done, exit the polling mode */ 6076 if (napi_complete_done(napi, rxtx_done)) { 6077 unsigned long flags; 6078 6079 spin_lock_irqsave(&ch->lock, flags); 6080 /* Both RX and TX work done are complete, 6081 * so enable both RX & TX IRQs. 6082 */ 6083 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6084 spin_unlock_irqrestore(&ch->lock, flags); 6085 } 6086 6087 /* TX still have packet to handle, check if we need to arm tx timer */ 6088 if (tx_pending_packets) 6089 stmmac_tx_timer_arm(priv, chan); 6090 6091 return min(rxtx_done, budget - 1); 6092 } 6093 6094 /** 6095 * stmmac_tx_timeout 6096 * @dev : Pointer to net device structure 6097 * @txqueue: the index of the hanging transmit queue 6098 * Description: this function is called when a packet transmission fails to 6099 * complete within a reasonable time. The driver will mark the error in the 6100 * netdev structure and arrange for the device to be reset to a sane state 6101 * in order to transmit a new packet. 6102 */ 6103 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 6104 { 6105 struct stmmac_priv *priv = netdev_priv(dev); 6106 6107 stmmac_global_err(priv); 6108 } 6109 6110 /** 6111 * stmmac_set_rx_mode - entry point for multicast addressing 6112 * @dev : pointer to the device structure 6113 * Description: 6114 * This function is a driver entry point which gets called by the kernel 6115 * whenever multicast addresses must be enabled/disabled. 6116 * Return value: 6117 * void. 6118 * 6119 * FIXME: This may need RXC to be running, but it may be called with BH 6120 * disabled, which means we can't call phylink_rx_clk_stop*(). 6121 */ 6122 static void stmmac_set_rx_mode(struct net_device *dev) 6123 { 6124 struct stmmac_priv *priv = netdev_priv(dev); 6125 6126 stmmac_set_filter(priv, priv->hw, dev); 6127 } 6128 6129 /** 6130 * stmmac_change_mtu - entry point to change MTU size for the device. 6131 * @dev : device pointer. 6132 * @new_mtu : the new MTU size for the device. 6133 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 6134 * to drive packet transmission. Ethernet has an MTU of 1500 octets 6135 * (ETH_DATA_LEN). This value can be changed with ifconfig. 6136 * Return value: 6137 * 0 on success and an appropriate (-)ve integer as defined in errno.h 6138 * file on failure. 6139 */ 6140 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 6141 { 6142 struct stmmac_priv *priv = netdev_priv(dev); 6143 int txfifosz = priv->plat->tx_fifo_size; 6144 struct stmmac_dma_conf *dma_conf; 6145 const int mtu = new_mtu; 6146 int ret; 6147 6148 if (txfifosz == 0) 6149 txfifosz = priv->dma_cap.tx_fifo_size; 6150 6151 txfifosz /= priv->plat->tx_queues_to_use; 6152 6153 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 6154 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 6155 return -EINVAL; 6156 } 6157 6158 new_mtu = STMMAC_ALIGN(new_mtu); 6159 6160 /* If condition true, FIFO is too small or MTU too large */ 6161 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 6162 return -EINVAL; 6163 6164 if (netif_running(dev)) { 6165 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 6166 /* Try to allocate the new DMA conf with the new mtu */ 6167 dma_conf = stmmac_setup_dma_desc(priv, mtu); 6168 if (IS_ERR(dma_conf)) { 6169 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 6170 mtu); 6171 return PTR_ERR(dma_conf); 6172 } 6173 6174 __stmmac_release(dev); 6175 6176 ret = __stmmac_open(dev, dma_conf); 6177 if (ret) { 6178 free_dma_desc_resources(priv, dma_conf); 6179 kfree(dma_conf); 6180 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 6181 return ret; 6182 } 6183 6184 kfree(dma_conf); 6185 6186 stmmac_set_rx_mode(dev); 6187 } 6188 6189 WRITE_ONCE(dev->mtu, mtu); 6190 netdev_update_features(dev); 6191 6192 return 0; 6193 } 6194 6195 static netdev_features_t stmmac_fix_features(struct net_device *dev, 6196 netdev_features_t features) 6197 { 6198 struct stmmac_priv *priv = netdev_priv(dev); 6199 6200 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 6201 features &= ~NETIF_F_RXCSUM; 6202 6203 if (!priv->plat->tx_coe) 6204 features &= ~NETIF_F_CSUM_MASK; 6205 6206 /* Some GMAC devices have a bugged Jumbo frame support that 6207 * needs to have the Tx COE disabled for oversized frames 6208 * (due to limited buffer sizes). In this case we disable 6209 * the TX csum insertion in the TDES and not use SF. 6210 */ 6211 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 6212 features &= ~NETIF_F_CSUM_MASK; 6213 6214 return features; 6215 } 6216 6217 static int stmmac_set_features(struct net_device *netdev, 6218 netdev_features_t features) 6219 { 6220 struct stmmac_priv *priv = netdev_priv(netdev); 6221 6222 /* Keep the COE Type in case of csum is supporting */ 6223 if (features & NETIF_F_RXCSUM) 6224 priv->hw->rx_csum = priv->plat->rx_coe; 6225 else 6226 priv->hw->rx_csum = 0; 6227 /* No check needed because rx_coe has been set before and it will be 6228 * fixed in case of issue. 6229 */ 6230 stmmac_rx_ipc(priv, priv->hw); 6231 6232 if (priv->sph_capable) { 6233 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 6234 u8 chan; 6235 6236 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 6237 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6238 } 6239 6240 if (features & NETIF_F_HW_VLAN_CTAG_RX) 6241 priv->hw->hw_vlan_en = true; 6242 else 6243 priv->hw->hw_vlan_en = false; 6244 6245 phylink_rx_clk_stop_block(priv->phylink); 6246 stmmac_set_hw_vlan_mode(priv, priv->hw); 6247 phylink_rx_clk_stop_unblock(priv->phylink); 6248 6249 return 0; 6250 } 6251 6252 static void stmmac_common_interrupt(struct stmmac_priv *priv) 6253 { 6254 u8 rx_cnt = priv->plat->rx_queues_to_use; 6255 u8 tx_cnt = priv->plat->tx_queues_to_use; 6256 u8 queues_count; 6257 bool xmac; 6258 u8 queue; 6259 6260 xmac = dwmac_is_xmac(priv->plat->core_type); 6261 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 6262 6263 if (priv->irq_wake) 6264 pm_wakeup_event(priv->device, 0); 6265 6266 if (priv->dma_cap.estsel) 6267 stmmac_est_irq_status(priv, priv, priv->dev, 6268 &priv->xstats, tx_cnt); 6269 6270 if (stmmac_fpe_supported(priv)) 6271 stmmac_fpe_irq_status(priv); 6272 6273 /* To handle GMAC own interrupts */ 6274 if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) { 6275 int status = stmmac_host_irq_status(priv, &priv->xstats); 6276 6277 if (unlikely(status)) { 6278 /* For LPI we need to save the tx status */ 6279 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 6280 priv->tx_path_in_lpi_mode = true; 6281 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 6282 priv->tx_path_in_lpi_mode = false; 6283 } 6284 6285 for (queue = 0; queue < queues_count; queue++) 6286 stmmac_host_mtl_irq_status(priv, priv->hw, queue); 6287 6288 stmmac_timestamp_interrupt(priv, priv); 6289 } 6290 } 6291 6292 /** 6293 * stmmac_interrupt - main ISR 6294 * @irq: interrupt number. 6295 * @dev_id: to pass the net device pointer. 6296 * Description: this is the main driver interrupt service routine. 6297 * It can call: 6298 * o DMA service routine (to manage incoming frame reception and transmission 6299 * status) 6300 * o Core interrupts to manage: remote wake-up, management counter, LPI 6301 * interrupts. 6302 */ 6303 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6304 { 6305 struct net_device *dev = (struct net_device *)dev_id; 6306 struct stmmac_priv *priv = netdev_priv(dev); 6307 6308 /* Check if adapter is up */ 6309 if (test_bit(STMMAC_DOWN, &priv->state)) 6310 return IRQ_HANDLED; 6311 6312 /* Check ASP error if it isn't delivered via an individual IRQ */ 6313 if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) 6314 return IRQ_HANDLED; 6315 6316 /* To handle Common interrupts */ 6317 stmmac_common_interrupt(priv); 6318 6319 /* To handle DMA interrupts */ 6320 stmmac_dma_interrupt(priv); 6321 6322 return IRQ_HANDLED; 6323 } 6324 6325 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6326 { 6327 struct net_device *dev = (struct net_device *)dev_id; 6328 struct stmmac_priv *priv = netdev_priv(dev); 6329 6330 /* Check if adapter is up */ 6331 if (test_bit(STMMAC_DOWN, &priv->state)) 6332 return IRQ_HANDLED; 6333 6334 /* To handle Common interrupts */ 6335 stmmac_common_interrupt(priv); 6336 6337 return IRQ_HANDLED; 6338 } 6339 6340 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6341 { 6342 struct net_device *dev = (struct net_device *)dev_id; 6343 struct stmmac_priv *priv = netdev_priv(dev); 6344 6345 /* Check if adapter is up */ 6346 if (test_bit(STMMAC_DOWN, &priv->state)) 6347 return IRQ_HANDLED; 6348 6349 /* Check if a fatal error happened */ 6350 stmmac_safety_feat_interrupt(priv); 6351 6352 return IRQ_HANDLED; 6353 } 6354 6355 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6356 { 6357 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6358 struct stmmac_dma_conf *dma_conf; 6359 int chan = tx_q->queue_index; 6360 struct stmmac_priv *priv; 6361 int status; 6362 6363 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6364 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6365 6366 /* Check if adapter is up */ 6367 if (test_bit(STMMAC_DOWN, &priv->state)) 6368 return IRQ_HANDLED; 6369 6370 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6371 6372 if (unlikely(status & tx_hard_error_bump_tc)) { 6373 /* Try to bump up the dma threshold on this failure */ 6374 stmmac_bump_dma_threshold(priv, chan); 6375 } else if (unlikely(status == tx_hard_error)) { 6376 stmmac_tx_err(priv, chan); 6377 } 6378 6379 return IRQ_HANDLED; 6380 } 6381 6382 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6383 { 6384 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6385 struct stmmac_dma_conf *dma_conf; 6386 int chan = rx_q->queue_index; 6387 struct stmmac_priv *priv; 6388 6389 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6390 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6391 6392 /* Check if adapter is up */ 6393 if (test_bit(STMMAC_DOWN, &priv->state)) 6394 return IRQ_HANDLED; 6395 6396 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6397 6398 return IRQ_HANDLED; 6399 } 6400 6401 /** 6402 * stmmac_ioctl - Entry point for the Ioctl 6403 * @dev: Device pointer. 6404 * @rq: An IOCTL specific structure, that can contain a pointer to 6405 * a proprietary structure used to pass information to the driver. 6406 * @cmd: IOCTL command 6407 * Description: Forward the PHY ioctls to phylink 6408 * Return: Zero on success or negative error code. 6409 */ 6410 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6411 { 6412 struct stmmac_priv *priv = netdev_priv (dev); 6413 6414 if (!netif_running(dev)) 6415 return -EINVAL; 6416 6417 return phylink_mii_ioctl(priv->phylink, rq, cmd); 6418 } 6419 6420 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6421 void *cb_priv) 6422 { 6423 struct stmmac_priv *priv = cb_priv; 6424 int ret = -EOPNOTSUPP; 6425 6426 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6427 return ret; 6428 6429 __stmmac_disable_all_queues(priv); 6430 6431 switch (type) { 6432 case TC_SETUP_CLSU32: 6433 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6434 break; 6435 case TC_SETUP_CLSFLOWER: 6436 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6437 break; 6438 default: 6439 break; 6440 } 6441 6442 stmmac_enable_all_queues(priv); 6443 return ret; 6444 } 6445 6446 static LIST_HEAD(stmmac_block_cb_list); 6447 6448 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6449 void *type_data) 6450 { 6451 struct stmmac_priv *priv = netdev_priv(ndev); 6452 6453 switch (type) { 6454 case TC_QUERY_CAPS: 6455 return stmmac_tc_query_caps(priv, priv, type_data); 6456 case TC_SETUP_QDISC_MQPRIO: 6457 return stmmac_tc_setup_mqprio(priv, priv, type_data); 6458 case TC_SETUP_BLOCK: 6459 return flow_block_cb_setup_simple(type_data, 6460 &stmmac_block_cb_list, 6461 stmmac_setup_tc_block_cb, 6462 priv, priv, true); 6463 case TC_SETUP_QDISC_CBS: 6464 return stmmac_tc_setup_cbs(priv, priv, type_data); 6465 case TC_SETUP_QDISC_TAPRIO: 6466 return stmmac_tc_setup_taprio(priv, priv, type_data); 6467 case TC_SETUP_QDISC_ETF: 6468 return stmmac_tc_setup_etf(priv, priv, type_data); 6469 default: 6470 return -EOPNOTSUPP; 6471 } 6472 } 6473 6474 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6475 struct net_device *sb_dev) 6476 { 6477 int gso = skb_shinfo(skb)->gso_type; 6478 6479 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6480 /* 6481 * There is no way to determine the number of TSO/USO 6482 * capable Queues. Let's use always the Queue 0 6483 * because if TSO/USO is supported then at least this 6484 * one will be capable. 6485 */ 6486 return 0; 6487 } 6488 6489 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6490 } 6491 6492 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6493 { 6494 struct stmmac_priv *priv = netdev_priv(ndev); 6495 int ret = 0; 6496 6497 ret = pm_runtime_resume_and_get(priv->device); 6498 if (ret < 0) 6499 return ret; 6500 6501 ret = eth_mac_addr(ndev, addr); 6502 if (ret) 6503 goto set_mac_error; 6504 6505 phylink_rx_clk_stop_block(priv->phylink); 6506 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6507 phylink_rx_clk_stop_unblock(priv->phylink); 6508 6509 set_mac_error: 6510 pm_runtime_put(priv->device); 6511 6512 return ret; 6513 } 6514 6515 #ifdef CONFIG_DEBUG_FS 6516 static struct dentry *stmmac_fs_dir; 6517 6518 static void sysfs_display_ring(void *head, int size, int extend_desc, 6519 struct seq_file *seq, dma_addr_t dma_phy_addr) 6520 { 6521 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6522 struct dma_desc *p = (struct dma_desc *)head; 6523 unsigned int desc_size; 6524 dma_addr_t dma_addr; 6525 int i; 6526 6527 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6528 for (i = 0; i < size; i++) { 6529 dma_addr = dma_phy_addr + i * desc_size; 6530 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6531 i, &dma_addr, 6532 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6533 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6534 if (extend_desc) 6535 p = &(++ep)->basic; 6536 else 6537 p++; 6538 } 6539 } 6540 6541 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6542 { 6543 struct net_device *dev = seq->private; 6544 struct stmmac_priv *priv = netdev_priv(dev); 6545 u8 rx_count = priv->plat->rx_queues_to_use; 6546 u8 tx_count = priv->plat->tx_queues_to_use; 6547 u8 queue; 6548 6549 if ((dev->flags & IFF_UP) == 0) 6550 return 0; 6551 6552 for (queue = 0; queue < rx_count; queue++) { 6553 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6554 6555 seq_printf(seq, "RX Queue %d:\n", queue); 6556 6557 if (priv->extend_desc) { 6558 seq_printf(seq, "Extended descriptor ring:\n"); 6559 sysfs_display_ring((void *)rx_q->dma_erx, 6560 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6561 } else { 6562 seq_printf(seq, "Descriptor ring:\n"); 6563 sysfs_display_ring((void *)rx_q->dma_rx, 6564 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6565 } 6566 } 6567 6568 for (queue = 0; queue < tx_count; queue++) { 6569 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6570 6571 seq_printf(seq, "TX Queue %d:\n", queue); 6572 6573 if (priv->extend_desc) { 6574 seq_printf(seq, "Extended descriptor ring:\n"); 6575 sysfs_display_ring((void *)tx_q->dma_etx, 6576 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6577 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6578 seq_printf(seq, "Descriptor ring:\n"); 6579 sysfs_display_ring((void *)tx_q->dma_tx, 6580 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6581 } 6582 } 6583 6584 return 0; 6585 } 6586 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6587 6588 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6589 { 6590 static const char * const dwxgmac_timestamp_source[] = { 6591 "None", 6592 "Internal", 6593 "External", 6594 "Both", 6595 }; 6596 static const char * const dwxgmac_safety_feature_desc[] = { 6597 "No", 6598 "All Safety Features with ECC and Parity", 6599 "All Safety Features without ECC or Parity", 6600 "All Safety Features with Parity Only", 6601 "ECC Only", 6602 "UNDEFINED", 6603 "UNDEFINED", 6604 "UNDEFINED", 6605 }; 6606 struct net_device *dev = seq->private; 6607 struct stmmac_priv *priv = netdev_priv(dev); 6608 6609 if (!priv->hw_cap_support) { 6610 seq_printf(seq, "DMA HW features not supported\n"); 6611 return 0; 6612 } 6613 6614 seq_printf(seq, "==============================\n"); 6615 seq_printf(seq, "\tDMA HW features\n"); 6616 seq_printf(seq, "==============================\n"); 6617 6618 seq_printf(seq, "\t10/100 Mbps: %s\n", 6619 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6620 seq_printf(seq, "\t1000 Mbps: %s\n", 6621 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6622 seq_printf(seq, "\tHalf duplex: %s\n", 6623 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6624 if (priv->plat->core_type == DWMAC_CORE_XGMAC) { 6625 seq_printf(seq, 6626 "\tNumber of Additional MAC address registers: %d\n", 6627 priv->dma_cap.multi_addr); 6628 } else { 6629 seq_printf(seq, "\tHash Filter: %s\n", 6630 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6631 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6632 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6633 } 6634 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6635 (priv->dma_cap.pcs) ? "Y" : "N"); 6636 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6637 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6638 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6639 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6640 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6641 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6642 seq_printf(seq, "\tRMON module: %s\n", 6643 (priv->dma_cap.rmon) ? "Y" : "N"); 6644 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6645 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6646 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6647 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6648 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 6649 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6650 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6651 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6652 (priv->dma_cap.eee) ? "Y" : "N"); 6653 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6654 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6655 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6656 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6657 priv->plat->core_type == DWMAC_CORE_XGMAC) { 6658 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6659 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6660 } else { 6661 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6662 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6663 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6664 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6665 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6666 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6667 } 6668 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6669 priv->dma_cap.number_rx_channel); 6670 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6671 priv->dma_cap.number_tx_channel); 6672 seq_printf(seq, "\tNumber of Additional RX queues: %u\n", 6673 priv->dma_cap.number_rx_queues); 6674 seq_printf(seq, "\tNumber of Additional TX queues: %u\n", 6675 priv->dma_cap.number_tx_queues); 6676 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6677 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6678 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6679 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6680 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6681 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6682 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6683 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6684 priv->dma_cap.pps_out_num); 6685 seq_printf(seq, "\tSafety Features: %s\n", 6686 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6687 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6688 priv->dma_cap.frpsel ? "Y" : "N"); 6689 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6690 priv->dma_cap.host_dma_width); 6691 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6692 priv->dma_cap.rssen ? "Y" : "N"); 6693 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6694 priv->dma_cap.vlhash ? "Y" : "N"); 6695 seq_printf(seq, "\tSplit Header: %s\n", 6696 priv->dma_cap.sphen ? "Y" : "N"); 6697 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6698 priv->dma_cap.vlins ? "Y" : "N"); 6699 seq_printf(seq, "\tDouble VLAN: %s\n", 6700 priv->dma_cap.dvlan ? "Y" : "N"); 6701 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6702 priv->dma_cap.l3l4fnum); 6703 seq_printf(seq, "\tARP Offloading: %s\n", 6704 priv->dma_cap.arpoffsel ? "Y" : "N"); 6705 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6706 priv->dma_cap.estsel ? "Y" : "N"); 6707 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6708 priv->dma_cap.fpesel ? "Y" : "N"); 6709 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6710 priv->dma_cap.tbssel ? "Y" : "N"); 6711 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6712 priv->dma_cap.tbs_ch_num); 6713 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6714 priv->dma_cap.sgfsel ? "Y" : "N"); 6715 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6716 BIT(priv->dma_cap.ttsfd) >> 1); 6717 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6718 priv->dma_cap.numtc); 6719 seq_printf(seq, "\tDCB Feature: %s\n", 6720 priv->dma_cap.dcben ? "Y" : "N"); 6721 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6722 priv->dma_cap.advthword ? "Y" : "N"); 6723 seq_printf(seq, "\tPTP Offload: %s\n", 6724 priv->dma_cap.ptoen ? "Y" : "N"); 6725 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6726 priv->dma_cap.osten ? "Y" : "N"); 6727 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6728 priv->dma_cap.pfcen ? "Y" : "N"); 6729 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6730 BIT(priv->dma_cap.frpes) << 6); 6731 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6732 BIT(priv->dma_cap.frpbs) << 6); 6733 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6734 priv->dma_cap.frppipe_num); 6735 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6736 priv->dma_cap.nrvf_num ? 6737 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6738 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6739 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6740 seq_printf(seq, "\tDepth of GCL: %lu\n", 6741 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6742 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6743 priv->dma_cap.cbtisel ? "Y" : "N"); 6744 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6745 priv->dma_cap.aux_snapshot_n); 6746 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6747 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6748 seq_printf(seq, "\tEnhanced DMA: %s\n", 6749 priv->dma_cap.edma ? "Y" : "N"); 6750 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6751 priv->dma_cap.ediffc ? "Y" : "N"); 6752 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6753 priv->dma_cap.vxn ? "Y" : "N"); 6754 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6755 priv->dma_cap.dbgmem ? "Y" : "N"); 6756 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6757 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6758 return 0; 6759 } 6760 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6761 6762 /* Use network device events to rename debugfs file entries. 6763 */ 6764 static int stmmac_device_event(struct notifier_block *unused, 6765 unsigned long event, void *ptr) 6766 { 6767 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6768 struct stmmac_priv *priv = netdev_priv(dev); 6769 6770 if (dev->netdev_ops != &stmmac_netdev_ops) 6771 goto done; 6772 6773 switch (event) { 6774 case NETDEV_CHANGENAME: 6775 debugfs_change_name(priv->dbgfs_dir, "%s", dev->name); 6776 break; 6777 } 6778 done: 6779 return NOTIFY_DONE; 6780 } 6781 6782 static struct notifier_block stmmac_notifier = { 6783 .notifier_call = stmmac_device_event, 6784 }; 6785 6786 static void stmmac_init_fs(struct net_device *dev) 6787 { 6788 struct stmmac_priv *priv = netdev_priv(dev); 6789 6790 rtnl_lock(); 6791 6792 /* Create per netdev entries */ 6793 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6794 6795 /* Entry to report DMA RX/TX rings */ 6796 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6797 &stmmac_rings_status_fops); 6798 6799 /* Entry to report the DMA HW features */ 6800 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6801 &stmmac_dma_cap_fops); 6802 6803 rtnl_unlock(); 6804 } 6805 6806 static void stmmac_exit_fs(struct net_device *dev) 6807 { 6808 struct stmmac_priv *priv = netdev_priv(dev); 6809 6810 debugfs_remove_recursive(priv->dbgfs_dir); 6811 } 6812 #endif /* CONFIG_DEBUG_FS */ 6813 6814 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6815 { 6816 unsigned char *data = (unsigned char *)&vid_le; 6817 unsigned char data_byte = 0; 6818 u32 crc = ~0x0; 6819 u32 temp = 0; 6820 int i, bits; 6821 6822 bits = get_bitmask_order(VLAN_VID_MASK); 6823 for (i = 0; i < bits; i++) { 6824 if ((i % 8) == 0) 6825 data_byte = data[i / 8]; 6826 6827 temp = ((crc & 1) ^ data_byte) & 1; 6828 crc >>= 1; 6829 data_byte >>= 1; 6830 6831 if (temp) 6832 crc ^= 0xedb88320; 6833 } 6834 6835 return crc; 6836 } 6837 6838 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6839 { 6840 u32 crc, hash = 0; 6841 u16 pmatch = 0; 6842 int count = 0; 6843 u16 vid = 0; 6844 6845 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6846 __le16 vid_le = cpu_to_le16(vid); 6847 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6848 hash |= (1 << crc); 6849 count++; 6850 } 6851 6852 if (!priv->dma_cap.vlhash) { 6853 if (count > 2) /* VID = 0 always passes filter */ 6854 return -EOPNOTSUPP; 6855 6856 pmatch = vid; 6857 hash = 0; 6858 } 6859 6860 if (!netif_running(priv->dev)) 6861 return 0; 6862 6863 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6864 } 6865 6866 /* FIXME: This may need RXC to be running, but it may be called with BH 6867 * disabled, which means we can't call phylink_rx_clk_stop*(). 6868 */ 6869 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6870 { 6871 struct stmmac_priv *priv = netdev_priv(ndev); 6872 unsigned int num_double_vlans; 6873 bool is_double = false; 6874 int ret; 6875 6876 ret = pm_runtime_resume_and_get(priv->device); 6877 if (ret < 0) 6878 return ret; 6879 6880 if (be16_to_cpu(proto) == ETH_P_8021AD) 6881 is_double = true; 6882 6883 set_bit(vid, priv->active_vlans); 6884 num_double_vlans = priv->num_double_vlans + is_double; 6885 ret = stmmac_vlan_update(priv, num_double_vlans); 6886 if (ret) { 6887 clear_bit(vid, priv->active_vlans); 6888 goto err_pm_put; 6889 } 6890 6891 if (priv->hw->num_vlan) { 6892 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6893 if (ret) { 6894 clear_bit(vid, priv->active_vlans); 6895 stmmac_vlan_update(priv, priv->num_double_vlans); 6896 goto err_pm_put; 6897 } 6898 } 6899 6900 priv->num_double_vlans = num_double_vlans; 6901 6902 err_pm_put: 6903 pm_runtime_put(priv->device); 6904 6905 return ret; 6906 } 6907 6908 /* FIXME: This may need RXC to be running, but it may be called with BH 6909 * disabled, which means we can't call phylink_rx_clk_stop*(). 6910 */ 6911 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6912 { 6913 struct stmmac_priv *priv = netdev_priv(ndev); 6914 unsigned int num_double_vlans; 6915 bool is_double = false; 6916 int ret; 6917 6918 ret = pm_runtime_resume_and_get(priv->device); 6919 if (ret < 0) 6920 return ret; 6921 6922 if (be16_to_cpu(proto) == ETH_P_8021AD) 6923 is_double = true; 6924 6925 clear_bit(vid, priv->active_vlans); 6926 num_double_vlans = priv->num_double_vlans - is_double; 6927 ret = stmmac_vlan_update(priv, num_double_vlans); 6928 if (ret) { 6929 set_bit(vid, priv->active_vlans); 6930 goto del_vlan_error; 6931 } 6932 6933 if (priv->hw->num_vlan) { 6934 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6935 if (ret) { 6936 set_bit(vid, priv->active_vlans); 6937 stmmac_vlan_update(priv, priv->num_double_vlans); 6938 goto del_vlan_error; 6939 } 6940 } 6941 6942 priv->num_double_vlans = num_double_vlans; 6943 6944 del_vlan_error: 6945 pm_runtime_put(priv->device); 6946 6947 return ret; 6948 } 6949 6950 static void stmmac_vlan_restore(struct stmmac_priv *priv) 6951 { 6952 if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) 6953 return; 6954 6955 if (priv->hw->num_vlan) 6956 stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); 6957 6958 stmmac_vlan_update(priv, priv->num_double_vlans); 6959 } 6960 6961 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6962 { 6963 struct stmmac_priv *priv = netdev_priv(dev); 6964 6965 switch (bpf->command) { 6966 case XDP_SETUP_PROG: 6967 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6968 case XDP_SETUP_XSK_POOL: 6969 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6970 bpf->xsk.queue_id); 6971 default: 6972 return -EOPNOTSUPP; 6973 } 6974 } 6975 6976 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6977 struct xdp_frame **frames, u32 flags) 6978 { 6979 struct stmmac_priv *priv = netdev_priv(dev); 6980 int cpu = smp_processor_id(); 6981 struct netdev_queue *nq; 6982 int i, nxmit = 0; 6983 int queue; 6984 6985 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6986 return -ENETDOWN; 6987 6988 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6989 return -EINVAL; 6990 6991 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6992 nq = netdev_get_tx_queue(priv->dev, queue); 6993 6994 __netif_tx_lock(nq, cpu); 6995 /* Avoids TX time-out as we are sharing with slow path */ 6996 txq_trans_cond_update(nq); 6997 6998 for (i = 0; i < num_frames; i++) { 6999 int res; 7000 7001 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 7002 if (res == STMMAC_XDP_CONSUMED) 7003 break; 7004 7005 nxmit++; 7006 } 7007 7008 if (flags & XDP_XMIT_FLUSH) { 7009 stmmac_flush_tx_descriptors(priv, queue); 7010 stmmac_tx_timer_arm(priv, queue); 7011 } 7012 7013 __netif_tx_unlock(nq); 7014 7015 return nxmit; 7016 } 7017 7018 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 7019 { 7020 struct stmmac_channel *ch = &priv->channel[queue]; 7021 unsigned long flags; 7022 7023 spin_lock_irqsave(&ch->lock, flags); 7024 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7025 spin_unlock_irqrestore(&ch->lock, flags); 7026 7027 stmmac_stop_rx_dma(priv, queue); 7028 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7029 } 7030 7031 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 7032 { 7033 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7034 struct stmmac_channel *ch = &priv->channel[queue]; 7035 unsigned long flags; 7036 int ret; 7037 7038 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7039 if (ret) { 7040 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 7041 return; 7042 } 7043 7044 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 7045 if (ret) { 7046 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 7047 netdev_err(priv->dev, "Failed to init RX desc.\n"); 7048 return; 7049 } 7050 7051 stmmac_reset_rx_queue(priv, queue); 7052 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 7053 7054 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7055 rx_q->dma_rx_phy, queue); 7056 7057 stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->buf_alloc_num); 7058 7059 stmmac_set_queue_rx_buf_size(priv, rx_q, queue); 7060 7061 stmmac_start_rx_dma(priv, queue); 7062 7063 spin_lock_irqsave(&ch->lock, flags); 7064 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 7065 spin_unlock_irqrestore(&ch->lock, flags); 7066 } 7067 7068 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 7069 { 7070 struct stmmac_channel *ch = &priv->channel[queue]; 7071 unsigned long flags; 7072 7073 spin_lock_irqsave(&ch->lock, flags); 7074 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7075 spin_unlock_irqrestore(&ch->lock, flags); 7076 7077 stmmac_stop_tx_dma(priv, queue); 7078 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7079 } 7080 7081 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 7082 { 7083 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7084 struct stmmac_channel *ch = &priv->channel[queue]; 7085 unsigned long flags; 7086 int ret; 7087 7088 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7089 if (ret) { 7090 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 7091 return; 7092 } 7093 7094 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 7095 if (ret) { 7096 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 7097 netdev_err(priv->dev, "Failed to init TX desc.\n"); 7098 return; 7099 } 7100 7101 stmmac_reset_tx_queue(priv, queue); 7102 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 7103 7104 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7105 tx_q->dma_tx_phy, queue); 7106 7107 if (tx_q->tbs & STMMAC_TBS_AVAIL) 7108 stmmac_enable_tbs(priv, priv->ioaddr, 1, queue); 7109 7110 stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, 0); 7111 7112 stmmac_start_tx_dma(priv, queue); 7113 7114 spin_lock_irqsave(&ch->lock, flags); 7115 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 7116 spin_unlock_irqrestore(&ch->lock, flags); 7117 } 7118 7119 void stmmac_xdp_release(struct net_device *dev) 7120 { 7121 struct stmmac_priv *priv = netdev_priv(dev); 7122 u8 chan; 7123 7124 /* Ensure tx function is not running */ 7125 netif_tx_disable(dev); 7126 7127 /* Disable NAPI process */ 7128 stmmac_disable_all_queues(priv); 7129 7130 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7131 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7132 7133 /* Free the IRQ lines */ 7134 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 7135 7136 /* Stop TX/RX DMA channels */ 7137 stmmac_stop_all_dma(priv); 7138 7139 /* Release and free the Rx/Tx resources */ 7140 free_dma_desc_resources(priv, &priv->dma_conf); 7141 7142 /* Disable the MAC Rx/Tx */ 7143 stmmac_mac_set(priv, priv->ioaddr, false); 7144 7145 /* set trans_start so we don't get spurious 7146 * watchdogs during reset 7147 */ 7148 netif_trans_update(dev); 7149 netif_carrier_off(dev); 7150 } 7151 7152 int stmmac_xdp_open(struct net_device *dev) 7153 { 7154 struct stmmac_priv *priv = netdev_priv(dev); 7155 u8 rx_cnt = priv->plat->rx_queues_to_use; 7156 u8 tx_cnt = priv->plat->tx_queues_to_use; 7157 u8 dma_csr_ch = max(rx_cnt, tx_cnt); 7158 struct stmmac_rx_queue *rx_q; 7159 struct stmmac_tx_queue *tx_q; 7160 bool sph_en; 7161 u8 chan; 7162 int ret; 7163 7164 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 7165 if (ret < 0) { 7166 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 7167 __func__); 7168 goto dma_desc_error; 7169 } 7170 7171 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 7172 if (ret < 0) { 7173 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 7174 __func__); 7175 goto init_error; 7176 } 7177 7178 stmmac_reset_queues_param(priv); 7179 7180 /* DMA CSR Channel configuration */ 7181 for (chan = 0; chan < dma_csr_ch; chan++) { 7182 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 7183 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 7184 } 7185 7186 /* Adjust Split header */ 7187 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 7188 7189 /* DMA RX Channel Configuration */ 7190 for (chan = 0; chan < rx_cnt; chan++) { 7191 rx_q = &priv->dma_conf.rx_queue[chan]; 7192 7193 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7194 rx_q->dma_rx_phy, chan); 7195 7196 stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, 7197 rx_q->buf_alloc_num); 7198 7199 stmmac_set_queue_rx_buf_size(priv, rx_q, chan); 7200 7201 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 7202 } 7203 7204 /* DMA TX Channel Configuration */ 7205 for (chan = 0; chan < tx_cnt; chan++) { 7206 tx_q = &priv->dma_conf.tx_queue[chan]; 7207 7208 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7209 tx_q->dma_tx_phy, chan); 7210 7211 stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); 7212 7213 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 7214 } 7215 7216 /* Enable the MAC Rx/Tx */ 7217 stmmac_mac_set(priv, priv->ioaddr, true); 7218 7219 /* Start Rx & Tx DMA Channels */ 7220 stmmac_start_all_dma(priv); 7221 7222 ret = stmmac_request_irq(dev); 7223 if (ret) 7224 goto irq_error; 7225 7226 /* Enable NAPI process*/ 7227 stmmac_enable_all_queues(priv); 7228 netif_carrier_on(dev); 7229 netif_tx_start_all_queues(dev); 7230 stmmac_enable_all_dma_irq(priv); 7231 7232 return 0; 7233 7234 irq_error: 7235 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7236 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7237 7238 init_error: 7239 free_dma_desc_resources(priv, &priv->dma_conf); 7240 dma_desc_error: 7241 return ret; 7242 } 7243 7244 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 7245 { 7246 struct stmmac_priv *priv = netdev_priv(dev); 7247 struct stmmac_rx_queue *rx_q; 7248 struct stmmac_tx_queue *tx_q; 7249 struct stmmac_channel *ch; 7250 7251 if (test_bit(STMMAC_DOWN, &priv->state) || 7252 !netif_carrier_ok(priv->dev)) 7253 return -ENETDOWN; 7254 7255 if (!stmmac_xdp_is_enabled(priv)) 7256 return -EINVAL; 7257 7258 if (queue >= priv->plat->rx_queues_to_use || 7259 queue >= priv->plat->tx_queues_to_use) 7260 return -EINVAL; 7261 7262 rx_q = &priv->dma_conf.rx_queue[queue]; 7263 tx_q = &priv->dma_conf.tx_queue[queue]; 7264 ch = &priv->channel[queue]; 7265 7266 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 7267 return -EINVAL; 7268 7269 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 7270 /* EQoS does not have per-DMA channel SW interrupt, 7271 * so we schedule RX Napi straight-away. 7272 */ 7273 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 7274 __napi_schedule(&ch->rxtx_napi); 7275 } 7276 7277 return 0; 7278 } 7279 7280 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7281 { 7282 struct stmmac_priv *priv = netdev_priv(dev); 7283 u8 tx_cnt = priv->plat->tx_queues_to_use; 7284 u8 rx_cnt = priv->plat->rx_queues_to_use; 7285 unsigned int start; 7286 u8 q; 7287 7288 for (q = 0; q < tx_cnt; q++) { 7289 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7290 u64 tx_packets; 7291 u64 tx_bytes; 7292 7293 do { 7294 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 7295 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 7296 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 7297 do { 7298 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 7299 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 7300 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 7301 7302 stats->tx_packets += tx_packets; 7303 stats->tx_bytes += tx_bytes; 7304 } 7305 7306 for (q = 0; q < rx_cnt; q++) { 7307 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7308 u64 rx_packets; 7309 u64 rx_bytes; 7310 7311 do { 7312 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 7313 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 7314 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 7315 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 7316 7317 stats->rx_packets += rx_packets; 7318 stats->rx_bytes += rx_bytes; 7319 } 7320 7321 stats->rx_dropped = priv->xstats.rx_dropped; 7322 stats->rx_errors = priv->xstats.rx_errors; 7323 stats->tx_dropped = priv->xstats.tx_dropped; 7324 stats->tx_errors = priv->xstats.tx_errors; 7325 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7326 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7327 stats->rx_length_errors = priv->xstats.rx_length; 7328 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7329 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7330 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7331 } 7332 7333 static const struct net_device_ops stmmac_netdev_ops = { 7334 .ndo_open = stmmac_open, 7335 .ndo_start_xmit = stmmac_xmit, 7336 .ndo_features_check = stmmac_features_check, 7337 .ndo_stop = stmmac_release, 7338 .ndo_change_mtu = stmmac_change_mtu, 7339 .ndo_fix_features = stmmac_fix_features, 7340 .ndo_set_features = stmmac_set_features, 7341 .ndo_set_rx_mode = stmmac_set_rx_mode, 7342 .ndo_tx_timeout = stmmac_tx_timeout, 7343 .ndo_eth_ioctl = stmmac_ioctl, 7344 .ndo_get_stats64 = stmmac_get_stats64, 7345 .ndo_setup_tc = stmmac_setup_tc, 7346 .ndo_select_queue = stmmac_select_queue, 7347 .ndo_set_mac_address = stmmac_set_mac_address, 7348 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7349 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7350 .ndo_bpf = stmmac_bpf, 7351 .ndo_xdp_xmit = stmmac_xdp_xmit, 7352 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7353 .ndo_hwtstamp_get = stmmac_hwtstamp_get, 7354 .ndo_hwtstamp_set = stmmac_hwtstamp_set, 7355 }; 7356 7357 static void stmmac_reset_subtask(struct stmmac_priv *priv) 7358 { 7359 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7360 return; 7361 if (test_bit(STMMAC_DOWN, &priv->state)) 7362 return; 7363 7364 netdev_err(priv->dev, "Reset adapter.\n"); 7365 7366 rtnl_lock(); 7367 netif_trans_update(priv->dev); 7368 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7369 usleep_range(1000, 2000); 7370 7371 set_bit(STMMAC_DOWN, &priv->state); 7372 dev_close(priv->dev); 7373 dev_open(priv->dev, NULL); 7374 clear_bit(STMMAC_DOWN, &priv->state); 7375 clear_bit(STMMAC_RESETING, &priv->state); 7376 rtnl_unlock(); 7377 } 7378 7379 static void stmmac_service_task(struct work_struct *work) 7380 { 7381 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7382 service_task); 7383 7384 stmmac_reset_subtask(priv); 7385 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7386 } 7387 7388 static void stmmac_print_actphyif(struct stmmac_priv *priv) 7389 { 7390 const char **phyif_table; 7391 const char *actphyif_str; 7392 size_t phyif_table_size; 7393 7394 switch (priv->plat->core_type) { 7395 case DWMAC_CORE_MAC100: 7396 return; 7397 7398 case DWMAC_CORE_GMAC: 7399 case DWMAC_CORE_GMAC4: 7400 phyif_table = stmmac_dwmac_actphyif; 7401 phyif_table_size = ARRAY_SIZE(stmmac_dwmac_actphyif); 7402 break; 7403 7404 case DWMAC_CORE_XGMAC: 7405 phyif_table = stmmac_dwxgmac_phyif; 7406 phyif_table_size = ARRAY_SIZE(stmmac_dwxgmac_phyif); 7407 break; 7408 } 7409 7410 if (priv->dma_cap.actphyif < phyif_table_size) 7411 actphyif_str = phyif_table[priv->dma_cap.actphyif]; 7412 else 7413 actphyif_str = NULL; 7414 7415 if (!actphyif_str) 7416 actphyif_str = "unknown"; 7417 7418 dev_info(priv->device, "Active PHY interface: %s (%u)\n", 7419 actphyif_str, priv->dma_cap.actphyif); 7420 } 7421 7422 /** 7423 * stmmac_hw_init - Init the MAC device 7424 * @priv: driver private structure 7425 * Description: this function is to configure the MAC device according to 7426 * some platform parameters or the HW capability register. It prepares the 7427 * driver to use either ring or chain modes and to setup either enhanced or 7428 * normal descriptors. 7429 */ 7430 static int stmmac_hw_init(struct stmmac_priv *priv) 7431 { 7432 int ret; 7433 7434 /* dwmac-sun8i only work in chain mode */ 7435 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7436 chain_mode = 1; 7437 priv->chain_mode = !!chain_mode; 7438 7439 /* Initialize HW Interface */ 7440 ret = stmmac_hwif_init(priv); 7441 if (ret) 7442 return ret; 7443 7444 /* Get the HW capability (new GMAC newer than 3.50a) */ 7445 priv->hw_cap_support = stmmac_get_hw_features(priv); 7446 if (priv->hw_cap_support) { 7447 dev_info(priv->device, "DMA HW capability register supported\n"); 7448 7449 /* We can override some gmac/dma configuration fields: e.g. 7450 * enh_desc, tx_coe (e.g. that are passed through the 7451 * platform) with the values from the HW capability 7452 * register (if supported). 7453 */ 7454 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7455 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7456 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7457 if (priv->dma_cap.hash_tb_sz) { 7458 priv->hw->multicast_filter_bins = 7459 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7460 priv->hw->mcast_bits_log2 = 7461 ilog2(priv->hw->multicast_filter_bins); 7462 } 7463 7464 /* TXCOE doesn't work in thresh DMA mode */ 7465 if (priv->plat->force_thresh_dma_mode) 7466 priv->plat->tx_coe = false; 7467 else 7468 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7469 7470 /* In case of GMAC4 rx_coe is from HW cap register. */ 7471 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7472 7473 if (priv->dma_cap.rx_coe_type2) 7474 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7475 else if (priv->dma_cap.rx_coe_type1) 7476 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7477 7478 stmmac_print_actphyif(priv); 7479 } else { 7480 dev_info(priv->device, "No HW DMA feature register supported\n"); 7481 } 7482 7483 if (priv->plat->rx_coe) { 7484 priv->hw->rx_csum = priv->plat->rx_coe; 7485 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7486 if (priv->synopsys_id < DWMAC_CORE_4_00) 7487 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7488 } 7489 if (priv->plat->tx_coe) 7490 dev_info(priv->device, "TX Checksum insertion supported\n"); 7491 7492 if (priv->plat->pmt) { 7493 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7494 device_set_wakeup_capable(priv->device, 1); 7495 devm_pm_set_wake_irq(priv->device, priv->wol_irq); 7496 } 7497 7498 if (priv->dma_cap.number_rx_queues && 7499 priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) { 7500 dev_warn(priv->device, 7501 "Number of Rx queues (%u) exceeds dma capability\n", 7502 priv->plat->rx_queues_to_use); 7503 priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues; 7504 } 7505 if (priv->dma_cap.number_tx_queues && 7506 priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) { 7507 dev_warn(priv->device, 7508 "Number of Tx queues (%u) exceeds dma capability\n", 7509 priv->plat->tx_queues_to_use); 7510 priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues; 7511 } 7512 7513 if (priv->dma_cap.rx_fifo_size && 7514 priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) { 7515 dev_warn(priv->device, 7516 "Rx FIFO size (%u) exceeds dma capability\n", 7517 priv->plat->rx_fifo_size); 7518 priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size; 7519 } 7520 if (priv->dma_cap.tx_fifo_size && 7521 priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) { 7522 dev_warn(priv->device, 7523 "Tx FIFO size (%u) exceeds dma capability\n", 7524 priv->plat->tx_fifo_size); 7525 priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size; 7526 } 7527 7528 priv->hw->vlan_fail_q_en = 7529 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7530 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7531 7532 /* Run HW quirks, if any */ 7533 if (priv->hwif_quirks) { 7534 ret = priv->hwif_quirks(priv); 7535 if (ret) 7536 return ret; 7537 } 7538 7539 /* Set alternate descriptor size (which tells the hardware that 7540 * descriptors are 8 32-bit words) when using extended descriptors 7541 * with ring mode. Only applicable for pre-v4.0 cores. Platform glue 7542 * is not expected to change this. 7543 */ 7544 priv->plat->dma_cfg->atds = priv->extend_desc && 7545 priv->descriptor_mode == STMMAC_RING_MODE; 7546 7547 /* Rx Watchdog is available in the COREs newer than the 3.40. 7548 * In some case, for example on bugged HW this feature 7549 * has to be disable and this can be done by passing the 7550 * riwt_off field from the platform. 7551 */ 7552 if ((priv->synopsys_id >= DWMAC_CORE_3_50 || 7553 priv->plat->core_type == DWMAC_CORE_XGMAC) && 7554 !priv->plat->riwt_off) { 7555 priv->use_riwt = 1; 7556 dev_info(priv->device, 7557 "Enable RX Mitigation via HW Watchdog Timer\n"); 7558 } 7559 7560 /* Unimplemented PCS init (as indicated by stmmac_do_callback() 7561 * perversely returning -EINVAL) is non-fatal. 7562 */ 7563 ret = stmmac_mac_pcs_init(priv); 7564 if (ret != -EINVAL) 7565 return ret; 7566 7567 return 0; 7568 } 7569 7570 static void stmmac_napi_add(struct net_device *dev) 7571 { 7572 struct stmmac_priv *priv = netdev_priv(dev); 7573 u8 queue, maxq; 7574 7575 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7576 7577 for (queue = 0; queue < maxq; queue++) { 7578 struct stmmac_channel *ch = &priv->channel[queue]; 7579 7580 ch->priv_data = priv; 7581 ch->index = queue; 7582 spin_lock_init(&ch->lock); 7583 7584 if (queue < priv->plat->rx_queues_to_use) { 7585 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7586 } 7587 if (queue < priv->plat->tx_queues_to_use) { 7588 netif_napi_add_tx(dev, &ch->tx_napi, 7589 stmmac_napi_poll_tx); 7590 } 7591 if (queue < priv->plat->rx_queues_to_use && 7592 queue < priv->plat->tx_queues_to_use) { 7593 netif_napi_add(dev, &ch->rxtx_napi, 7594 stmmac_napi_poll_rxtx); 7595 } 7596 } 7597 } 7598 7599 static void stmmac_napi_del(struct net_device *dev) 7600 { 7601 struct stmmac_priv *priv = netdev_priv(dev); 7602 u8 queue, maxq; 7603 7604 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7605 7606 for (queue = 0; queue < maxq; queue++) { 7607 struct stmmac_channel *ch = &priv->channel[queue]; 7608 7609 if (queue < priv->plat->rx_queues_to_use) 7610 netif_napi_del(&ch->rx_napi); 7611 if (queue < priv->plat->tx_queues_to_use) 7612 netif_napi_del(&ch->tx_napi); 7613 if (queue < priv->plat->rx_queues_to_use && 7614 queue < priv->plat->tx_queues_to_use) { 7615 netif_napi_del(&ch->rxtx_napi); 7616 } 7617 } 7618 } 7619 7620 int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt) 7621 { 7622 struct stmmac_priv *priv = netdev_priv(dev); 7623 int ret = 0, i; 7624 7625 if (netif_running(dev)) 7626 stmmac_release(dev); 7627 7628 stmmac_napi_del(dev); 7629 7630 priv->plat->rx_queues_to_use = rx_cnt; 7631 priv->plat->tx_queues_to_use = tx_cnt; 7632 if (!netif_is_rxfh_configured(dev)) 7633 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7634 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7635 rx_cnt); 7636 7637 stmmac_napi_add(dev); 7638 7639 if (netif_running(dev)) 7640 ret = stmmac_open(dev); 7641 7642 return ret; 7643 } 7644 7645 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7646 { 7647 struct stmmac_priv *priv = netdev_priv(dev); 7648 int ret = 0; 7649 7650 if (netif_running(dev)) 7651 stmmac_release(dev); 7652 7653 priv->dma_conf.dma_rx_size = rx_size; 7654 priv->dma_conf.dma_tx_size = tx_size; 7655 7656 if (netif_running(dev)) 7657 ret = stmmac_open(dev); 7658 7659 return ret; 7660 } 7661 7662 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7663 { 7664 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7665 struct dma_desc *desc_contains_ts = ctx->desc; 7666 struct stmmac_priv *priv = ctx->priv; 7667 struct dma_desc *ndesc = ctx->ndesc; 7668 struct dma_desc *desc = ctx->desc; 7669 u64 ns = 0; 7670 7671 if (!priv->hwts_rx_en) 7672 return -ENODATA; 7673 7674 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7675 if (dwmac_is_xmac(priv->plat->core_type)) 7676 desc_contains_ts = ndesc; 7677 7678 /* Check if timestamp is available */ 7679 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7680 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7681 ns -= priv->plat->cdc_error_adj; 7682 *timestamp = ns_to_ktime(ns); 7683 return 0; 7684 } 7685 7686 return -ENODATA; 7687 } 7688 7689 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7690 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7691 }; 7692 7693 static int stmmac_dl_ts_coarse_set(struct devlink *dl, u32 id, 7694 struct devlink_param_gset_ctx *ctx, 7695 struct netlink_ext_ack *extack) 7696 { 7697 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7698 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7699 7700 priv->tsfupdt_coarse = ctx->val.vbool; 7701 7702 if (priv->tsfupdt_coarse) 7703 priv->systime_flags &= ~PTP_TCR_TSCFUPDT; 7704 else 7705 priv->systime_flags |= PTP_TCR_TSCFUPDT; 7706 7707 /* In Coarse mode, we can use a smaller subsecond increment, let's 7708 * reconfigure the systime, subsecond increment and addend. 7709 */ 7710 stmmac_update_subsecond_increment(priv); 7711 7712 return 0; 7713 } 7714 7715 static int stmmac_dl_ts_coarse_get(struct devlink *dl, u32 id, 7716 struct devlink_param_gset_ctx *ctx, 7717 struct netlink_ext_ack *extack) 7718 { 7719 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7720 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7721 7722 ctx->val.vbool = priv->tsfupdt_coarse; 7723 7724 return 0; 7725 } 7726 7727 static const struct devlink_param stmmac_devlink_params[] = { 7728 DEVLINK_PARAM_DRIVER(STMMAC_DEVLINK_PARAM_ID_TS_COARSE, "phc_coarse_adj", 7729 DEVLINK_PARAM_TYPE_BOOL, 7730 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 7731 stmmac_dl_ts_coarse_get, 7732 stmmac_dl_ts_coarse_set, NULL), 7733 }; 7734 7735 /* None of the generic devlink parameters are implemented */ 7736 static const struct devlink_ops stmmac_devlink_ops = {}; 7737 7738 static int stmmac_register_devlink(struct stmmac_priv *priv) 7739 { 7740 struct stmmac_devlink_priv *dl_priv; 7741 int ret; 7742 7743 /* For now, what is exposed over devlink is only relevant when 7744 * timestamping is available and we have a valid ptp clock rate 7745 */ 7746 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp) || 7747 !priv->plat->clk_ptp_rate) 7748 return 0; 7749 7750 priv->devlink = devlink_alloc(&stmmac_devlink_ops, sizeof(*dl_priv), 7751 priv->device); 7752 if (!priv->devlink) 7753 return -ENOMEM; 7754 7755 dl_priv = devlink_priv(priv->devlink); 7756 dl_priv->stmmac_priv = priv; 7757 7758 ret = devlink_params_register(priv->devlink, stmmac_devlink_params, 7759 ARRAY_SIZE(stmmac_devlink_params)); 7760 if (ret) 7761 goto dl_free; 7762 7763 devlink_register(priv->devlink); 7764 return 0; 7765 7766 dl_free: 7767 devlink_free(priv->devlink); 7768 7769 return ret; 7770 } 7771 7772 static void stmmac_unregister_devlink(struct stmmac_priv *priv) 7773 { 7774 if (!priv->devlink) 7775 return; 7776 7777 devlink_unregister(priv->devlink); 7778 devlink_params_unregister(priv->devlink, stmmac_devlink_params, 7779 ARRAY_SIZE(stmmac_devlink_params)); 7780 devlink_free(priv->devlink); 7781 } 7782 7783 struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev) 7784 { 7785 struct plat_stmmacenet_data *plat_dat; 7786 int i; 7787 7788 plat_dat = devm_kzalloc(dev, sizeof(*plat_dat), GFP_KERNEL); 7789 if (!plat_dat) 7790 return NULL; 7791 7792 plat_dat->dma_cfg = &plat_dat->__dma_cfg; 7793 7794 /* Set the defaults: 7795 * - phy autodetection 7796 * - determine GMII_Address CR field from CSR clock 7797 * - allow MTU up to JUMBO_LEN 7798 * - hash table size 7799 * - one unicast filter entry 7800 */ 7801 plat_dat->phy_addr = -1; 7802 plat_dat->clk_csr = -1; 7803 plat_dat->maxmtu = JUMBO_LEN; 7804 plat_dat->multicast_filter_bins = HASH_TABLE_SIZE; 7805 plat_dat->unicast_filter_entries = 1; 7806 7807 /* Set the mtl defaults */ 7808 plat_dat->tx_queues_to_use = 1; 7809 plat_dat->rx_queues_to_use = 1; 7810 7811 /* Setup the default RX queue channel map */ 7812 for (i = 0; i < ARRAY_SIZE(plat_dat->rx_queues_cfg); i++) 7813 plat_dat->rx_queues_cfg[i].chan = i; 7814 7815 return plat_dat; 7816 } 7817 EXPORT_SYMBOL_GPL(stmmac_plat_dat_alloc); 7818 7819 static int __stmmac_dvr_probe(struct device *device, 7820 struct plat_stmmacenet_data *plat_dat, 7821 struct stmmac_resources *res) 7822 { 7823 struct net_device *ndev = NULL; 7824 struct stmmac_priv *priv; 7825 int i, ret = 0; 7826 u8 rxq; 7827 7828 if (!plat_dat->dma_cfg || !plat_dat->dma_cfg->pbl) { 7829 dev_err(device, "invalid DMA configuration\n"); 7830 return -EINVAL; 7831 } 7832 7833 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7834 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7835 if (!ndev) 7836 return -ENOMEM; 7837 7838 SET_NETDEV_DEV(ndev, device); 7839 7840 priv = netdev_priv(ndev); 7841 priv->device = device; 7842 priv->dev = ndev; 7843 7844 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7845 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7846 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7847 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7848 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7849 } 7850 7851 priv->xstats.pcpu_stats = 7852 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7853 if (!priv->xstats.pcpu_stats) 7854 return -ENOMEM; 7855 7856 stmmac_set_ethtool_ops(ndev); 7857 priv->pause_time = pause; 7858 priv->plat = plat_dat; 7859 priv->ioaddr = res->addr; 7860 priv->dev->base_addr = (unsigned long)res->addr; 7861 priv->plat->dma_cfg->multi_msi_en = 7862 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7863 7864 priv->dev->irq = res->irq; 7865 priv->wol_irq = res->wol_irq; 7866 priv->sfty_irq = res->sfty_irq; 7867 7868 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) { 7869 ret = stmmac_msi_init(priv, res); 7870 if (ret) 7871 return ret; 7872 } 7873 7874 if (!is_zero_ether_addr(res->mac)) 7875 eth_hw_addr_set(priv->dev, res->mac); 7876 7877 dev_set_drvdata(device, priv->dev); 7878 7879 /* Verify driver arguments */ 7880 stmmac_verify_args(); 7881 7882 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7883 if (!priv->af_xdp_zc_qps) 7884 return -ENOMEM; 7885 7886 /* Allocate workqueue */ 7887 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7888 if (!priv->wq) { 7889 dev_err(priv->device, "failed to create workqueue\n"); 7890 ret = -ENOMEM; 7891 goto error_wq_init; 7892 } 7893 7894 INIT_WORK(&priv->service_task, stmmac_service_task); 7895 7896 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 7897 7898 /* Override with kernel parameters if supplied XXX CRS XXX 7899 * this needs to have multiple instances 7900 */ 7901 if ((phyaddr >= 0) && (phyaddr <= 31)) 7902 priv->plat->phy_addr = phyaddr; 7903 7904 if (priv->plat->stmmac_rst) { 7905 ret = reset_control_assert(priv->plat->stmmac_rst); 7906 reset_control_deassert(priv->plat->stmmac_rst); 7907 /* Some reset controllers have only reset callback instead of 7908 * assert + deassert callbacks pair. 7909 */ 7910 if (ret == -ENOTSUPP) 7911 reset_control_reset(priv->plat->stmmac_rst); 7912 } 7913 7914 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7915 if (ret == -ENOTSUPP) 7916 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7917 ERR_PTR(ret)); 7918 7919 /* Wait a bit for the reset to take effect */ 7920 udelay(10); 7921 7922 /* Init MAC and get the capabilities */ 7923 ret = stmmac_hw_init(priv); 7924 if (ret) 7925 goto error_hw_init; 7926 7927 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7928 */ 7929 if (priv->synopsys_id < DWMAC_CORE_5_20) 7930 priv->plat->dma_cfg->dche = false; 7931 7932 stmmac_check_ether_addr(priv); 7933 7934 ndev->netdev_ops = &stmmac_netdev_ops; 7935 7936 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7937 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7938 7939 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7940 NETIF_F_RXCSUM; 7941 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7942 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7943 7944 ret = stmmac_tc_init(priv, priv); 7945 if (!ret) { 7946 ndev->hw_features |= NETIF_F_HW_TC; 7947 } 7948 7949 stmmac_set_gso_features(ndev); 7950 7951 if (priv->dma_cap.sphen && 7952 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7953 ndev->hw_features |= NETIF_F_GRO; 7954 priv->sph_capable = true; 7955 priv->sph_active = priv->sph_capable; 7956 dev_info(priv->device, "SPH feature enabled\n"); 7957 } 7958 7959 /* Ideally our host DMA address width is the same as for the 7960 * device. However, it may differ and then we have to use our 7961 * host DMA width for allocation and the device DMA width for 7962 * register handling. 7963 */ 7964 if (priv->plat->host_dma_width) 7965 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7966 else 7967 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7968 7969 if (priv->dma_cap.host_dma_width) { 7970 ret = dma_set_mask_and_coherent(device, 7971 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7972 if (!ret) { 7973 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7974 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7975 7976 /* 7977 * If more than 32 bits can be addressed, make sure to 7978 * enable enhanced addressing mode. 7979 */ 7980 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7981 priv->plat->dma_cfg->eame = true; 7982 } else { 7983 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7984 if (ret) { 7985 dev_err(priv->device, "Failed to set DMA Mask\n"); 7986 goto error_hw_init; 7987 } 7988 7989 priv->dma_cap.host_dma_width = 32; 7990 } 7991 } 7992 7993 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7994 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7995 #ifdef STMMAC_VLAN_TAG_USED 7996 /* Both mac100 and gmac support receive VLAN tag detection */ 7997 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7998 if (dwmac_is_xmac(priv->plat->core_type)) { 7999 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 8000 priv->hw->hw_vlan_en = true; 8001 } 8002 if (priv->dma_cap.vlhash) { 8003 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 8004 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 8005 } 8006 if (priv->dma_cap.vlins) 8007 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 8008 #endif 8009 priv->msg_enable = netif_msg_init(debug, default_msg_level); 8010 8011 priv->xstats.threshold = tc; 8012 8013 /* Initialize RSS */ 8014 rxq = priv->plat->rx_queues_to_use; 8015 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 8016 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 8017 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 8018 8019 if (priv->dma_cap.rssen && priv->plat->rss_en) 8020 ndev->features |= NETIF_F_RXHASH; 8021 8022 ndev->vlan_features |= ndev->features; 8023 8024 /* MTU range: 46 - hw-specific max */ 8025 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 8026 8027 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 8028 ndev->max_mtu = XGMAC_JUMBO_LEN; 8029 else if (priv->plat->enh_desc || priv->synopsys_id >= DWMAC_CORE_4_00) 8030 ndev->max_mtu = JUMBO_LEN; 8031 else 8032 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 8033 8034 /* Warn if the platform's maxmtu is smaller than the minimum MTU, 8035 * otherwise clamp the maximum MTU above to the platform's maxmtu. 8036 */ 8037 if (priv->plat->maxmtu < ndev->min_mtu) 8038 dev_warn(priv->device, 8039 "%s: warning: maxmtu having invalid value (%d)\n", 8040 __func__, priv->plat->maxmtu); 8041 else if (priv->plat->maxmtu < ndev->max_mtu) 8042 ndev->max_mtu = priv->plat->maxmtu; 8043 8044 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 8045 8046 /* Setup channels NAPI */ 8047 stmmac_napi_add(ndev); 8048 8049 mutex_init(&priv->lock); 8050 rwlock_init(&priv->ptp_lock); 8051 8052 stmmac_fpe_init(priv); 8053 8054 stmmac_check_pcs_mode(priv); 8055 8056 pm_runtime_get_noresume(device); 8057 pm_runtime_set_active(device); 8058 if (!pm_runtime_enabled(device)) 8059 pm_runtime_enable(device); 8060 8061 ret = stmmac_mdio_register(ndev); 8062 if (ret < 0) { 8063 dev_err_probe(priv->device, ret, 8064 "MDIO bus (id: %d) registration failed\n", 8065 priv->plat->bus_id); 8066 goto error_mdio_register; 8067 } 8068 8069 ret = stmmac_pcs_setup(ndev); 8070 if (ret) 8071 goto error_pcs_setup; 8072 8073 ret = stmmac_phylink_setup(priv); 8074 if (ret) { 8075 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 8076 goto error_phy_setup; 8077 } 8078 8079 ret = stmmac_register_devlink(priv); 8080 if (ret) 8081 goto error_devlink_setup; 8082 8083 ret = register_netdev(ndev); 8084 if (ret) { 8085 dev_err(priv->device, "%s: ERROR %i registering the device\n", 8086 __func__, ret); 8087 goto error_netdev_register; 8088 } 8089 8090 #ifdef CONFIG_DEBUG_FS 8091 stmmac_init_fs(ndev); 8092 #endif 8093 8094 if (priv->plat->dump_debug_regs) 8095 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 8096 8097 /* Let pm_runtime_put() disable the clocks. 8098 * If CONFIG_PM is not enabled, the clocks will stay powered. 8099 */ 8100 pm_runtime_put(device); 8101 8102 return ret; 8103 8104 error_netdev_register: 8105 stmmac_unregister_devlink(priv); 8106 error_devlink_setup: 8107 phylink_destroy(priv->phylink); 8108 error_phy_setup: 8109 stmmac_pcs_clean(ndev); 8110 error_pcs_setup: 8111 stmmac_mdio_unregister(ndev); 8112 error_mdio_register: 8113 stmmac_napi_del(ndev); 8114 error_hw_init: 8115 destroy_workqueue(priv->wq); 8116 error_wq_init: 8117 bitmap_free(priv->af_xdp_zc_qps); 8118 8119 return ret; 8120 } 8121 8122 /** 8123 * stmmac_dvr_probe 8124 * @dev: device pointer 8125 * @plat_dat: platform data pointer 8126 * @res: stmmac resource pointer 8127 * Description: this is the main probe function used to 8128 * call the alloc_etherdev, allocate the priv structure. 8129 * Return: 8130 * returns 0 on success, otherwise errno. 8131 */ 8132 int stmmac_dvr_probe(struct device *dev, struct plat_stmmacenet_data *plat_dat, 8133 struct stmmac_resources *res) 8134 { 8135 int ret; 8136 8137 if (plat_dat->init) { 8138 ret = plat_dat->init(dev, plat_dat->bsp_priv); 8139 if (ret) 8140 return ret; 8141 } 8142 8143 ret = __stmmac_dvr_probe(dev, plat_dat, res); 8144 if (ret && plat_dat->exit) 8145 plat_dat->exit(dev, plat_dat->bsp_priv); 8146 8147 return ret; 8148 } 8149 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 8150 8151 /** 8152 * stmmac_dvr_remove 8153 * @dev: device pointer 8154 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 8155 * changes the link status, releases the DMA descriptor rings. 8156 */ 8157 void stmmac_dvr_remove(struct device *dev) 8158 { 8159 struct net_device *ndev = dev_get_drvdata(dev); 8160 struct stmmac_priv *priv = netdev_priv(ndev); 8161 8162 netdev_info(priv->dev, "%s: removing driver", __func__); 8163 8164 pm_runtime_get_sync(dev); 8165 8166 unregister_netdev(ndev); 8167 8168 #ifdef CONFIG_DEBUG_FS 8169 stmmac_exit_fs(ndev); 8170 #endif 8171 stmmac_unregister_devlink(priv); 8172 8173 phylink_destroy(priv->phylink); 8174 if (priv->plat->stmmac_rst) 8175 reset_control_assert(priv->plat->stmmac_rst); 8176 reset_control_assert(priv->plat->stmmac_ahb_rst); 8177 8178 stmmac_pcs_clean(ndev); 8179 stmmac_mdio_unregister(ndev); 8180 8181 destroy_workqueue(priv->wq); 8182 mutex_destroy(&priv->lock); 8183 bitmap_free(priv->af_xdp_zc_qps); 8184 8185 pm_runtime_disable(dev); 8186 pm_runtime_put_noidle(dev); 8187 8188 if (priv->plat->exit) 8189 priv->plat->exit(dev, priv->plat->bsp_priv); 8190 } 8191 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 8192 8193 /** 8194 * stmmac_suspend - suspend callback 8195 * @dev: device pointer 8196 * Description: this is the function to suspend the device and it is called 8197 * by the platform driver to stop the network queue, release the resources, 8198 * program the PMT register (for WoL), clean and release driver resources. 8199 */ 8200 int stmmac_suspend(struct device *dev) 8201 { 8202 struct net_device *ndev = dev_get_drvdata(dev); 8203 struct stmmac_priv *priv = netdev_priv(ndev); 8204 u8 chan; 8205 8206 if (!ndev || !netif_running(ndev)) 8207 goto suspend_bsp; 8208 8209 mutex_lock(&priv->lock); 8210 8211 netif_device_detach(ndev); 8212 8213 stmmac_disable_all_queues(priv); 8214 8215 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 8216 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 8217 8218 if (priv->eee_sw_timer_en) { 8219 priv->tx_path_in_lpi_mode = false; 8220 timer_delete_sync(&priv->eee_ctrl_timer); 8221 } 8222 8223 /* Stop TX/RX DMA */ 8224 stmmac_stop_all_dma(priv); 8225 8226 stmmac_legacy_serdes_power_down(priv); 8227 8228 /* Enable Power down mode by programming the PMT regs */ 8229 if (priv->wolopts) { 8230 stmmac_pmt(priv, priv->hw, priv->wolopts); 8231 priv->irq_wake = 1; 8232 } else { 8233 stmmac_mac_set(priv, priv->ioaddr, false); 8234 pinctrl_pm_select_sleep_state(priv->device); 8235 } 8236 8237 mutex_unlock(&priv->lock); 8238 8239 rtnl_lock(); 8240 phylink_suspend(priv->phylink, !!priv->wolopts); 8241 rtnl_unlock(); 8242 8243 if (stmmac_fpe_supported(priv)) 8244 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 8245 8246 suspend_bsp: 8247 if (priv->plat->suspend) 8248 return priv->plat->suspend(dev, priv->plat->bsp_priv); 8249 8250 return 0; 8251 } 8252 EXPORT_SYMBOL_GPL(stmmac_suspend); 8253 8254 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 8255 { 8256 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 8257 8258 rx_q->cur_rx = 0; 8259 rx_q->dirty_rx = 0; 8260 } 8261 8262 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 8263 { 8264 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 8265 8266 tx_q->cur_tx = 0; 8267 tx_q->dirty_tx = 0; 8268 tx_q->mss = 0; 8269 8270 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 8271 } 8272 8273 /** 8274 * stmmac_reset_queues_param - reset queue parameters 8275 * @priv: device pointer 8276 */ 8277 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 8278 { 8279 u8 rx_cnt = priv->plat->rx_queues_to_use; 8280 u8 tx_cnt = priv->plat->tx_queues_to_use; 8281 u8 queue; 8282 8283 for (queue = 0; queue < rx_cnt; queue++) 8284 stmmac_reset_rx_queue(priv, queue); 8285 8286 for (queue = 0; queue < tx_cnt; queue++) 8287 stmmac_reset_tx_queue(priv, queue); 8288 } 8289 8290 /** 8291 * stmmac_resume - resume callback 8292 * @dev: device pointer 8293 * Description: when resume this function is invoked to setup the DMA and CORE 8294 * in a usable state. 8295 */ 8296 int stmmac_resume(struct device *dev) 8297 { 8298 struct net_device *ndev = dev_get_drvdata(dev); 8299 struct stmmac_priv *priv = netdev_priv(ndev); 8300 int ret; 8301 8302 if (priv->plat->resume) { 8303 ret = priv->plat->resume(dev, priv->plat->bsp_priv); 8304 if (ret) 8305 return ret; 8306 } 8307 8308 if (!netif_running(ndev)) 8309 return 0; 8310 8311 /* Power Down bit, into the PM register, is cleared 8312 * automatically as soon as a magic packet or a Wake-up frame 8313 * is received. Anyway, it's better to manually clear 8314 * this bit because it can generate problems while resuming 8315 * from another devices (e.g. serial console). 8316 */ 8317 if (priv->wolopts) { 8318 mutex_lock(&priv->lock); 8319 stmmac_pmt(priv, priv->hw, 0); 8320 mutex_unlock(&priv->lock); 8321 priv->irq_wake = 0; 8322 } else { 8323 pinctrl_pm_select_default_state(priv->device); 8324 /* reset the phy so that it's ready */ 8325 if (priv->mii) 8326 stmmac_mdio_reset(priv->mii); 8327 } 8328 8329 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP)) { 8330 ret = stmmac_legacy_serdes_power_up(priv); 8331 if (ret < 0) 8332 return ret; 8333 } 8334 8335 rtnl_lock(); 8336 8337 /* Prepare the PHY to resume, ensuring that its clocks which are 8338 * necessary for the MAC DMA reset to complete are running 8339 */ 8340 phylink_prepare_resume(priv->phylink); 8341 8342 mutex_lock(&priv->lock); 8343 8344 stmmac_reset_queues_param(priv); 8345 8346 stmmac_free_tx_skbufs(priv); 8347 stmmac_clear_descriptors(priv, &priv->dma_conf); 8348 8349 ret = stmmac_hw_setup(ndev); 8350 if (ret < 0) { 8351 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 8352 stmmac_legacy_serdes_power_down(priv); 8353 mutex_unlock(&priv->lock); 8354 rtnl_unlock(); 8355 return ret; 8356 } 8357 8358 stmmac_init_timestamping(priv); 8359 8360 stmmac_init_coalesce(priv); 8361 phylink_rx_clk_stop_block(priv->phylink); 8362 stmmac_set_rx_mode(ndev); 8363 phylink_rx_clk_stop_unblock(priv->phylink); 8364 8365 stmmac_vlan_restore(priv); 8366 8367 stmmac_enable_all_queues(priv); 8368 stmmac_enable_all_dma_irq(priv); 8369 8370 mutex_unlock(&priv->lock); 8371 8372 /* phylink_resume() must be called after the hardware has been 8373 * initialised because it may bring the link up immediately in a 8374 * workqueue thread, which will race with initialisation. 8375 */ 8376 phylink_resume(priv->phylink); 8377 rtnl_unlock(); 8378 8379 netif_device_attach(ndev); 8380 8381 return 0; 8382 } 8383 EXPORT_SYMBOL_GPL(stmmac_resume); 8384 8385 /* This is not the same as EXPORT_GPL_SIMPLE_DEV_PM_OPS() when CONFIG_PM=n */ 8386 DEFINE_SIMPLE_DEV_PM_OPS(stmmac_simple_pm_ops, stmmac_suspend, stmmac_resume); 8387 EXPORT_SYMBOL_GPL(stmmac_simple_pm_ops); 8388 8389 #ifndef MODULE 8390 static int __init stmmac_cmdline_opt(char *str) 8391 { 8392 char *opt; 8393 8394 if (!str || !*str) 8395 return 1; 8396 while ((opt = strsep(&str, ",")) != NULL) { 8397 if (!strncmp(opt, "debug:", 6)) { 8398 if (kstrtoint(opt + 6, 0, &debug)) 8399 goto err; 8400 } else if (!strncmp(opt, "phyaddr:", 8)) { 8401 if (kstrtoint(opt + 8, 0, &phyaddr)) 8402 goto err; 8403 } else if (!strncmp(opt, "tc:", 3)) { 8404 if (kstrtoint(opt + 3, 0, &tc)) 8405 goto err; 8406 } else if (!strncmp(opt, "watchdog:", 9)) { 8407 if (kstrtoint(opt + 9, 0, &watchdog)) 8408 goto err; 8409 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 8410 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 8411 goto err; 8412 } else if (!strncmp(opt, "pause:", 6)) { 8413 if (kstrtoint(opt + 6, 0, &pause)) 8414 goto err; 8415 } else if (!strncmp(opt, "eee_timer:", 10)) { 8416 if (kstrtoint(opt + 10, 0, &eee_timer)) 8417 goto err; 8418 } else if (!strncmp(opt, "chain_mode:", 11)) { 8419 if (kstrtoint(opt + 11, 0, &chain_mode)) 8420 goto err; 8421 } 8422 } 8423 return 1; 8424 8425 err: 8426 pr_err("%s: ERROR broken module parameter conversion", __func__); 8427 return 1; 8428 } 8429 8430 __setup("stmmaceth=", stmmac_cmdline_opt); 8431 #endif /* MODULE */ 8432 8433 static int __init stmmac_init(void) 8434 { 8435 #ifdef CONFIG_DEBUG_FS 8436 /* Create debugfs main directory if it doesn't exist yet */ 8437 if (!stmmac_fs_dir) 8438 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 8439 register_netdevice_notifier(&stmmac_notifier); 8440 #endif 8441 8442 return 0; 8443 } 8444 8445 static void __exit stmmac_exit(void) 8446 { 8447 #ifdef CONFIG_DEBUG_FS 8448 unregister_netdevice_notifier(&stmmac_notifier); 8449 debugfs_remove_recursive(stmmac_fs_dir); 8450 #endif 8451 } 8452 8453 module_init(stmmac_init) 8454 module_exit(stmmac_exit) 8455 8456 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 8457 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 8458 MODULE_LICENSE("GPL"); 8459