135 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue);
136 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue);
137 static void stmmac_reset_queues_param(struct stmmac_priv *priv);
138 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
139 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
140 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
151 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled)  in stmmac_bus_clks_config()  argument
156 		ret = clk_prepare_enable(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
159 		ret = clk_prepare_enable(priv->plat->pclk);  in stmmac_bus_clks_config()
161 			clk_disable_unprepare(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
164 		if (priv->plat->clks_config) {  in stmmac_bus_clks_config()
165 			ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled);  in stmmac_bus_clks_config()
167 				clk_disable_unprepare(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
168 				clk_disable_unprepare(priv->plat->pclk);  in stmmac_bus_clks_config()
173 		clk_disable_unprepare(priv->plat->stmmac_clk);  in stmmac_bus_clks_config()
174 		clk_disable_unprepare(priv->plat->pclk);  in stmmac_bus_clks_config()
175 		if (priv->plat->clks_config)  in stmmac_bus_clks_config()
176 			priv->plat->clks_config(priv->plat->bsp_priv, enabled);  in stmmac_bus_clks_config()
204 static void __stmmac_disable_all_queues(struct stmmac_priv *priv)  in __stmmac_disable_all_queues()  argument
206 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;  in __stmmac_disable_all_queues()
207 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;  in __stmmac_disable_all_queues()
212 		struct stmmac_channel *ch = &priv->channel[queue];  in __stmmac_disable_all_queues()
214 		if (stmmac_xdp_is_enabled(priv) &&  in __stmmac_disable_all_queues()
215 		    test_bit(queue, priv->af_xdp_zc_qps)) {  in __stmmac_disable_all_queues()
231 static void stmmac_disable_all_queues(struct stmmac_priv *priv)  in stmmac_disable_all_queues()  argument
233 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;  in stmmac_disable_all_queues()
239 		rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_disable_all_queues()
246 	__stmmac_disable_all_queues(priv);  in stmmac_disable_all_queues()
253 static void stmmac_enable_all_queues(struct stmmac_priv *priv)  in stmmac_enable_all_queues()  argument
255 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;  in stmmac_enable_all_queues()
256 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;  in stmmac_enable_all_queues()
261 		struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_enable_all_queues()
263 		if (stmmac_xdp_is_enabled(priv) &&  in stmmac_enable_all_queues()
264 		    test_bit(queue, priv->af_xdp_zc_qps)) {  in stmmac_enable_all_queues()
276 static void stmmac_service_event_schedule(struct stmmac_priv *priv)  in stmmac_service_event_schedule()  argument
278 	if (!test_bit(STMMAC_DOWN, &priv->state) &&  in stmmac_service_event_schedule()
279 	    !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))  in stmmac_service_event_schedule()
280 		queue_work(priv->wq, &priv->service_task);  in stmmac_service_event_schedule()
283 static void stmmac_global_err(struct stmmac_priv *priv)  in stmmac_global_err()  argument
285 	netif_carrier_off(priv->dev);  in stmmac_global_err()
286 	set_bit(STMMAC_RESET_REQUESTED, &priv->state);  in stmmac_global_err()
287 	stmmac_service_event_schedule(priv);  in stmmac_global_err()
302 static void stmmac_clk_csr_set(struct stmmac_priv *priv)  in stmmac_clk_csr_set()  argument
306 	clk_rate = clk_get_rate(priv->plat->stmmac_clk);  in stmmac_clk_csr_set()
315 	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {  in stmmac_clk_csr_set()
317 			priv->clk_csr = STMMAC_CSR_20_35M;  in stmmac_clk_csr_set()
319 			priv->clk_csr = STMMAC_CSR_35_60M;  in stmmac_clk_csr_set()
321 			priv->clk_csr = STMMAC_CSR_60_100M;  in stmmac_clk_csr_set()
323 			priv->clk_csr = STMMAC_CSR_100_150M;  in stmmac_clk_csr_set()
325 			priv->clk_csr = STMMAC_CSR_150_250M;  in stmmac_clk_csr_set()
327 			priv->clk_csr = STMMAC_CSR_250_300M;  in stmmac_clk_csr_set()
330 	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) {  in stmmac_clk_csr_set()
332 			priv->clk_csr = 0x03;  in stmmac_clk_csr_set()
334 			priv->clk_csr = 0x02;  in stmmac_clk_csr_set()
336 			priv->clk_csr = 0x01;  in stmmac_clk_csr_set()
338 			priv->clk_csr = 0;  in stmmac_clk_csr_set()
341 	if (priv->plat->has_xgmac) {  in stmmac_clk_csr_set()
343 			priv->clk_csr = 0x5;  in stmmac_clk_csr_set()
345 			priv->clk_csr = 0x4;  in stmmac_clk_csr_set()
347 			priv->clk_csr = 0x3;  in stmmac_clk_csr_set()
349 			priv->clk_csr = 0x2;  in stmmac_clk_csr_set()
351 			priv->clk_csr = 0x1;  in stmmac_clk_csr_set()
353 			priv->clk_csr = 0x0;  in stmmac_clk_csr_set()
363 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)  in stmmac_tx_avail()  argument
365 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tx_avail()
371 		avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;  in stmmac_tx_avail()
381 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)  in stmmac_rx_dirty()  argument
383 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_dirty()
389 		dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;  in stmmac_rx_dirty()
394 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en)  in stmmac_lpi_entry_timer_config()  argument
399 	priv->eee_sw_timer_en = en ? 0 : 1;  in stmmac_lpi_entry_timer_config()
400 	tx_lpi_timer  = en ? priv->tx_lpi_timer : 0;  in stmmac_lpi_entry_timer_config()
401 	stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer);  in stmmac_lpi_entry_timer_config()
410 static int stmmac_enable_eee_mode(struct stmmac_priv *priv)  in stmmac_enable_eee_mode()  argument
412 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_enable_eee_mode()
417 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_enable_eee_mode()
424 	if (!priv->tx_path_in_lpi_mode)  in stmmac_enable_eee_mode()
425 		stmmac_set_eee_mode(priv, priv->hw,  in stmmac_enable_eee_mode()
426 			priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING);  in stmmac_enable_eee_mode()
436 void stmmac_disable_eee_mode(struct stmmac_priv *priv)  in stmmac_disable_eee_mode()  argument
438 	if (!priv->eee_sw_timer_en) {  in stmmac_disable_eee_mode()
439 		stmmac_lpi_entry_timer_config(priv, 0);  in stmmac_disable_eee_mode()
443 	stmmac_reset_eee_mode(priv, priv->hw);  in stmmac_disable_eee_mode()
444 	del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_disable_eee_mode()
445 	priv->tx_path_in_lpi_mode = false;  in stmmac_disable_eee_mode()
457 	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);  in stmmac_eee_ctrl_timer()  local
459 	if (stmmac_enable_eee_mode(priv))  in stmmac_eee_ctrl_timer()
460 		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));  in stmmac_eee_ctrl_timer()
471 bool stmmac_eee_init(struct stmmac_priv *priv)  in stmmac_eee_init()  argument
473 	int eee_tw_timer = priv->eee_tw_timer;  in stmmac_eee_init()
476 	if (!priv->dma_cap.eee)  in stmmac_eee_init()
479 	mutex_lock(&priv->lock);  in stmmac_eee_init()
482 	if (!priv->eee_active) {  in stmmac_eee_init()
483 		if (priv->eee_enabled) {  in stmmac_eee_init()
484 			netdev_dbg(priv->dev, "disable EEE\n");  in stmmac_eee_init()
485 			stmmac_lpi_entry_timer_config(priv, 0);  in stmmac_eee_init()
486 			del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_eee_init()
487 			stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer);  in stmmac_eee_init()
488 			if (priv->hw->xpcs)  in stmmac_eee_init()
489 				xpcs_config_eee(priv->hw->xpcs,  in stmmac_eee_init()
490 						priv->plat->mult_fact_100ns,  in stmmac_eee_init()
493 		mutex_unlock(&priv->lock);  in stmmac_eee_init()
497 	if (priv->eee_active && !priv->eee_enabled) {  in stmmac_eee_init()
498 		timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);  in stmmac_eee_init()
499 		stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,  in stmmac_eee_init()
501 		if (priv->hw->xpcs)  in stmmac_eee_init()
502 			xpcs_config_eee(priv->hw->xpcs,  in stmmac_eee_init()
503 					priv->plat->mult_fact_100ns,  in stmmac_eee_init()
507 	if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) {  in stmmac_eee_init()
508 		del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_eee_init()
509 		priv->tx_path_in_lpi_mode = false;  in stmmac_eee_init()
510 		stmmac_lpi_entry_timer_config(priv, 1);  in stmmac_eee_init()
512 		stmmac_lpi_entry_timer_config(priv, 0);  in stmmac_eee_init()
513 		mod_timer(&priv->eee_ctrl_timer,  in stmmac_eee_init()
514 			  STMMAC_LPI_T(priv->tx_lpi_timer));  in stmmac_eee_init()
517 	mutex_unlock(&priv->lock);  in stmmac_eee_init()
518 	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");  in stmmac_eee_init()
530 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,  in stmmac_get_tx_hwtstamp()  argument
537 	if (!priv->hwts_tx_en)  in stmmac_get_tx_hwtstamp()
545 	if (stmmac_get_tx_timestamp_status(priv, p)) {  in stmmac_get_tx_hwtstamp()
546 		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);  in stmmac_get_tx_hwtstamp()
548 	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {  in stmmac_get_tx_hwtstamp()
553 		ns -= priv->plat->cdc_error_adj;  in stmmac_get_tx_hwtstamp()
558 		netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);  in stmmac_get_tx_hwtstamp()
573 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,  in stmmac_get_rx_hwtstamp()  argument
580 	if (!priv->hwts_rx_en)  in stmmac_get_rx_hwtstamp()
583 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)  in stmmac_get_rx_hwtstamp()
587 	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {  in stmmac_get_rx_hwtstamp()
588 		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);  in stmmac_get_rx_hwtstamp()
590 		ns -= priv->plat->cdc_error_adj;  in stmmac_get_rx_hwtstamp()
592 		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);  in stmmac_get_rx_hwtstamp()
597 		netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");  in stmmac_get_rx_hwtstamp()
614 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hwtstamp_set()  local
625 	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {  in stmmac_hwtstamp_set()
626 		netdev_alert(priv->dev, "No support for HW time stamping\n");  in stmmac_hwtstamp_set()
627 		priv->hwts_tx_en = 0;  in stmmac_hwtstamp_set()
628 		priv->hwts_rx_en = 0;  in stmmac_hwtstamp_set()
637 	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",  in stmmac_hwtstamp_set()
644 	if (priv->adv_ts) {  in stmmac_hwtstamp_set()
725 			if (priv->synopsys_id < DWMAC_CORE_4_10)  in stmmac_hwtstamp_set()
778 	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);  in stmmac_hwtstamp_set()
779 	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;  in stmmac_hwtstamp_set()
781 	priv->systime_flags = STMMAC_HWTS_ACTIVE;  in stmmac_hwtstamp_set()
783 	if (priv->hwts_tx_en || priv->hwts_rx_en) {  in stmmac_hwtstamp_set()
784 		priv->systime_flags |= tstamp_all | ptp_v2 |  in stmmac_hwtstamp_set()
790 	stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);  in stmmac_hwtstamp_set()
792 	memcpy(&priv->tstamp_config, &config, sizeof(config));  in stmmac_hwtstamp_set()
809 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hwtstamp_get()  local
810 	struct hwtstamp_config *config = &priv->tstamp_config;  in stmmac_hwtstamp_get()
812 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))  in stmmac_hwtstamp_get()
829 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags)  in stmmac_init_tstamp_counter()  argument
831 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;  in stmmac_init_tstamp_counter()
836 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))  in stmmac_init_tstamp_counter()
839 	stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);  in stmmac_init_tstamp_counter()
840 	priv->systime_flags = systime_flags;  in stmmac_init_tstamp_counter()
843 	stmmac_config_sub_second_increment(priv, priv->ptpaddr,  in stmmac_init_tstamp_counter()
844 					   priv->plat->clk_ptp_rate,  in stmmac_init_tstamp_counter()
849 	priv->sub_second_inc = sec_inc;  in stmmac_init_tstamp_counter()
857 	priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);  in stmmac_init_tstamp_counter()
858 	stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);  in stmmac_init_tstamp_counter()
864 	stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec);  in stmmac_init_tstamp_counter()
877 static int stmmac_init_ptp(struct stmmac_priv *priv)  in stmmac_init_ptp()  argument
879 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;  in stmmac_init_ptp()
882 	if (priv->plat->ptp_clk_freq_config)  in stmmac_init_ptp()
883 		priv->plat->ptp_clk_freq_config(priv);  in stmmac_init_ptp()
885 	ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE);  in stmmac_init_ptp()
889 	priv->adv_ts = 0;  in stmmac_init_ptp()
891 	if (xmac && priv->dma_cap.atime_stamp)  in stmmac_init_ptp()
892 		priv->adv_ts = 1;  in stmmac_init_ptp()
894 	else if (priv->extend_desc && priv->dma_cap.atime_stamp)  in stmmac_init_ptp()
895 		priv->adv_ts = 1;  in stmmac_init_ptp()
897 	if (priv->dma_cap.time_stamp)  in stmmac_init_ptp()
898 		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");  in stmmac_init_ptp()
900 	if (priv->adv_ts)  in stmmac_init_ptp()
901 		netdev_info(priv->dev,  in stmmac_init_ptp()
904 	priv->hwts_tx_en = 0;  in stmmac_init_ptp()
905 	priv->hwts_rx_en = 0;  in stmmac_init_ptp()
907 	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)  in stmmac_init_ptp()
908 		stmmac_hwtstamp_correct_latency(priv, priv);  in stmmac_init_ptp()
913 static void stmmac_release_ptp(struct stmmac_priv *priv)  in stmmac_release_ptp()  argument
915 	clk_disable_unprepare(priv->plat->clk_ptp_ref);  in stmmac_release_ptp()
916 	stmmac_ptp_unregister(priv);  in stmmac_release_ptp()
925 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)  in stmmac_mac_flow_ctrl()  argument
927 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_mac_flow_ctrl()
929 	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,  in stmmac_mac_flow_ctrl()
930 			priv->pause, tx_cnt);  in stmmac_mac_flow_ctrl()
936 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_get_caps()  local
939 	stmmac_mac_update_caps(priv);  in stmmac_mac_get_caps()
941 	config->mac_capabilities = priv->hw->link.caps;  in stmmac_mac_get_caps()
943 	if (priv->plat->max_speed)  in stmmac_mac_get_caps()
944 		phylink_limit_mac_speed(config, priv->plat->max_speed);  in stmmac_mac_get_caps()
952 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_select_pcs()  local
955 	if (priv->plat->select_pcs) {  in stmmac_mac_select_pcs()
956 		pcs = priv->plat->select_pcs(priv, interface);  in stmmac_mac_select_pcs()
973 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_link_down()  local
975 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_mac_link_down()
976 	priv->eee_active = false;  in stmmac_mac_link_down()
977 	priv->tx_lpi_enabled = false;  in stmmac_mac_link_down()
978 	priv->eee_enabled = stmmac_eee_init(priv);  in stmmac_mac_link_down()
979 	stmmac_set_eee_pls(priv, priv->hw, false);  in stmmac_mac_link_down()
981 	if (stmmac_fpe_supported(priv))  in stmmac_mac_link_down()
982 		stmmac_fpe_link_state_handle(priv, false);  in stmmac_mac_link_down()
991 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_link_up()  local
994 	if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&  in stmmac_mac_link_up()
995 	    priv->plat->serdes_powerup)  in stmmac_mac_link_up()
996 		priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv);  in stmmac_mac_link_up()
998 	old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG);  in stmmac_mac_link_up()
999 	ctrl = old_ctrl & ~priv->hw->link.speed_mask;  in stmmac_mac_link_up()
1004 			ctrl |= priv->hw->link.xgmii.speed10000;  in stmmac_mac_link_up()
1007 			ctrl |= priv->hw->link.xgmii.speed5000;  in stmmac_mac_link_up()
1010 			ctrl |= priv->hw->link.xgmii.speed2500;  in stmmac_mac_link_up()
1018 			ctrl |= priv->hw->link.xlgmii.speed100000;  in stmmac_mac_link_up()
1021 			ctrl |= priv->hw->link.xlgmii.speed50000;  in stmmac_mac_link_up()
1024 			ctrl |= priv->hw->link.xlgmii.speed40000;  in stmmac_mac_link_up()
1027 			ctrl |= priv->hw->link.xlgmii.speed25000;  in stmmac_mac_link_up()
1030 			ctrl |= priv->hw->link.xgmii.speed10000;  in stmmac_mac_link_up()
1033 			ctrl |= priv->hw->link.speed2500;  in stmmac_mac_link_up()
1036 			ctrl |= priv->hw->link.speed1000;  in stmmac_mac_link_up()
1044 			ctrl |= priv->hw->link.speed2500;  in stmmac_mac_link_up()
1047 			ctrl |= priv->hw->link.speed1000;  in stmmac_mac_link_up()
1050 			ctrl |= priv->hw->link.speed100;  in stmmac_mac_link_up()
1053 			ctrl |= priv->hw->link.speed10;  in stmmac_mac_link_up()
1060 	priv->speed = speed;  in stmmac_mac_link_up()
1062 	if (priv->plat->fix_mac_speed)  in stmmac_mac_link_up()
1063 		priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode);  in stmmac_mac_link_up()
1066 		ctrl &= ~priv->hw->link.duplex;  in stmmac_mac_link_up()
1068 		ctrl |= priv->hw->link.duplex;  in stmmac_mac_link_up()
1072 		priv->flow_ctrl = FLOW_AUTO;  in stmmac_mac_link_up()
1074 		priv->flow_ctrl = FLOW_RX;  in stmmac_mac_link_up()
1076 		priv->flow_ctrl = FLOW_TX;  in stmmac_mac_link_up()
1078 		priv->flow_ctrl = FLOW_OFF;  in stmmac_mac_link_up()
1080 	stmmac_mac_flow_ctrl(priv, duplex);  in stmmac_mac_link_up()
1083 		writel(ctrl, priv->ioaddr + MAC_CTRL_REG);  in stmmac_mac_link_up()
1085 	stmmac_mac_set(priv, priv->ioaddr, true);  in stmmac_mac_link_up()
1086 	if (phy && priv->dma_cap.eee) {  in stmmac_mac_link_up()
1087 		priv->eee_active =  in stmmac_mac_link_up()
1088 			phy_init_eee(phy, !(priv->plat->flags &  in stmmac_mac_link_up()
1090 		priv->eee_enabled = stmmac_eee_init(priv);  in stmmac_mac_link_up()
1091 		priv->tx_lpi_enabled = priv->eee_enabled;  in stmmac_mac_link_up()
1092 		stmmac_set_eee_pls(priv, priv->hw, true);  in stmmac_mac_link_up()
1095 	if (stmmac_fpe_supported(priv))  in stmmac_mac_link_up()
1096 		stmmac_fpe_link_state_handle(priv, true);  in stmmac_mac_link_up()
1098 	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)  in stmmac_mac_link_up()
1099 		stmmac_hwtstamp_correct_latency(priv, priv);  in stmmac_mac_link_up()
1117 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)  in stmmac_check_pcs_mode()  argument
1119 	int interface = priv->plat->mac_interface;  in stmmac_check_pcs_mode()
1121 	if (priv->dma_cap.pcs) {  in stmmac_check_pcs_mode()
1126 			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");  in stmmac_check_pcs_mode()
1127 			priv->hw->pcs = STMMAC_PCS_RGMII;  in stmmac_check_pcs_mode()
1129 			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");  in stmmac_check_pcs_mode()
1130 			priv->hw->pcs = STMMAC_PCS_SGMII;  in stmmac_check_pcs_mode()
1145 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_init_phy()  local
1150 	if (!phylink_expects_phy(priv->phylink))  in stmmac_init_phy()
1153 	fwnode = priv->plat->port_node;  in stmmac_init_phy()
1155 		fwnode = dev_fwnode(priv->device);  in stmmac_init_phy()
1166 		int addr = priv->plat->phy_addr;  in stmmac_init_phy()
1170 			netdev_err(priv->dev, "no phy found\n");  in stmmac_init_phy()
1174 		phydev = mdiobus_get_phy(priv->mii, addr);  in stmmac_init_phy()
1176 			netdev_err(priv->dev, "no phy at addr %d\n", addr);  in stmmac_init_phy()
1180 		if (priv->dma_cap.eee)  in stmmac_init_phy()
1183 		ret = phylink_connect_phy(priv->phylink, phydev);  in stmmac_init_phy()
1186 		ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0);  in stmmac_init_phy()
1189 	if (!priv->plat->pmt) {  in stmmac_init_phy()
1192 		phylink_ethtool_get_wol(priv->phylink, &wol);  in stmmac_init_phy()
1193 		device_set_wakeup_capable(priv->device, !!wol.supported);  in stmmac_init_phy()
1194 		device_set_wakeup_enable(priv->device, !!wol.wolopts);  in stmmac_init_phy()
1200 static int stmmac_phy_setup(struct stmmac_priv *priv)  in stmmac_phy_setup()  argument
1203 	int mode = priv->plat->phy_interface;  in stmmac_phy_setup()
1207 	priv->phylink_config.dev = &priv->dev->dev;  in stmmac_phy_setup()
1208 	priv->phylink_config.type = PHYLINK_NETDEV;  in stmmac_phy_setup()
1209 	priv->phylink_config.mac_managed_pm = true;  in stmmac_phy_setup()
1212 	priv->phylink_config.mac_requires_rxc = true;  in stmmac_phy_setup()
1214 	mdio_bus_data = priv->plat->mdio_bus_data;  in stmmac_phy_setup()
1216 		priv->phylink_config.default_an_inband =  in stmmac_phy_setup()
1222 	__set_bit(mode, priv->phylink_config.supported_interfaces);  in stmmac_phy_setup()
1225 	if (priv->hw->xpcs)  in stmmac_phy_setup()
1226 		xpcs_get_interfaces(priv->hw->xpcs,  in stmmac_phy_setup()
1227 				    priv->phylink_config.supported_interfaces);  in stmmac_phy_setup()
1229 	fwnode = priv->plat->port_node;  in stmmac_phy_setup()
1231 		fwnode = dev_fwnode(priv->device);  in stmmac_phy_setup()
1233 	phylink = phylink_create(&priv->phylink_config, fwnode,  in stmmac_phy_setup()
1238 	priv->phylink = phylink;  in stmmac_phy_setup()
1242 static void stmmac_display_rx_rings(struct stmmac_priv *priv,  in stmmac_display_rx_rings()  argument
1245 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_display_rx_rings()
1256 		if (priv->extend_desc) {  in stmmac_display_rx_rings()
1265 		stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true,  in stmmac_display_rx_rings()
1270 static void stmmac_display_tx_rings(struct stmmac_priv *priv,  in stmmac_display_tx_rings()  argument
1273 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_display_tx_rings()
1284 		if (priv->extend_desc) {  in stmmac_display_tx_rings()
1295 		stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false,  in stmmac_display_tx_rings()
1300 static void stmmac_display_rings(struct stmmac_priv *priv,  in stmmac_display_rings()  argument
1304 	stmmac_display_rx_rings(priv, dma_conf);  in stmmac_display_rings()
1307 	stmmac_display_tx_rings(priv, dma_conf);  in stmmac_display_rings()
1336 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv,  in stmmac_clear_rx_descriptors()  argument
1345 		if (priv->extend_desc)  in stmmac_clear_rx_descriptors()
1346 			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,  in stmmac_clear_rx_descriptors()
1347 					priv->use_riwt, priv->mode,  in stmmac_clear_rx_descriptors()
1351 			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],  in stmmac_clear_rx_descriptors()
1352 					priv->use_riwt, priv->mode,  in stmmac_clear_rx_descriptors()
1365 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv,  in stmmac_clear_tx_descriptors()  argument
1377 		if (priv->extend_desc)  in stmmac_clear_tx_descriptors()
1384 		stmmac_init_tx_desc(priv, p, priv->mode, last);  in stmmac_clear_tx_descriptors()
1395 static void stmmac_clear_descriptors(struct stmmac_priv *priv,  in stmmac_clear_descriptors()  argument
1398 	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;  in stmmac_clear_descriptors()
1399 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;  in stmmac_clear_descriptors()
1404 		stmmac_clear_rx_descriptors(priv, dma_conf, queue);  in stmmac_clear_descriptors()
1408 		stmmac_clear_tx_descriptors(priv, dma_conf, queue);  in stmmac_clear_descriptors()
1422 static int stmmac_init_rx_buffers(struct stmmac_priv *priv,  in stmmac_init_rx_buffers()  argument
1431 	if (priv->dma_cap.host_dma_width <= 32)  in stmmac_init_rx_buffers()
1438 		buf->page_offset = stmmac_rx_offset(priv);  in stmmac_init_rx_buffers()
1441 	if (priv->sph && !buf->sec_page) {  in stmmac_init_rx_buffers()
1447 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);  in stmmac_init_rx_buffers()
1450 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);  in stmmac_init_rx_buffers()
1455 	stmmac_set_desc_addr(priv, p, buf->addr);  in stmmac_init_rx_buffers()
1457 		stmmac_init_desc3(priv, p);  in stmmac_init_rx_buffers()
1468 static void stmmac_free_rx_buffer(struct stmmac_priv *priv,  in stmmac_free_rx_buffer()  argument
1490 static void stmmac_free_tx_buffer(struct stmmac_priv *priv,  in stmmac_free_tx_buffer()  argument
1499 			dma_unmap_page(priv->device,  in stmmac_free_tx_buffer()
1504 			dma_unmap_single(priv->device,  in stmmac_free_tx_buffer()
1536 static void dma_free_rx_skbufs(struct stmmac_priv *priv,  in dma_free_rx_skbufs()  argument
1544 		stmmac_free_rx_buffer(priv, rx_q, i);  in dma_free_rx_skbufs()
1547 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv,  in stmmac_alloc_rx_buffers()  argument
1558 		if (priv->extend_desc)  in stmmac_alloc_rx_buffers()
1563 		ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags,  in stmmac_alloc_rx_buffers()
1580 static void dma_free_rx_xskbufs(struct stmmac_priv *priv,  in dma_free_rx_xskbufs()  argument
1598 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv,  in stmmac_alloc_rx_buffers_zc()  argument
1616 		if (priv->extend_desc)  in stmmac_alloc_rx_buffers_zc()
1628 		stmmac_set_desc_addr(priv, p, dma_addr);  in stmmac_alloc_rx_buffers_zc()
1635 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue)  in stmmac_get_xsk_pool()  argument
1637 	if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps))  in stmmac_get_xsk_pool()
1640 	return xsk_get_pool_from_qid(priv->dev, queue);  in stmmac_get_xsk_pool()
1653 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,  in __init_dma_rx_desc_rings()  argument
1660 	netif_dbg(priv, probe, priv->dev,  in __init_dma_rx_desc_rings()
1664 	stmmac_clear_rx_descriptors(priv, dma_conf, queue);  in __init_dma_rx_desc_rings()
1668 	rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);  in __init_dma_rx_desc_rings()
1674 		netdev_info(priv->dev,  in __init_dma_rx_desc_rings()
1682 		netdev_info(priv->dev,  in __init_dma_rx_desc_rings()
1691 		stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue);  in __init_dma_rx_desc_rings()
1693 		ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags);  in __init_dma_rx_desc_rings()
1699 	if (priv->mode == STMMAC_CHAIN_MODE) {  in __init_dma_rx_desc_rings()
1700 		if (priv->extend_desc)  in __init_dma_rx_desc_rings()
1701 			stmmac_mode_init(priv, rx_q->dma_erx,  in __init_dma_rx_desc_rings()
1705 			stmmac_mode_init(priv, rx_q->dma_rx,  in __init_dma_rx_desc_rings()
1717 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_rx_desc_rings()  local
1718 	u32 rx_count = priv->plat->rx_queues_to_use;  in init_dma_rx_desc_rings()
1723 	netif_dbg(priv, probe, priv->dev,  in init_dma_rx_desc_rings()
1727 		ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags);  in init_dma_rx_desc_rings()
1739 			dma_free_rx_xskbufs(priv, dma_conf, queue);  in init_dma_rx_desc_rings()
1741 			dma_free_rx_skbufs(priv, dma_conf, queue);  in init_dma_rx_desc_rings()
1761 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,  in __init_dma_tx_desc_rings()  argument
1768 	netif_dbg(priv, probe, priv->dev,  in __init_dma_tx_desc_rings()
1773 	if (priv->mode == STMMAC_CHAIN_MODE) {  in __init_dma_tx_desc_rings()
1774 		if (priv->extend_desc)  in __init_dma_tx_desc_rings()
1775 			stmmac_mode_init(priv, tx_q->dma_etx,  in __init_dma_tx_desc_rings()
1779 			stmmac_mode_init(priv, tx_q->dma_tx,  in __init_dma_tx_desc_rings()
1784 	tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);  in __init_dma_tx_desc_rings()
1789 		if (priv->extend_desc)  in __init_dma_tx_desc_rings()
1796 		stmmac_clear_desc(priv, p);  in __init_dma_tx_desc_rings()
1811 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_tx_desc_rings()  local
1815 	tx_queue_cnt = priv->plat->tx_queues_to_use;  in init_dma_tx_desc_rings()
1818 		__init_dma_tx_desc_rings(priv, dma_conf, queue);  in init_dma_tx_desc_rings()
1836 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_desc_rings()  local
1845 	stmmac_clear_descriptors(priv, dma_conf);  in init_dma_desc_rings()
1847 	if (netif_msg_hw(priv))  in init_dma_desc_rings()
1848 		stmmac_display_rings(priv, dma_conf);  in init_dma_desc_rings()
1859 static void dma_free_tx_skbufs(struct stmmac_priv *priv,  in dma_free_tx_skbufs()  argument
1869 		stmmac_free_tx_buffer(priv, dma_conf, queue, i);  in dma_free_tx_skbufs()
1882 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)  in stmmac_free_tx_skbufs()  argument
1884 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;  in stmmac_free_tx_skbufs()
1888 		dma_free_tx_skbufs(priv, &priv->dma_conf, queue);  in stmmac_free_tx_skbufs()
1897 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,  in __free_dma_rx_desc_resources()  argument
1905 		dma_free_rx_xskbufs(priv, dma_conf, queue);  in __free_dma_rx_desc_resources()
1907 		dma_free_rx_skbufs(priv, dma_conf, queue);  in __free_dma_rx_desc_resources()
1913 	if (!priv->extend_desc)  in __free_dma_rx_desc_resources()
1914 		dma_free_coherent(priv->device, dma_conf->dma_rx_size *  in __free_dma_rx_desc_resources()
1918 		dma_free_coherent(priv->device, dma_conf->dma_rx_size *  in __free_dma_rx_desc_resources()
1930 static void free_dma_rx_desc_resources(struct stmmac_priv *priv,  in free_dma_rx_desc_resources()  argument
1933 	u32 rx_count = priv->plat->rx_queues_to_use;  in free_dma_rx_desc_resources()
1938 		__free_dma_rx_desc_resources(priv, dma_conf, queue);  in free_dma_rx_desc_resources()
1947 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,  in __free_dma_tx_desc_resources()  argument
1956 	dma_free_tx_skbufs(priv, dma_conf, queue);  in __free_dma_tx_desc_resources()
1958 	if (priv->extend_desc) {  in __free_dma_tx_desc_resources()
1971 	dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);  in __free_dma_tx_desc_resources()
1977 static void free_dma_tx_desc_resources(struct stmmac_priv *priv,  in free_dma_tx_desc_resources()  argument
1980 	u32 tx_count = priv->plat->tx_queues_to_use;  in free_dma_tx_desc_resources()
1985 		__free_dma_tx_desc_resources(priv, dma_conf, queue);  in free_dma_tx_desc_resources()
1998 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,  in __alloc_dma_rx_desc_resources()  argument
2003 	struct stmmac_channel *ch = &priv->channel[queue];  in __alloc_dma_rx_desc_resources()
2004 	bool xdp_prog = stmmac_xdp_is_enabled(priv);  in __alloc_dma_rx_desc_resources()
2011 	rx_q->priv_data = priv;  in __alloc_dma_rx_desc_resources()
2017 	pp_params.nid = dev_to_node(priv->device);  in __alloc_dma_rx_desc_resources()
2018 	pp_params.dev = priv->device;  in __alloc_dma_rx_desc_resources()
2020 	pp_params.offset = stmmac_rx_offset(priv);  in __alloc_dma_rx_desc_resources()
2036 	if (priv->extend_desc) {  in __alloc_dma_rx_desc_resources()
2037 		rx_q->dma_erx = dma_alloc_coherent(priv->device,  in __alloc_dma_rx_desc_resources()
2046 		rx_q->dma_rx = dma_alloc_coherent(priv->device,  in __alloc_dma_rx_desc_resources()
2055 	if (stmmac_xdp_is_enabled(priv) &&  in __alloc_dma_rx_desc_resources()
2056 	    test_bit(queue, priv->af_xdp_zc_qps))  in __alloc_dma_rx_desc_resources()
2061 	ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev,  in __alloc_dma_rx_desc_resources()
2065 		netdev_err(priv->dev, "Failed to register xdp rxq info\n");  in __alloc_dma_rx_desc_resources()
2072 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv,  in alloc_dma_rx_desc_resources()  argument
2075 	u32 rx_count = priv->plat->rx_queues_to_use;  in alloc_dma_rx_desc_resources()
2081 		ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue);  in alloc_dma_rx_desc_resources()
2089 	free_dma_rx_desc_resources(priv, dma_conf);  in alloc_dma_rx_desc_resources()
2104 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,  in __alloc_dma_tx_desc_resources()  argument
2113 	tx_q->priv_data = priv;  in __alloc_dma_tx_desc_resources()
2127 	if (priv->extend_desc)  in __alloc_dma_tx_desc_resources()
2136 	addr = dma_alloc_coherent(priv->device, size,  in __alloc_dma_tx_desc_resources()
2141 	if (priv->extend_desc)  in __alloc_dma_tx_desc_resources()
2151 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv,  in alloc_dma_tx_desc_resources()  argument
2154 	u32 tx_count = priv->plat->tx_queues_to_use;  in alloc_dma_tx_desc_resources()
2160 		ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue);  in alloc_dma_tx_desc_resources()
2168 	free_dma_tx_desc_resources(priv, dma_conf);  in alloc_dma_tx_desc_resources()
2181 static int alloc_dma_desc_resources(struct stmmac_priv *priv,  in alloc_dma_desc_resources()  argument
2185 	int ret = alloc_dma_rx_desc_resources(priv, dma_conf);  in alloc_dma_desc_resources()
2190 	ret = alloc_dma_tx_desc_resources(priv, dma_conf);  in alloc_dma_desc_resources()
2200 static void free_dma_desc_resources(struct stmmac_priv *priv,  in free_dma_desc_resources()  argument
2204 	free_dma_tx_desc_resources(priv, dma_conf);  in free_dma_desc_resources()
2209 	free_dma_rx_desc_resources(priv, dma_conf);  in free_dma_desc_resources()
2217 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)  in stmmac_mac_enable_rx_queues()  argument
2219 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mac_enable_rx_queues()
2224 		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;  in stmmac_mac_enable_rx_queues()
2225 		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);  in stmmac_mac_enable_rx_queues()
2236 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_start_rx_dma()  argument
2238 	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);  in stmmac_start_rx_dma()
2239 	stmmac_start_rx(priv, priv->ioaddr, chan);  in stmmac_start_rx_dma()
2249 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_start_tx_dma()  argument
2251 	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);  in stmmac_start_tx_dma()
2252 	stmmac_start_tx(priv, priv->ioaddr, chan);  in stmmac_start_tx_dma()
2262 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_stop_rx_dma()  argument
2264 	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);  in stmmac_stop_rx_dma()
2265 	stmmac_stop_rx(priv, priv->ioaddr, chan);  in stmmac_stop_rx_dma()
2275 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_stop_tx_dma()  argument
2277 	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);  in stmmac_stop_tx_dma()
2278 	stmmac_stop_tx(priv, priv->ioaddr, chan);  in stmmac_stop_tx_dma()
2281 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)  in stmmac_enable_all_dma_irq()  argument
2283 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_enable_all_dma_irq()
2284 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_enable_all_dma_irq()
2289 		struct stmmac_channel *ch = &priv->channel[chan];  in stmmac_enable_all_dma_irq()
2293 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_enable_all_dma_irq()
2304 static void stmmac_start_all_dma(struct stmmac_priv *priv)  in stmmac_start_all_dma()  argument
2306 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_start_all_dma()
2307 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_start_all_dma()
2311 		stmmac_start_rx_dma(priv, chan);  in stmmac_start_all_dma()
2314 		stmmac_start_tx_dma(priv, chan);  in stmmac_start_all_dma()
2323 static void stmmac_stop_all_dma(struct stmmac_priv *priv)  in stmmac_stop_all_dma()  argument
2325 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_stop_all_dma()
2326 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_stop_all_dma()
2330 		stmmac_stop_rx_dma(priv, chan);  in stmmac_stop_all_dma()
2333 		stmmac_stop_tx_dma(priv, chan);  in stmmac_stop_all_dma()
2342 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)  in stmmac_dma_operation_mode()  argument
2344 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_dma_operation_mode()
2345 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_dma_operation_mode()
2346 	int rxfifosz = priv->plat->rx_fifo_size;  in stmmac_dma_operation_mode()
2347 	int txfifosz = priv->plat->tx_fifo_size;  in stmmac_dma_operation_mode()
2354 		rxfifosz = priv->dma_cap.rx_fifo_size;  in stmmac_dma_operation_mode()
2356 		txfifosz = priv->dma_cap.tx_fifo_size;  in stmmac_dma_operation_mode()
2359 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac) {  in stmmac_dma_operation_mode()
2364 	if (priv->plat->force_thresh_dma_mode) {  in stmmac_dma_operation_mode()
2367 	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {  in stmmac_dma_operation_mode()
2377 		priv->xstats.threshold = SF_DMA_MODE;  in stmmac_dma_operation_mode()
2385 		struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_dma_operation_mode()
2388 		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;  in stmmac_dma_operation_mode()
2390 		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,  in stmmac_dma_operation_mode()
2395 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_dma_operation_mode()
2399 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_dma_operation_mode()
2400 					      priv->dma_conf.dma_buf_sz,  in stmmac_dma_operation_mode()
2406 		qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;  in stmmac_dma_operation_mode()
2408 		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,  in stmmac_dma_operation_mode()
2417 	stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc);  in stmmac_xsk_request_timestamp()
2424 	struct stmmac_priv *priv = tx_compl->priv;  in stmmac_xsk_fill_timestamp()  local
2429 	if (!priv->hwts_tx_en)  in stmmac_xsk_fill_timestamp()
2433 	if (stmmac_get_tx_timestamp_status(priv, desc)) {  in stmmac_xsk_fill_timestamp()
2434 		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);  in stmmac_xsk_fill_timestamp()
2436 	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {  in stmmac_xsk_fill_timestamp()
2441 		ns -= priv->plat->cdc_error_adj;  in stmmac_xsk_fill_timestamp()
2453 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)  in stmmac_xdp_xmit_zc()  argument
2455 	struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue);  in stmmac_xdp_xmit_zc()
2456 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xdp_xmit_zc()
2457 	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_xdp_xmit_zc()
2468 	budget = min(budget, stmmac_tx_avail(priv, queue));  in stmmac_xdp_xmit_zc()
2479 		if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) ||  in stmmac_xdp_xmit_zc()
2480 		    !netif_carrier_ok(priv->dev)) {  in stmmac_xdp_xmit_zc()
2488 		if (priv->est && priv->est->enable &&  in stmmac_xdp_xmit_zc()
2489 		    priv->est->max_sdu[queue] &&  in stmmac_xdp_xmit_zc()
2490 		    xdp_desc.len > priv->est->max_sdu[queue]) {  in stmmac_xdp_xmit_zc()
2491 			priv->xstats.max_sdu_txq_drop[queue]++;  in stmmac_xdp_xmit_zc()
2495 		if (likely(priv->extend_desc))  in stmmac_xdp_xmit_zc()
2520 		stmmac_set_desc_addr(priv, tx_desc, dma_addr);  in stmmac_xdp_xmit_zc()
2524 		if (!priv->tx_coal_frames[queue])  in stmmac_xdp_xmit_zc()
2526 		else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)  in stmmac_xdp_xmit_zc()
2531 		meta_req.priv = priv;  in stmmac_xdp_xmit_zc()
2538 			stmmac_set_tx_ic(priv, tx_desc);  in stmmac_xdp_xmit_zc()
2542 		stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len,  in stmmac_xdp_xmit_zc()
2543 				       true, priv->mode, true, true,  in stmmac_xdp_xmit_zc()
2546 		stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);  in stmmac_xdp_xmit_zc()
2551 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);  in stmmac_xdp_xmit_zc()
2559 		stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xdp_xmit_zc()
2571 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan)  in stmmac_bump_dma_threshold()  argument
2573 	if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) {  in stmmac_bump_dma_threshold()
2576 		if (priv->plat->force_thresh_dma_mode)  in stmmac_bump_dma_threshold()
2577 			stmmac_set_dma_operation_mode(priv, tc, tc, chan);  in stmmac_bump_dma_threshold()
2579 			stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE,  in stmmac_bump_dma_threshold()
2582 		priv->xstats.threshold = tc;  in stmmac_bump_dma_threshold()
2596 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,  in stmmac_tx_clean()  argument
2599 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tx_clean()
2600 	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_tx_clean()
2605 	__netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tx_clean()
2612 	while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) {  in stmmac_tx_clean()
2630 		if (priv->extend_desc)  in stmmac_tx_clean()
2637 		status = stmmac_tx_status(priv,	&priv->xstats, p, priv->ioaddr);  in stmmac_tx_clean()
2655 					stmmac_bump_dma_threshold(priv, queue);  in stmmac_tx_clean()
2660 				stmmac_get_tx_hwtstamp(priv, p, skb);  in stmmac_tx_clean()
2664 					.priv = priv,  in stmmac_tx_clean()
2677 				dma_unmap_page(priv->device,  in stmmac_tx_clean()
2682 				dma_unmap_single(priv->device,  in stmmac_tx_clean()
2691 		stmmac_clean_desc3(priv, tx_q, p);  in stmmac_tx_clean()
2720 		stmmac_release_tx_desc(priv, p, priv->mode);  in stmmac_tx_clean()
2722 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_tx_clean()
2726 	netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),  in stmmac_tx_clean()
2729 	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,  in stmmac_tx_clean()
2731 	    stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {  in stmmac_tx_clean()
2733 		netif_dbg(priv, tx_done, priv->dev,  in stmmac_tx_clean()
2735 		netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tx_clean()
2752 		work_done = stmmac_xdp_xmit_zc(priv, queue,  in stmmac_tx_clean()
2760 	if (priv->eee_enabled && !priv->tx_path_in_lpi_mode &&  in stmmac_tx_clean()
2761 	    priv->eee_sw_timer_en) {  in stmmac_tx_clean()
2762 		if (stmmac_enable_eee_mode(priv))  in stmmac_tx_clean()
2763 			mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));  in stmmac_tx_clean()
2776 	priv->xstats.tx_errors += tx_errors;  in stmmac_tx_clean()
2778 	__netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tx_clean()
2791 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)  in stmmac_tx_err()  argument
2793 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_tx_err()
2795 	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));  in stmmac_tx_err()
2797 	stmmac_stop_tx_dma(priv, chan);  in stmmac_tx_err()
2798 	dma_free_tx_skbufs(priv, &priv->dma_conf, chan);  in stmmac_tx_err()
2799 	stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan);  in stmmac_tx_err()
2800 	stmmac_reset_tx_queue(priv, chan);  in stmmac_tx_err()
2801 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_tx_err()
2803 	stmmac_start_tx_dma(priv, chan);  in stmmac_tx_err()
2805 	priv->xstats.tx_errors++;  in stmmac_tx_err()
2806 	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));  in stmmac_tx_err()
2819 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,  in stmmac_set_dma_operation_mode()  argument
2822 	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;  in stmmac_set_dma_operation_mode()
2823 	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;  in stmmac_set_dma_operation_mode()
2824 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_set_dma_operation_mode()
2825 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_set_dma_operation_mode()
2826 	int rxfifosz = priv->plat->rx_fifo_size;  in stmmac_set_dma_operation_mode()
2827 	int txfifosz = priv->plat->tx_fifo_size;  in stmmac_set_dma_operation_mode()
2830 		rxfifosz = priv->dma_cap.rx_fifo_size;  in stmmac_set_dma_operation_mode()
2832 		txfifosz = priv->dma_cap.tx_fifo_size;  in stmmac_set_dma_operation_mode()
2838 	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);  in stmmac_set_dma_operation_mode()
2839 	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);  in stmmac_set_dma_operation_mode()
2842 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)  in stmmac_safety_feat_interrupt()  argument
2846 	ret = stmmac_safety_feat_irq_status(priv, priv->dev,  in stmmac_safety_feat_interrupt()
2847 			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);  in stmmac_safety_feat_interrupt()
2849 		stmmac_global_err(priv);  in stmmac_safety_feat_interrupt()
2856 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)  in stmmac_napi_check()  argument
2858 	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,  in stmmac_napi_check()
2859 						 &priv->xstats, chan, dir);  in stmmac_napi_check()
2860 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_napi_check()
2861 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_napi_check()
2862 	struct stmmac_channel *ch = &priv->channel[chan];  in stmmac_napi_check()
2870 	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {  in stmmac_napi_check()
2873 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);  in stmmac_napi_check()
2879 	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {  in stmmac_napi_check()
2882 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);  in stmmac_napi_check()
2898 static void stmmac_dma_interrupt(struct stmmac_priv *priv)  in stmmac_dma_interrupt()  argument
2900 	u32 tx_channel_count = priv->plat->tx_queues_to_use;  in stmmac_dma_interrupt()
2901 	u32 rx_channel_count = priv->plat->rx_queues_to_use;  in stmmac_dma_interrupt()
2912 		status[chan] = stmmac_napi_check(priv, chan,  in stmmac_dma_interrupt()
2918 			stmmac_bump_dma_threshold(priv, chan);  in stmmac_dma_interrupt()
2920 			stmmac_tx_err(priv, chan);  in stmmac_dma_interrupt()
2930 static void stmmac_mmc_setup(struct stmmac_priv *priv)  in stmmac_mmc_setup()  argument
2935 	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);  in stmmac_mmc_setup()
2937 	if (priv->dma_cap.rmon) {  in stmmac_mmc_setup()
2938 		stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);  in stmmac_mmc_setup()
2939 		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));  in stmmac_mmc_setup()
2941 		netdev_info(priv->dev, "No MAC Management Counters available\n");  in stmmac_mmc_setup()
2953 static int stmmac_get_hw_features(struct stmmac_priv *priv)  in stmmac_get_hw_features()  argument
2955 	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;  in stmmac_get_hw_features()
2965 static void stmmac_check_ether_addr(struct stmmac_priv *priv)  in stmmac_check_ether_addr()  argument
2969 	if (!is_valid_ether_addr(priv->dev->dev_addr)) {  in stmmac_check_ether_addr()
2970 		stmmac_get_umac_addr(priv, priv->hw, addr, 0);  in stmmac_check_ether_addr()
2972 			eth_hw_addr_set(priv->dev, addr);  in stmmac_check_ether_addr()
2974 			eth_hw_addr_random(priv->dev);  in stmmac_check_ether_addr()
2975 		dev_info(priv->device, "device MAC address %pM\n",  in stmmac_check_ether_addr()
2976 			 priv->dev->dev_addr);  in stmmac_check_ether_addr()
2988 static int stmmac_init_dma_engine(struct stmmac_priv *priv)  in stmmac_init_dma_engine()  argument
2990 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_init_dma_engine()
2991 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_init_dma_engine()
2998 	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {  in stmmac_init_dma_engine()
2999 		dev_err(priv->device, "Invalid DMA configuration\n");  in stmmac_init_dma_engine()
3003 	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))  in stmmac_init_dma_engine()
3004 		priv->plat->dma_cfg->atds = 1;  in stmmac_init_dma_engine()
3006 	ret = stmmac_reset(priv, priv->ioaddr);  in stmmac_init_dma_engine()
3008 		dev_err(priv->device, "Failed to reset the dma\n");  in stmmac_init_dma_engine()
3013 	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg);  in stmmac_init_dma_engine()
3015 	if (priv->plat->axi)  in stmmac_init_dma_engine()
3016 		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);  in stmmac_init_dma_engine()
3020 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);  in stmmac_init_dma_engine()
3021 		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_init_dma_engine()
3026 		rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_init_dma_engine()
3028 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_init_dma_engine()
3034 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,  in stmmac_init_dma_engine()
3040 		tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_init_dma_engine()
3042 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_init_dma_engine()
3046 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,  in stmmac_init_dma_engine()
3053 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)  in stmmac_tx_timer_arm()  argument
3055 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tx_timer_arm()
3056 	u32 tx_coal_timer = priv->tx_coal_timer[queue];  in stmmac_tx_timer_arm()
3063 	ch = &priv->channel[tx_q->queue_index];  in stmmac_tx_timer_arm()
3087 	struct stmmac_priv *priv = tx_q->priv_data;  in stmmac_tx_timer()  local
3091 	ch = &priv->channel[tx_q->queue_index];  in stmmac_tx_timer()
3098 		stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);  in stmmac_tx_timer()
3114 static void stmmac_init_coalesce(struct stmmac_priv *priv)  in stmmac_init_coalesce()  argument
3116 	u32 tx_channel_count = priv->plat->tx_queues_to_use;  in stmmac_init_coalesce()
3117 	u32 rx_channel_count = priv->plat->rx_queues_to_use;  in stmmac_init_coalesce()
3121 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_init_coalesce()
3123 		priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES;  in stmmac_init_coalesce()
3124 		priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER;  in stmmac_init_coalesce()
3131 		priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES;  in stmmac_init_coalesce()
3134 static void stmmac_set_rings_length(struct stmmac_priv *priv)  in stmmac_set_rings_length()  argument
3136 	u32 rx_channels_count = priv->plat->rx_queues_to_use;  in stmmac_set_rings_length()
3137 	u32 tx_channels_count = priv->plat->tx_queues_to_use;  in stmmac_set_rings_length()
3142 		stmmac_set_tx_ring_len(priv, priv->ioaddr,  in stmmac_set_rings_length()
3143 				       (priv->dma_conf.dma_tx_size - 1), chan);  in stmmac_set_rings_length()
3147 		stmmac_set_rx_ring_len(priv, priv->ioaddr,  in stmmac_set_rings_length()
3148 				       (priv->dma_conf.dma_rx_size - 1), chan);  in stmmac_set_rings_length()
3156 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)  in stmmac_set_tx_queue_weight()  argument
3158 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_set_tx_queue_weight()
3163 		weight = priv->plat->tx_queues_cfg[queue].weight;  in stmmac_set_tx_queue_weight()
3164 		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);  in stmmac_set_tx_queue_weight()
3173 static void stmmac_configure_cbs(struct stmmac_priv *priv)  in stmmac_configure_cbs()  argument
3175 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_configure_cbs()
3181 		mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;  in stmmac_configure_cbs()
3185 		stmmac_config_cbs(priv, priv->hw,  in stmmac_configure_cbs()
3186 				priv->plat->tx_queues_cfg[queue].send_slope,  in stmmac_configure_cbs()
3187 				priv->plat->tx_queues_cfg[queue].idle_slope,  in stmmac_configure_cbs()
3188 				priv->plat->tx_queues_cfg[queue].high_credit,  in stmmac_configure_cbs()
3189 				priv->plat->tx_queues_cfg[queue].low_credit,  in stmmac_configure_cbs()
3199 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)  in stmmac_rx_queue_dma_chan_map()  argument
3201 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_rx_queue_dma_chan_map()
3206 		chan = priv->plat->rx_queues_cfg[queue].chan;  in stmmac_rx_queue_dma_chan_map()
3207 		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);  in stmmac_rx_queue_dma_chan_map()
3216 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)  in stmmac_mac_config_rx_queues_prio()  argument
3218 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mac_config_rx_queues_prio()
3223 		if (!priv->plat->rx_queues_cfg[queue].use_prio)  in stmmac_mac_config_rx_queues_prio()
3226 		prio = priv->plat->rx_queues_cfg[queue].prio;  in stmmac_mac_config_rx_queues_prio()
3227 		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);  in stmmac_mac_config_rx_queues_prio()
3236 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)  in stmmac_mac_config_tx_queues_prio()  argument
3238 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_mac_config_tx_queues_prio()
3243 		if (!priv->plat->tx_queues_cfg[queue].use_prio)  in stmmac_mac_config_tx_queues_prio()
3246 		prio = priv->plat->tx_queues_cfg[queue].prio;  in stmmac_mac_config_tx_queues_prio()
3247 		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);  in stmmac_mac_config_tx_queues_prio()
3256 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)  in stmmac_mac_config_rx_queues_routing()  argument
3258 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mac_config_rx_queues_routing()
3264 		if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)  in stmmac_mac_config_rx_queues_routing()
3267 		packet = priv->plat->rx_queues_cfg[queue].pkt_route;  in stmmac_mac_config_rx_queues_routing()
3268 		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);  in stmmac_mac_config_rx_queues_routing()
3272 static void stmmac_mac_config_rss(struct stmmac_priv *priv)  in stmmac_mac_config_rss()  argument
3274 	if (!priv->dma_cap.rssen || !priv->plat->rss_en) {  in stmmac_mac_config_rss()
3275 		priv->rss.enable = false;  in stmmac_mac_config_rss()
3279 	if (priv->dev->features & NETIF_F_RXHASH)  in stmmac_mac_config_rss()
3280 		priv->rss.enable = true;  in stmmac_mac_config_rss()
3282 		priv->rss.enable = false;  in stmmac_mac_config_rss()
3284 	stmmac_rss_configure(priv, priv->hw, &priv->rss,  in stmmac_mac_config_rss()
3285 			     priv->plat->rx_queues_to_use);  in stmmac_mac_config_rss()
3293 static void stmmac_mtl_configuration(struct stmmac_priv *priv)  in stmmac_mtl_configuration()  argument
3295 	u32 rx_queues_count = priv->plat->rx_queues_to_use;  in stmmac_mtl_configuration()
3296 	u32 tx_queues_count = priv->plat->tx_queues_to_use;  in stmmac_mtl_configuration()
3299 		stmmac_set_tx_queue_weight(priv);  in stmmac_mtl_configuration()
3303 		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,  in stmmac_mtl_configuration()
3304 				priv->plat->rx_sched_algorithm);  in stmmac_mtl_configuration()
3308 		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,  in stmmac_mtl_configuration()
3309 				priv->plat->tx_sched_algorithm);  in stmmac_mtl_configuration()
3313 		stmmac_configure_cbs(priv);  in stmmac_mtl_configuration()
3316 	stmmac_rx_queue_dma_chan_map(priv);  in stmmac_mtl_configuration()
3319 	stmmac_mac_enable_rx_queues(priv);  in stmmac_mtl_configuration()
3323 		stmmac_mac_config_rx_queues_prio(priv);  in stmmac_mtl_configuration()
3327 		stmmac_mac_config_tx_queues_prio(priv);  in stmmac_mtl_configuration()
3331 		stmmac_mac_config_rx_queues_routing(priv);  in stmmac_mtl_configuration()
3335 		stmmac_mac_config_rss(priv);  in stmmac_mtl_configuration()
3338 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)  in stmmac_safety_feat_configuration()  argument
3340 	if (priv->dma_cap.asp) {  in stmmac_safety_feat_configuration()
3341 		netdev_info(priv->dev, "Enabling Safety Features\n");  in stmmac_safety_feat_configuration()
3342 		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp,  in stmmac_safety_feat_configuration()
3343 					  priv->plat->safety_feat_cfg);  in stmmac_safety_feat_configuration()
3345 		netdev_info(priv->dev, "No Safety Features support found\n");  in stmmac_safety_feat_configuration()
3364 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hw_setup()  local
3365 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_hw_setup()
3366 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_hw_setup()
3372 	if (priv->hw->phylink_pcs)  in stmmac_hw_setup()
3373 		phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs);  in stmmac_hw_setup()
3376 	ret = stmmac_init_dma_engine(priv);  in stmmac_hw_setup()
3378 		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",  in stmmac_hw_setup()
3384 	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);  in stmmac_hw_setup()
3387 	if (priv->hw->pcs) {  in stmmac_hw_setup()
3388 		int speed = priv->plat->mac_port_sel_speed;  in stmmac_hw_setup()
3392 			priv->hw->ps = speed;  in stmmac_hw_setup()
3394 			dev_warn(priv->device, "invalid port speed\n");  in stmmac_hw_setup()
3395 			priv->hw->ps = 0;  in stmmac_hw_setup()
3400 	stmmac_core_init(priv, priv->hw, dev);  in stmmac_hw_setup()
3403 	stmmac_mtl_configuration(priv);  in stmmac_hw_setup()
3406 	stmmac_safety_feat_configuration(priv);  in stmmac_hw_setup()
3408 	ret = stmmac_rx_ipc(priv, priv->hw);  in stmmac_hw_setup()
3410 		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");  in stmmac_hw_setup()
3411 		priv->plat->rx_coe = STMMAC_RX_COE_NONE;  in stmmac_hw_setup()
3412 		priv->hw->rx_csum = 0;  in stmmac_hw_setup()
3416 	stmmac_mac_set(priv, priv->ioaddr, true);  in stmmac_hw_setup()
3419 	stmmac_dma_operation_mode(priv);  in stmmac_hw_setup()
3421 	stmmac_mmc_setup(priv);  in stmmac_hw_setup()
3424 		ret = clk_prepare_enable(priv->plat->clk_ptp_ref);  in stmmac_hw_setup()
3426 			netdev_warn(priv->dev,  in stmmac_hw_setup()
3431 	ret = stmmac_init_ptp(priv);  in stmmac_hw_setup()
3433 		netdev_info(priv->dev, "PTP not supported by HW\n");  in stmmac_hw_setup()
3435 		netdev_warn(priv->dev, "PTP init failed\n");  in stmmac_hw_setup()
3437 		stmmac_ptp_register(priv);  in stmmac_hw_setup()
3439 	priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;  in stmmac_hw_setup()
3442 	if (!priv->tx_lpi_timer)  in stmmac_hw_setup()
3443 		priv->tx_lpi_timer = eee_timer * 1000;  in stmmac_hw_setup()
3445 	if (priv->use_riwt) {  in stmmac_hw_setup()
3449 			if (!priv->rx_riwt[queue])  in stmmac_hw_setup()
3450 				priv->rx_riwt[queue] = DEF_DMA_RIWT;  in stmmac_hw_setup()
3452 			stmmac_rx_watchdog(priv, priv->ioaddr,  in stmmac_hw_setup()
3453 					   priv->rx_riwt[queue], queue);  in stmmac_hw_setup()
3457 	if (priv->hw->pcs)  in stmmac_hw_setup()
3458 		stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);  in stmmac_hw_setup()
3461 	stmmac_set_rings_length(priv);  in stmmac_hw_setup()
3464 	if (priv->tso) {  in stmmac_hw_setup()
3466 			struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_hw_setup()
3472 			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);  in stmmac_hw_setup()
3477 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;  in stmmac_hw_setup()
3479 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);  in stmmac_hw_setup()
3483 	if (priv->dma_cap.vlins)  in stmmac_hw_setup()
3484 		stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);  in stmmac_hw_setup()
3488 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_hw_setup()
3491 		stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);  in stmmac_hw_setup()
3495 	netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use);  in stmmac_hw_setup()
3496 	netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use);  in stmmac_hw_setup()
3499 	stmmac_start_all_dma(priv);  in stmmac_hw_setup()
3501 	stmmac_set_hw_vlan_mode(priv, priv->hw);  in stmmac_hw_setup()
3508 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hw_teardown()  local
3510 	clk_disable_unprepare(priv->plat->clk_ptp_ref);  in stmmac_hw_teardown()
3516 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_free_irq()  local
3521 		irq_idx = priv->plat->tx_queues_to_use;  in stmmac_free_irq()
3525 			if (priv->tx_irq[j] > 0) {  in stmmac_free_irq()
3526 				irq_set_affinity_hint(priv->tx_irq[j], NULL);  in stmmac_free_irq()
3527 				free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]);  in stmmac_free_irq()
3530 		irq_idx = priv->plat->rx_queues_to_use;  in stmmac_free_irq()
3534 			if (priv->rx_irq[j] > 0) {  in stmmac_free_irq()
3535 				irq_set_affinity_hint(priv->rx_irq[j], NULL);  in stmmac_free_irq()
3536 				free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]);  in stmmac_free_irq()
3540 		if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq)  in stmmac_free_irq()
3541 			free_irq(priv->sfty_ue_irq, dev);  in stmmac_free_irq()
3544 		if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq)  in stmmac_free_irq()
3545 			free_irq(priv->sfty_ce_irq, dev);  in stmmac_free_irq()
3548 		if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq)  in stmmac_free_irq()
3549 			free_irq(priv->lpi_irq, dev);  in stmmac_free_irq()
3552 		if (priv->wol_irq > 0 && priv->wol_irq != dev->irq)  in stmmac_free_irq()
3553 			free_irq(priv->wol_irq, dev);  in stmmac_free_irq()
3556 		if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq)  in stmmac_free_irq()
3557 			free_irq(priv->sfty_irq, dev);  in stmmac_free_irq()
3571 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq_multi_msi()  local
3580 	int_name = priv->int_name_mac;  in stmmac_request_irq_multi_msi()
3585 		netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3595 	priv->wol_irq_disabled = true;  in stmmac_request_irq_multi_msi()
3596 	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3597 		int_name = priv->int_name_wol;  in stmmac_request_irq_multi_msi()
3599 		ret = request_irq(priv->wol_irq,  in stmmac_request_irq_multi_msi()
3603 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3605 				   __func__, priv->wol_irq, ret);  in stmmac_request_irq_multi_msi()
3614 	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3615 		int_name = priv->int_name_lpi;  in stmmac_request_irq_multi_msi()
3617 		ret = request_irq(priv->lpi_irq,  in stmmac_request_irq_multi_msi()
3621 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3623 				   __func__, priv->lpi_irq, ret);  in stmmac_request_irq_multi_msi()
3632 	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3633 		int_name = priv->int_name_sfty;  in stmmac_request_irq_multi_msi()
3635 		ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt,  in stmmac_request_irq_multi_msi()
3638 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3640 				   __func__, priv->sfty_irq, ret);  in stmmac_request_irq_multi_msi()
3649 	if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3650 		int_name = priv->int_name_sfty_ce;  in stmmac_request_irq_multi_msi()
3652 		ret = request_irq(priv->sfty_ce_irq,  in stmmac_request_irq_multi_msi()
3656 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3658 				   __func__, priv->sfty_ce_irq, ret);  in stmmac_request_irq_multi_msi()
3667 	if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) {  in stmmac_request_irq_multi_msi()
3668 		int_name = priv->int_name_sfty_ue;  in stmmac_request_irq_multi_msi()
3670 		ret = request_irq(priv->sfty_ue_irq,  in stmmac_request_irq_multi_msi()
3674 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3676 				   __func__, priv->sfty_ue_irq, ret);  in stmmac_request_irq_multi_msi()
3683 	for (i = 0; i < priv->plat->rx_queues_to_use; i++) {  in stmmac_request_irq_multi_msi()
3686 		if (priv->rx_irq[i] == 0)  in stmmac_request_irq_multi_msi()
3689 		int_name = priv->int_name_rx_irq[i];  in stmmac_request_irq_multi_msi()
3691 		ret = request_irq(priv->rx_irq[i],  in stmmac_request_irq_multi_msi()
3693 				  0, int_name, &priv->dma_conf.rx_queue[i]);  in stmmac_request_irq_multi_msi()
3695 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3697 				   __func__, i, priv->rx_irq[i], ret);  in stmmac_request_irq_multi_msi()
3704 		irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask);  in stmmac_request_irq_multi_msi()
3708 	for (i = 0; i < priv->plat->tx_queues_to_use; i++) {  in stmmac_request_irq_multi_msi()
3711 		if (priv->tx_irq[i] == 0)  in stmmac_request_irq_multi_msi()
3714 		int_name = priv->int_name_tx_irq[i];  in stmmac_request_irq_multi_msi()
3716 		ret = request_irq(priv->tx_irq[i],  in stmmac_request_irq_multi_msi()
3718 				  0, int_name, &priv->dma_conf.tx_queue[i]);  in stmmac_request_irq_multi_msi()
3720 			netdev_err(priv->dev,  in stmmac_request_irq_multi_msi()
3722 				   __func__, i, priv->tx_irq[i], ret);  in stmmac_request_irq_multi_msi()
3729 		irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask);  in stmmac_request_irq_multi_msi()
3741 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq_single()  local
3748 		netdev_err(priv->dev,  in stmmac_request_irq_single()
3758 	priv->wol_irq_disabled = true;  in stmmac_request_irq_single()
3759 	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {  in stmmac_request_irq_single()
3760 		ret = request_irq(priv->wol_irq, stmmac_interrupt,  in stmmac_request_irq_single()
3763 			netdev_err(priv->dev,  in stmmac_request_irq_single()
3765 				   __func__, priv->wol_irq, ret);  in stmmac_request_irq_single()
3772 	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {  in stmmac_request_irq_single()
3773 		ret = request_irq(priv->lpi_irq, stmmac_interrupt,  in stmmac_request_irq_single()
3776 			netdev_err(priv->dev,  in stmmac_request_irq_single()
3778 				   __func__, priv->lpi_irq, ret);  in stmmac_request_irq_single()
3787 	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {  in stmmac_request_irq_single()
3788 		ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt,  in stmmac_request_irq_single()
3791 			netdev_err(priv->dev,  in stmmac_request_irq_single()
3793 				   __func__, priv->sfty_irq, ret);  in stmmac_request_irq_single()
3808 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq()  local
3812 	if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN)  in stmmac_request_irq()
3830 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)  in stmmac_setup_dma_desc()  argument
3837 		netdev_err(priv->dev, "%s: DMA conf allocation failed\n",  in stmmac_setup_dma_desc()
3842 	bfsize = stmmac_set_16kib_bfsize(priv, mtu);  in stmmac_setup_dma_desc()
3853 	dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size;  in stmmac_setup_dma_desc()
3854 	dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size;  in stmmac_setup_dma_desc()
3862 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {  in stmmac_setup_dma_desc()
3864 		int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;  in stmmac_setup_dma_desc()
3870 	ret = alloc_dma_desc_resources(priv, dma_conf);  in stmmac_setup_dma_desc()
3872 		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",  in stmmac_setup_dma_desc()
3877 	ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL);  in stmmac_setup_dma_desc()
3879 		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",  in stmmac_setup_dma_desc()
3887 	free_dma_desc_resources(priv, dma_conf);  in stmmac_setup_dma_desc()
3906 	struct stmmac_priv *priv = netdev_priv(dev);  in __stmmac_open()  local
3907 	int mode = priv->plat->phy_interface;  in __stmmac_open()
3911 	ret = pm_runtime_resume_and_get(priv->device);  in __stmmac_open()
3915 	if ((!priv->hw->xpcs ||  in __stmmac_open()
3916 	     xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) {  in __stmmac_open()
3919 			netdev_err(priv->dev,  in __stmmac_open()
3926 	priv->rx_copybreak = STMMAC_RX_COPYBREAK;  in __stmmac_open()
3930 		if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN)  in __stmmac_open()
3931 			dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs;  in __stmmac_open()
3932 	memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf));  in __stmmac_open()
3934 	stmmac_reset_queues_param(priv);  in __stmmac_open()
3936 	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&  in __stmmac_open()
3937 	    priv->plat->serdes_powerup) {  in __stmmac_open()
3938 		ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv);  in __stmmac_open()
3940 			netdev_err(priv->dev, "%s: Serdes powerup failed\n",  in __stmmac_open()
3948 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);  in __stmmac_open()
3952 	stmmac_init_coalesce(priv);  in __stmmac_open()
3954 	phylink_start(priv->phylink);  in __stmmac_open()
3956 	phylink_speed_up(priv->phylink);  in __stmmac_open()
3962 	stmmac_enable_all_queues(priv);  in __stmmac_open()
3963 	netif_tx_start_all_queues(priv->dev);  in __stmmac_open()
3964 	stmmac_enable_all_dma_irq(priv);  in __stmmac_open()
3969 	phylink_stop(priv->phylink);  in __stmmac_open()
3971 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in __stmmac_open()
3972 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in __stmmac_open()
3976 	phylink_disconnect_phy(priv->phylink);  in __stmmac_open()
3978 	pm_runtime_put(priv->device);  in __stmmac_open()
3984 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_open()  local
3988 	dma_conf = stmmac_setup_dma_desc(priv, dev->mtu);  in stmmac_open()
3994 		free_dma_desc_resources(priv, dma_conf);  in stmmac_open()
4008 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_release()  local
4011 	if (device_may_wakeup(priv->device))  in stmmac_release()
4012 		phylink_speed_down(priv->phylink, false);  in stmmac_release()
4014 	phylink_stop(priv->phylink);  in stmmac_release()
4015 	phylink_disconnect_phy(priv->phylink);  in stmmac_release()
4017 	stmmac_disable_all_queues(priv);  in stmmac_release()
4019 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_release()
4020 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_release()
4027 	if (priv->eee_enabled) {  in stmmac_release()
4028 		priv->tx_path_in_lpi_mode = false;  in stmmac_release()
4029 		del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_release()
4033 	stmmac_stop_all_dma(priv);  in stmmac_release()
4036 	free_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_release()
4039 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_release()
4042 	if (priv->plat->serdes_powerdown)  in stmmac_release()
4043 		priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv);  in stmmac_release()
4045 	stmmac_release_ptp(priv);  in stmmac_release()
4047 	if (stmmac_fpe_supported(priv))  in stmmac_release()
4048 		timer_shutdown_sync(&priv->fpe_cfg.verify_timer);  in stmmac_release()
4050 	pm_runtime_put(priv->device);  in stmmac_release()
4055 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,  in stmmac_vlan_insert()  argument
4062 	if (!priv->dma_cap.vlins)  in stmmac_vlan_insert()
4078 	if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))  in stmmac_vlan_insert()
4081 	stmmac_set_tx_owner(priv, p);  in stmmac_vlan_insert()
4082 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);  in stmmac_vlan_insert()
4097 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,  in stmmac_tso_allocator()  argument
4100 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tso_allocator()
4111 						priv->dma_conf.dma_tx_size);  in stmmac_tso_allocator()
4120 		if (priv->dma_cap.addr64 <= 32)  in stmmac_tso_allocator()
4123 			stmmac_set_desc_addr(priv, desc, curr_addr);  in stmmac_tso_allocator()
4128 		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,  in stmmac_tso_allocator()
4137 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)  in stmmac_flush_tx_descriptors()  argument
4139 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_flush_tx_descriptors()
4142 	if (likely(priv->extend_desc))  in stmmac_flush_tx_descriptors()
4156 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);  in stmmac_flush_tx_descriptors()
4189 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_tso_xmit()  local
4208 			priv->xstats.tx_dropped++;  in stmmac_tso_xmit()
4216 	tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_tso_xmit()
4217 	txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_tso_xmit()
4230 	if (unlikely(stmmac_tx_avail(priv, queue) <  in stmmac_tso_xmit()
4233 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,  in stmmac_tso_xmit()
4236 			netdev_err(priv->dev,  in stmmac_tso_xmit()
4254 		stmmac_set_mss(priv, mss_desc, mss);  in stmmac_tso_xmit()
4257 						priv->dma_conf.dma_tx_size);  in stmmac_tso_xmit()
4261 	if (netif_msg_tx_queued(priv)) {  in stmmac_tso_xmit()
4278 	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),  in stmmac_tso_xmit()
4280 	if (dma_mapping_error(priv->device, des))  in stmmac_tso_xmit()
4283 	if (priv->dma_cap.addr64 <= 32) {  in stmmac_tso_xmit()
4294 		stmmac_set_desc_addr(priv, first, des);  in stmmac_tso_xmit()
4300 	stmmac_tso_allocator(priv, tso_des, tmp_pay_len, (nfrags == 0), queue);  in stmmac_tso_xmit()
4323 		des = skb_frag_dma_map(priv->device, frag, 0,  in stmmac_tso_xmit()
4326 		if (dma_mapping_error(priv->device, des))  in stmmac_tso_xmit()
4329 		stmmac_tso_allocator(priv, des, skb_frag_size(frag),  in stmmac_tso_xmit()
4348 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)  in stmmac_tso_xmit()
4350 	else if (!priv->tx_coal_frames[queue])  in stmmac_tso_xmit()
4352 	else if (tx_packets > priv->tx_coal_frames[queue])  in stmmac_tso_xmit()
4355 		  priv->tx_coal_frames[queue]) < tx_packets)  in stmmac_tso_xmit()
4367 		stmmac_set_tx_ic(priv, desc);  in stmmac_tso_xmit()
4375 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);  in stmmac_tso_xmit()
4377 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {  in stmmac_tso_xmit()
4378 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",  in stmmac_tso_xmit()
4380 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_tso_xmit()
4391 	if (priv->sarc_type)  in stmmac_tso_xmit()
4392 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);  in stmmac_tso_xmit()
4397 		     priv->hwts_tx_en)) {  in stmmac_tso_xmit()
4400 		stmmac_enable_tx_timestamp(priv, first);  in stmmac_tso_xmit()
4404 	stmmac_prepare_tso_tx_desc(priv, first, 1,  in stmmac_tso_xmit()
4418 		stmmac_set_tx_owner(priv, mss_desc);  in stmmac_tso_xmit()
4421 	if (netif_msg_pktdata(priv)) {  in stmmac_tso_xmit()
4431 	stmmac_flush_tx_descriptors(priv, queue);  in stmmac_tso_xmit()
4432 	stmmac_tx_timer_arm(priv, queue);  in stmmac_tso_xmit()
4437 	dev_err(priv->device, "Tx dma map failed\n");  in stmmac_tso_xmit()
4439 	priv->xstats.tx_dropped++;  in stmmac_tso_xmit()
4476 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xmit()  local
4490 	tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xmit()
4491 	txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_xmit()
4494 	if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en)  in stmmac_xmit()
4495 		stmmac_disable_eee_mode(priv);  in stmmac_xmit()
4498 	if (skb_is_gso(skb) && priv->tso) {  in stmmac_xmit()
4501 		if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))  in stmmac_xmit()
4505 	if (priv->est && priv->est->enable &&  in stmmac_xmit()
4506 	    priv->est->max_sdu[queue] &&  in stmmac_xmit()
4507 	    skb->len > priv->est->max_sdu[queue]){  in stmmac_xmit()
4508 		priv->xstats.max_sdu_txq_drop[queue]++;  in stmmac_xmit()
4512 	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {  in stmmac_xmit()
4514 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,  in stmmac_xmit()
4517 			netdev_err(priv->dev,  in stmmac_xmit()
4525 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);  in stmmac_xmit()
4540 	    (priv->plat->tx_queues_cfg[queue].coe_unsupported ||  in stmmac_xmit()
4547 	if (likely(priv->extend_desc))  in stmmac_xmit()
4557 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);  in stmmac_xmit()
4559 	enh_desc = priv->plat->enh_desc;  in stmmac_xmit()
4562 		is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);  in stmmac_xmit()
4565 		entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);  in stmmac_xmit()
4575 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_xmit()
4578 		if (likely(priv->extend_desc))  in stmmac_xmit()
4585 		des = skb_frag_dma_map(priv->device, frag, 0, len,  in stmmac_xmit()
4587 		if (dma_mapping_error(priv->device, des))  in stmmac_xmit()
4592 		stmmac_set_desc_addr(priv, desc, des);  in stmmac_xmit()
4600 		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,  in stmmac_xmit()
4601 				priv->mode, 1, last_segment, skb->len);  in stmmac_xmit()
4616 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)  in stmmac_xmit()
4618 	else if (!priv->tx_coal_frames[queue])  in stmmac_xmit()
4620 	else if (tx_packets > priv->tx_coal_frames[queue])  in stmmac_xmit()
4623 		  priv->tx_coal_frames[queue]) < tx_packets)  in stmmac_xmit()
4629 		if (likely(priv->extend_desc))  in stmmac_xmit()
4637 		stmmac_set_tx_ic(priv, desc);  in stmmac_xmit()
4645 	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_xmit()
4648 	if (netif_msg_pktdata(priv)) {  in stmmac_xmit()
4649 		netdev_dbg(priv->dev,  in stmmac_xmit()
4654 		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");  in stmmac_xmit()
4658 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {  in stmmac_xmit()
4659 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",  in stmmac_xmit()
4661 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_xmit()
4670 	if (priv->sarc_type)  in stmmac_xmit()
4671 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);  in stmmac_xmit()
4682 		des = dma_map_single(priv->device, skb->data,  in stmmac_xmit()
4684 		if (dma_mapping_error(priv->device, des))  in stmmac_xmit()
4691 		stmmac_set_desc_addr(priv, first, des);  in stmmac_xmit()
4697 			     priv->hwts_tx_en)) {  in stmmac_xmit()
4700 			stmmac_enable_tx_timestamp(priv, first);  in stmmac_xmit()
4704 		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,  in stmmac_xmit()
4705 				csum_insertion, priv->mode, 0, last_segment,  in stmmac_xmit()
4713 		stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);  in stmmac_xmit()
4716 	stmmac_set_tx_owner(priv, first);  in stmmac_xmit()
4720 	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);  in stmmac_xmit()
4722 	stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xmit()
4723 	stmmac_tx_timer_arm(priv, queue);  in stmmac_xmit()
4728 	netdev_err(priv->dev, "Tx DMA map failed\n");  in stmmac_xmit()
4731 	priv->xstats.tx_dropped++;  in stmmac_xmit()
4760 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)  in stmmac_rx_refill()  argument
4762 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_refill()
4763 	int dirty = stmmac_rx_dirty(priv, queue);  in stmmac_rx_refill()
4767 	if (priv->dma_cap.host_dma_width <= 32)  in stmmac_rx_refill()
4775 		if (priv->extend_desc)  in stmmac_rx_refill()
4786 		if (priv->sph && !buf->sec_page) {  in stmmac_rx_refill()
4796 		stmmac_set_desc_addr(priv, p, buf->addr);  in stmmac_rx_refill()
4797 		if (priv->sph)  in stmmac_rx_refill()
4798 			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);  in stmmac_rx_refill()
4800 			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);  in stmmac_rx_refill()
4801 		stmmac_refill_desc3(priv, rx_q, p);  in stmmac_rx_refill()
4804 		rx_q->rx_count_frames += priv->rx_coal_frames[queue];  in stmmac_rx_refill()
4805 		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])  in stmmac_rx_refill()
4808 		use_rx_wd = !priv->rx_coal_frames[queue];  in stmmac_rx_refill()
4810 		if (!priv->use_riwt)  in stmmac_rx_refill()
4814 		stmmac_set_rx_owner(priv, p, use_rx_wd);  in stmmac_rx_refill()
4816 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);  in stmmac_rx_refill()
4821 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);  in stmmac_rx_refill()
4824 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,  in stmmac_rx_buf1_len()  argument
4829 	int coe = priv->hw->rx_csum;  in stmmac_rx_buf1_len()
4832 	if (priv->sph && len)  in stmmac_rx_buf1_len()
4836 	stmmac_get_rx_header_len(priv, p, &hlen);  in stmmac_rx_buf1_len()
4837 	if (priv->sph && hlen) {  in stmmac_rx_buf1_len()
4838 		priv->xstats.rx_split_hdr_pkt_n++;  in stmmac_rx_buf1_len()
4844 		return priv->dma_conf.dma_buf_sz;  in stmmac_rx_buf1_len()
4846 	plen = stmmac_get_rx_frame_len(priv, p, coe);  in stmmac_rx_buf1_len()
4849 	return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen);  in stmmac_rx_buf1_len()
4852 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,  in stmmac_rx_buf2_len()  argument
4856 	int coe = priv->hw->rx_csum;  in stmmac_rx_buf2_len()
4860 	if (!priv->sph)  in stmmac_rx_buf2_len()
4865 		return priv->dma_conf.dma_buf_sz;  in stmmac_rx_buf2_len()
4867 	plen = stmmac_get_rx_frame_len(priv, p, coe);  in stmmac_rx_buf2_len()
4873 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,  in stmmac_xdp_xmit_xdpf()  argument
4876 	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];  in stmmac_xdp_xmit_xdpf()
4877 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xdp_xmit_xdpf()
4883 	if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv))  in stmmac_xdp_xmit_xdpf()
4886 	if (priv->est && priv->est->enable &&  in stmmac_xdp_xmit_xdpf()
4887 	    priv->est->max_sdu[queue] &&  in stmmac_xdp_xmit_xdpf()
4888 	    xdpf->len > priv->est->max_sdu[queue]) {  in stmmac_xdp_xmit_xdpf()
4889 		priv->xstats.max_sdu_txq_drop[queue]++;  in stmmac_xdp_xmit_xdpf()
4893 	if (likely(priv->extend_desc))  in stmmac_xdp_xmit_xdpf()
4901 		dma_addr = dma_map_single(priv->device, xdpf->data,  in stmmac_xdp_xmit_xdpf()
4903 		if (dma_mapping_error(priv->device, dma_addr))  in stmmac_xdp_xmit_xdpf()
4912 		dma_sync_single_for_device(priv->device, dma_addr,  in stmmac_xdp_xmit_xdpf()
4926 	stmmac_set_desc_addr(priv, tx_desc, dma_addr);  in stmmac_xdp_xmit_xdpf()
4928 	stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len,  in stmmac_xdp_xmit_xdpf()
4929 			       true, priv->mode, true, true,  in stmmac_xdp_xmit_xdpf()
4934 	if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)  in stmmac_xdp_xmit_xdpf()
4941 		stmmac_set_tx_ic(priv, tx_desc);  in stmmac_xdp_xmit_xdpf()
4947 	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);  in stmmac_xdp_xmit_xdpf()
4949 	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);  in stmmac_xdp_xmit_xdpf()
4955 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv,  in stmmac_xdp_get_tx_queue()  argument
4963 	while (index >= priv->plat->tx_queues_to_use)  in stmmac_xdp_get_tx_queue()
4964 		index -= priv->plat->tx_queues_to_use;  in stmmac_xdp_get_tx_queue()
4969 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv,  in stmmac_xdp_xmit_back()  argument
4981 	queue = stmmac_xdp_get_tx_queue(priv, cpu);  in stmmac_xdp_xmit_back()
4982 	nq = netdev_get_tx_queue(priv->dev, queue);  in stmmac_xdp_xmit_back()
4988 	res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false);  in stmmac_xdp_xmit_back()
4990 		stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xdp_xmit_back()
4997 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,  in __stmmac_xdp_run_prog()  argument
5010 		res = stmmac_xdp_xmit_back(priv, xdp);  in __stmmac_xdp_run_prog()
5013 		if (xdp_do_redirect(priv->dev, xdp, prog) < 0)  in __stmmac_xdp_run_prog()
5019 		bpf_warn_invalid_xdp_action(priv->dev, prog, act);  in __stmmac_xdp_run_prog()
5022 		trace_xdp_exception(priv->dev, prog, act);  in __stmmac_xdp_run_prog()
5032 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv,  in stmmac_xdp_run_prog()  argument
5038 	prog = READ_ONCE(priv->xdp_prog);  in stmmac_xdp_run_prog()
5044 	res = __stmmac_xdp_run_prog(priv, prog, xdp);  in stmmac_xdp_run_prog()
5049 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv,  in stmmac_finalize_xdp_rx()  argument
5055 	queue = stmmac_xdp_get_tx_queue(priv, cpu);  in stmmac_finalize_xdp_rx()
5058 		stmmac_tx_timer_arm(priv, queue);  in stmmac_finalize_xdp_rx()
5084 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,  in stmmac_dispatch_skb_zc()  argument
5088 	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];  in stmmac_dispatch_skb_zc()
5089 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_dispatch_skb_zc()
5092 	int coe = priv->hw->rx_csum;  in stmmac_dispatch_skb_zc()
5098 		priv->xstats.rx_dropped++;  in stmmac_dispatch_skb_zc()
5102 	stmmac_get_rx_hwtstamp(priv, p, np, skb);  in stmmac_dispatch_skb_zc()
5103 	if (priv->hw->hw_vlan_en)  in stmmac_dispatch_skb_zc()
5105 		stmmac_rx_hw_vlan(priv, priv->hw, p, skb);  in stmmac_dispatch_skb_zc()
5108 		stmmac_rx_vlan(priv->dev, skb);  in stmmac_dispatch_skb_zc()
5109 	skb->protocol = eth_type_trans(skb, priv->dev);  in stmmac_dispatch_skb_zc()
5116 	if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))  in stmmac_dispatch_skb_zc()
5128 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)  in stmmac_rx_refill_zc()  argument
5130 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_refill_zc()
5135 	budget = min(budget, stmmac_rx_dirty(priv, queue));  in stmmac_rx_refill_zc()
5150 		if (priv->extend_desc)  in stmmac_rx_refill_zc()
5156 		stmmac_set_desc_addr(priv, rx_desc, dma_addr);  in stmmac_rx_refill_zc()
5157 		stmmac_set_desc_sec_addr(priv, rx_desc, 0, false);  in stmmac_rx_refill_zc()
5158 		stmmac_refill_desc3(priv, rx_q, rx_desc);  in stmmac_rx_refill_zc()
5161 		rx_q->rx_count_frames += priv->rx_coal_frames[queue];  in stmmac_rx_refill_zc()
5162 		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])  in stmmac_rx_refill_zc()
5165 		use_rx_wd = !priv->rx_coal_frames[queue];  in stmmac_rx_refill_zc()
5167 		if (!priv->use_riwt)  in stmmac_rx_refill_zc()
5171 		stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);  in stmmac_rx_refill_zc()
5173 		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);  in stmmac_rx_refill_zc()
5180 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);  in stmmac_rx_refill_zc()
5196 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)  in stmmac_rx_zc()  argument
5198 	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];  in stmmac_rx_zc()
5199 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx_zc()
5201 	int dirty = stmmac_rx_dirty(priv, queue);  in stmmac_rx_zc()
5210 	if (netif_msg_rx_status(priv)) {  in stmmac_rx_zc()
5213 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);  in stmmac_rx_zc()
5214 		if (priv->extend_desc) {  in stmmac_rx_zc()
5222 		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,  in stmmac_rx_zc()
5252 				  !stmmac_rx_refill_zc(priv, queue, dirty);  in stmmac_rx_zc()
5256 		if (priv->extend_desc)  in stmmac_rx_zc()
5262 		status = stmmac_rx_status(priv, &priv->xstats, p);  in stmmac_rx_zc()
5269 						priv->dma_conf.dma_rx_size);  in stmmac_rx_zc()
5272 		if (priv->extend_desc)  in stmmac_rx_zc()
5283 		if (priv->extend_desc)  in stmmac_rx_zc()
5284 			stmmac_rx_extended_status(priv, &priv->xstats,  in stmmac_rx_zc()
5291 			if (!priv->hwts_rx_en)  in stmmac_rx_zc()
5312 		ctx->priv = priv;  in stmmac_rx_zc()
5317 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);  in stmmac_rx_zc()
5330 		prog = READ_ONCE(priv->xdp_prog);  in stmmac_rx_zc()
5331 		res = __stmmac_xdp_run_prog(priv, prog, buf->xdp);  in stmmac_rx_zc()
5335 			stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp);  in stmmac_rx_zc()
5359 	stmmac_finalize_xdp_rx(priv, xdp_status);  in stmmac_rx_zc()
5365 	priv->xstats.rx_dropped += rx_dropped;  in stmmac_rx_zc()
5366 	priv->xstats.rx_errors += rx_errors;  in stmmac_rx_zc()
5369 		if (failure || stmmac_rx_dirty(priv, queue) > 0)  in stmmac_rx_zc()
5388 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)  in stmmac_rx()  argument
5391 	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];  in stmmac_rx()
5392 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rx()
5393 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_rx()
5395 	int status = 0, coe = priv->hw->rx_csum;  in stmmac_rx()
5405 	buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;  in stmmac_rx()
5406 	limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit);  in stmmac_rx()
5408 	if (netif_msg_rx_status(priv)) {  in stmmac_rx()
5411 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);  in stmmac_rx()
5412 		if (priv->extend_desc) {  in stmmac_rx()
5420 		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,  in stmmac_rx()
5451 		if (priv->extend_desc)  in stmmac_rx()
5457 		status = stmmac_rx_status(priv, &priv->xstats, p);  in stmmac_rx()
5463 						priv->dma_conf.dma_rx_size);  in stmmac_rx()
5466 		if (priv->extend_desc)  in stmmac_rx()
5473 		if (priv->extend_desc)  in stmmac_rx()
5474 			stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry);  in stmmac_rx()
5479 			if (!priv->hwts_rx_en)  in stmmac_rx()
5498 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);  in stmmac_rx()
5500 		buf2_len = stmmac_rx_buf2_len(priv, p, status, len);  in stmmac_rx()
5517 			dma_sync_single_for_cpu(priv->device, buf->addr,  in stmmac_rx()
5527 			ctx.priv = priv;  in stmmac_rx()
5531 			skb = stmmac_xdp_run_prog(priv, &ctx.xdp);  in stmmac_rx()
5590 			dma_sync_single_for_cpu(priv->device, buf->addr,  in stmmac_rx()
5594 					priv->dma_conf.dma_buf_sz);  in stmmac_rx()
5602 			dma_sync_single_for_cpu(priv->device, buf->sec_addr,  in stmmac_rx()
5606 					priv->dma_conf.dma_buf_sz);  in stmmac_rx()
5621 		stmmac_get_rx_hwtstamp(priv, p, np, skb);  in stmmac_rx()
5623 		if (priv->hw->hw_vlan_en)  in stmmac_rx()
5625 			stmmac_rx_hw_vlan(priv, priv->hw, p, skb);  in stmmac_rx()
5628 			stmmac_rx_vlan(priv->dev, skb);  in stmmac_rx()
5630 		skb->protocol = eth_type_trans(skb, priv->dev);  in stmmac_rx()
5637 		if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))  in stmmac_rx()
5656 	stmmac_finalize_xdp_rx(priv, xdp_status);  in stmmac_rx()
5658 	stmmac_rx_refill(priv, queue);  in stmmac_rx()
5666 	priv->xstats.rx_dropped += rx_dropped;  in stmmac_rx()
5667 	priv->xstats.rx_errors += rx_errors;  in stmmac_rx()
5676 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_rx()  local
5681 	rxq_stats = &priv->xstats.rxq_stats[chan];  in stmmac_napi_poll_rx()
5686 	work_done = stmmac_rx(priv, budget, chan);  in stmmac_napi_poll_rx()
5691 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);  in stmmac_napi_poll_rx()
5702 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_tx()  local
5708 	txq_stats = &priv->xstats.txq_stats[chan];  in stmmac_napi_poll_tx()
5713 	work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets);  in stmmac_napi_poll_tx()
5720 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);  in stmmac_napi_poll_tx()
5726 		stmmac_tx_timer_arm(priv, chan);  in stmmac_napi_poll_tx()
5735 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_rxtx()  local
5742 	rxq_stats = &priv->xstats.rxq_stats[chan];  in stmmac_napi_poll_rxtx()
5747 	txq_stats = &priv->xstats.txq_stats[chan];  in stmmac_napi_poll_rxtx()
5752 	tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets);  in stmmac_napi_poll_rxtx()
5755 	rx_done = stmmac_rx_zc(priv, budget, chan);  in stmmac_napi_poll_rxtx()
5773 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_napi_poll_rxtx()
5779 		stmmac_tx_timer_arm(priv, chan);  in stmmac_napi_poll_rxtx()
5795 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_tx_timeout()  local
5797 	stmmac_global_err(priv);  in stmmac_tx_timeout()
5811 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_set_rx_mode()  local
5813 	stmmac_set_filter(priv, priv->hw, dev);  in stmmac_set_rx_mode()
5829 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_change_mtu()  local
5830 	int txfifosz = priv->plat->tx_fifo_size;  in stmmac_change_mtu()
5836 		txfifosz = priv->dma_cap.tx_fifo_size;  in stmmac_change_mtu()
5838 	txfifosz /= priv->plat->tx_queues_to_use;  in stmmac_change_mtu()
5840 	if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) {  in stmmac_change_mtu()
5841 		netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n");  in stmmac_change_mtu()
5852 		netdev_dbg(priv->dev, "restarting interface to change its MTU\n");  in stmmac_change_mtu()
5854 		dma_conf = stmmac_setup_dma_desc(priv, mtu);  in stmmac_change_mtu()
5856 			netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n",  in stmmac_change_mtu()
5865 			free_dma_desc_resources(priv, dma_conf);  in stmmac_change_mtu()
5867 			netdev_err(priv->dev, "failed reopening the interface after MTU change\n");  in stmmac_change_mtu()
5885 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_fix_features()  local
5887 	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)  in stmmac_fix_features()
5890 	if (!priv->plat->tx_coe)  in stmmac_fix_features()
5898 	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))  in stmmac_fix_features()
5902 	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {  in stmmac_fix_features()
5904 			priv->tso = true;  in stmmac_fix_features()
5906 			priv->tso = false;  in stmmac_fix_features()
5915 	struct stmmac_priv *priv = netdev_priv(netdev);  in stmmac_set_features()  local
5919 		priv->hw->rx_csum = priv->plat->rx_coe;  in stmmac_set_features()
5921 		priv->hw->rx_csum = 0;  in stmmac_set_features()
5925 	stmmac_rx_ipc(priv, priv->hw);  in stmmac_set_features()
5927 	if (priv->sph_cap) {  in stmmac_set_features()
5928 		bool sph_en = (priv->hw->rx_csum > 0) && priv->sph;  in stmmac_set_features()
5931 		for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)  in stmmac_set_features()
5932 			stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);  in stmmac_set_features()
5936 		priv->hw->hw_vlan_en = true;  in stmmac_set_features()
5938 		priv->hw->hw_vlan_en = false;  in stmmac_set_features()
5940 	stmmac_set_hw_vlan_mode(priv, priv->hw);  in stmmac_set_features()
5945 static void stmmac_common_interrupt(struct stmmac_priv *priv)  in stmmac_common_interrupt()  argument
5947 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_common_interrupt()
5948 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_common_interrupt()
5953 	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;  in stmmac_common_interrupt()
5956 	if (priv->irq_wake)  in stmmac_common_interrupt()
5957 		pm_wakeup_event(priv->device, 0);  in stmmac_common_interrupt()
5959 	if (priv->dma_cap.estsel)  in stmmac_common_interrupt()
5960 		stmmac_est_irq_status(priv, priv, priv->dev,  in stmmac_common_interrupt()
5961 				      &priv->xstats, tx_cnt);  in stmmac_common_interrupt()
5963 	if (stmmac_fpe_supported(priv))  in stmmac_common_interrupt()
5964 		stmmac_fpe_irq_status(priv);  in stmmac_common_interrupt()
5967 	if ((priv->plat->has_gmac) || xmac) {  in stmmac_common_interrupt()
5968 		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);  in stmmac_common_interrupt()
5973 				priv->tx_path_in_lpi_mode = true;  in stmmac_common_interrupt()
5975 				priv->tx_path_in_lpi_mode = false;  in stmmac_common_interrupt()
5979 			stmmac_host_mtl_irq_status(priv, priv->hw, queue);  in stmmac_common_interrupt()
5982 		if (priv->hw->pcs &&  in stmmac_common_interrupt()
5983 		    !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) {  in stmmac_common_interrupt()
5984 			if (priv->xstats.pcs_link)  in stmmac_common_interrupt()
5985 				netif_carrier_on(priv->dev);  in stmmac_common_interrupt()
5987 				netif_carrier_off(priv->dev);  in stmmac_common_interrupt()
5990 		stmmac_timestamp_interrupt(priv, priv);  in stmmac_common_interrupt()
6008 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_interrupt()  local
6011 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_interrupt()
6015 	if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv))  in stmmac_interrupt()
6019 	stmmac_common_interrupt(priv);  in stmmac_interrupt()
6022 	stmmac_dma_interrupt(priv);  in stmmac_interrupt()
6030 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_mac_interrupt()  local
6033 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_mac_interrupt()
6037 	stmmac_common_interrupt(priv);  in stmmac_mac_interrupt()
6045 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_safety_interrupt()  local
6048 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_safety_interrupt()
6052 	stmmac_safety_feat_interrupt(priv);  in stmmac_safety_interrupt()
6062 	struct stmmac_priv *priv;  in stmmac_msi_intr_tx()  local
6066 	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);  in stmmac_msi_intr_tx()
6069 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_msi_intr_tx()
6072 	status = stmmac_napi_check(priv, chan, DMA_DIR_TX);  in stmmac_msi_intr_tx()
6076 		stmmac_bump_dma_threshold(priv, chan);  in stmmac_msi_intr_tx()
6078 		stmmac_tx_err(priv, chan);  in stmmac_msi_intr_tx()
6089 	struct stmmac_priv *priv;  in stmmac_msi_intr_rx()  local
6092 	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);  in stmmac_msi_intr_rx()
6095 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_msi_intr_rx()
6098 	stmmac_napi_check(priv, chan, DMA_DIR_RX);  in stmmac_msi_intr_rx()
6114 	struct stmmac_priv *priv = netdev_priv (dev);  in stmmac_ioctl()  local
6124 		ret = phylink_mii_ioctl(priv->phylink, rq, cmd);  in stmmac_ioctl()
6142 	struct stmmac_priv *priv = cb_priv;  in stmmac_setup_tc_block_cb()  local
6145 	if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))  in stmmac_setup_tc_block_cb()
6148 	__stmmac_disable_all_queues(priv);  in stmmac_setup_tc_block_cb()
6152 		ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);  in stmmac_setup_tc_block_cb()
6155 		ret = stmmac_tc_setup_cls(priv, priv, type_data);  in stmmac_setup_tc_block_cb()
6161 	stmmac_enable_all_queues(priv);  in stmmac_setup_tc_block_cb()
6170 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_setup_tc()  local
6174 		return stmmac_tc_query_caps(priv, priv, type_data);  in stmmac_setup_tc()
6176 		return stmmac_tc_setup_mqprio(priv, priv, type_data);  in stmmac_setup_tc()
6181 						  priv, priv, true);  in stmmac_setup_tc()
6183 		return stmmac_tc_setup_cbs(priv, priv, type_data);  in stmmac_setup_tc()
6185 		return stmmac_tc_setup_taprio(priv, priv, type_data);  in stmmac_setup_tc()
6187 		return stmmac_tc_setup_etf(priv, priv, type_data);  in stmmac_setup_tc()
6213 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_set_mac_address()  local
6216 	ret = pm_runtime_resume_and_get(priv->device);  in stmmac_set_mac_address()
6224 	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);  in stmmac_set_mac_address()
6227 	pm_runtime_put(priv->device);  in stmmac_set_mac_address()
6261 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_rings_status_show()  local
6262 	u32 rx_count = priv->plat->rx_queues_to_use;  in stmmac_rings_status_show()
6263 	u32 tx_count = priv->plat->tx_queues_to_use;  in stmmac_rings_status_show()
6270 		struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_rings_status_show()
6274 		if (priv->extend_desc) {  in stmmac_rings_status_show()
6277 					   priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy);  in stmmac_rings_status_show()
6281 					   priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy);  in stmmac_rings_status_show()
6286 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_rings_status_show()
6290 		if (priv->extend_desc) {  in stmmac_rings_status_show()
6293 					   priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy);  in stmmac_rings_status_show()
6297 					   priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy);  in stmmac_rings_status_show()
6324 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_dma_cap_show()  local
6326 	if (!priv->hw_cap_support) {  in stmmac_dma_cap_show()
6336 		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");  in stmmac_dma_cap_show()
6338 		   (priv->dma_cap.mbps_1000) ? "Y" : "N");  in stmmac_dma_cap_show()
6340 		   (priv->dma_cap.half_duplex) ? "Y" : "N");  in stmmac_dma_cap_show()
6341 	if (priv->plat->has_xgmac) {  in stmmac_dma_cap_show()
6344 			   priv->dma_cap.multi_addr);  in stmmac_dma_cap_show()
6347 			   (priv->dma_cap.hash_filter) ? "Y" : "N");  in stmmac_dma_cap_show()
6349 			   (priv->dma_cap.multi_addr) ? "Y" : "N");  in stmmac_dma_cap_show()
6352 		   (priv->dma_cap.pcs) ? "Y" : "N");  in stmmac_dma_cap_show()
6354 		   (priv->dma_cap.sma_mdio) ? "Y" : "N");  in stmmac_dma_cap_show()
6356 		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");  in stmmac_dma_cap_show()
6358 		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");  in stmmac_dma_cap_show()
6360 		   (priv->dma_cap.rmon) ? "Y" : "N");  in stmmac_dma_cap_show()
6362 		   (priv->dma_cap.time_stamp) ? "Y" : "N");  in stmmac_dma_cap_show()
6364 		   (priv->dma_cap.atime_stamp) ? "Y" : "N");  in stmmac_dma_cap_show()
6365 	if (priv->plat->has_xgmac)  in stmmac_dma_cap_show()
6367 			   dwxgmac_timestamp_source[priv->dma_cap.tssrc]);  in stmmac_dma_cap_show()
6369 		   (priv->dma_cap.eee) ? "Y" : "N");  in stmmac_dma_cap_show()
6370 	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");  in stmmac_dma_cap_show()
6372 		   (priv->dma_cap.tx_coe) ? "Y" : "N");  in stmmac_dma_cap_show()
6373 	if (priv->synopsys_id >= DWMAC_CORE_4_00 ||  in stmmac_dma_cap_show()
6374 	    priv->plat->has_xgmac) {  in stmmac_dma_cap_show()
6376 			   (priv->dma_cap.rx_coe) ? "Y" : "N");  in stmmac_dma_cap_show()
6379 			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");  in stmmac_dma_cap_show()
6381 			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");  in stmmac_dma_cap_show()
6383 			   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");  in stmmac_dma_cap_show()
6386 		   priv->dma_cap.number_rx_channel);  in stmmac_dma_cap_show()
6388 		   priv->dma_cap.number_tx_channel);  in stmmac_dma_cap_show()
6390 		   priv->dma_cap.number_rx_queues);  in stmmac_dma_cap_show()
6392 		   priv->dma_cap.number_tx_queues);  in stmmac_dma_cap_show()
6394 		   (priv->dma_cap.enh_desc) ? "Y" : "N");  in stmmac_dma_cap_show()
6395 	seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);  in stmmac_dma_cap_show()
6396 	seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);  in stmmac_dma_cap_show()
6397 	seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ?  in stmmac_dma_cap_show()
6398 		   (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0);  in stmmac_dma_cap_show()
6399 	seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");  in stmmac_dma_cap_show()
6401 		   priv->dma_cap.pps_out_num);  in stmmac_dma_cap_show()
6403 		   dwxgmac_safety_feature_desc[priv->dma_cap.asp]);  in stmmac_dma_cap_show()
6405 		   priv->dma_cap.frpsel ? "Y" : "N");  in stmmac_dma_cap_show()
6407 		   priv->dma_cap.host_dma_width);  in stmmac_dma_cap_show()
6409 		   priv->dma_cap.rssen ? "Y" : "N");  in stmmac_dma_cap_show()
6411 		   priv->dma_cap.vlhash ? "Y" : "N");  in stmmac_dma_cap_show()
6413 		   priv->dma_cap.sphen ? "Y" : "N");  in stmmac_dma_cap_show()
6415 		   priv->dma_cap.vlins ? "Y" : "N");  in stmmac_dma_cap_show()
6417 		   priv->dma_cap.dvlan ? "Y" : "N");  in stmmac_dma_cap_show()
6419 		   priv->dma_cap.l3l4fnum);  in stmmac_dma_cap_show()
6421 		   priv->dma_cap.arpoffsel ? "Y" : "N");  in stmmac_dma_cap_show()
6423 		   priv->dma_cap.estsel ? "Y" : "N");  in stmmac_dma_cap_show()
6425 		   priv->dma_cap.fpesel ? "Y" : "N");  in stmmac_dma_cap_show()
6427 		   priv->dma_cap.tbssel ? "Y" : "N");  in stmmac_dma_cap_show()
6429 		   priv->dma_cap.tbs_ch_num);  in stmmac_dma_cap_show()
6431 		   priv->dma_cap.sgfsel ? "Y" : "N");  in stmmac_dma_cap_show()
6433 		   BIT(priv->dma_cap.ttsfd) >> 1);  in stmmac_dma_cap_show()
6435 		   priv->dma_cap.numtc);  in stmmac_dma_cap_show()
6437 		   priv->dma_cap.dcben ? "Y" : "N");  in stmmac_dma_cap_show()
6439 		   priv->dma_cap.advthword ? "Y" : "N");  in stmmac_dma_cap_show()
6441 		   priv->dma_cap.ptoen ? "Y" : "N");  in stmmac_dma_cap_show()
6443 		   priv->dma_cap.osten ? "Y" : "N");  in stmmac_dma_cap_show()
6445 		   priv->dma_cap.pfcen ? "Y" : "N");  in stmmac_dma_cap_show()
6447 		   BIT(priv->dma_cap.frpes) << 6);  in stmmac_dma_cap_show()
6449 		   BIT(priv->dma_cap.frpbs) << 6);  in stmmac_dma_cap_show()
6451 		   priv->dma_cap.frppipe_num);  in stmmac_dma_cap_show()
6453 		   priv->dma_cap.nrvf_num ?  in stmmac_dma_cap_show()
6454 		   (BIT(priv->dma_cap.nrvf_num) << 1) : 0);  in stmmac_dma_cap_show()
6456 		   priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0);  in stmmac_dma_cap_show()
6458 		   priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0);  in stmmac_dma_cap_show()
6460 		   priv->dma_cap.cbtisel ? "Y" : "N");  in stmmac_dma_cap_show()
6462 		   priv->dma_cap.aux_snapshot_n);  in stmmac_dma_cap_show()
6464 		   priv->dma_cap.pou_ost_en ? "Y" : "N");  in stmmac_dma_cap_show()
6466 		   priv->dma_cap.edma ? "Y" : "N");  in stmmac_dma_cap_show()
6468 		   priv->dma_cap.ediffc ? "Y" : "N");  in stmmac_dma_cap_show()
6470 		   priv->dma_cap.vxn ? "Y" : "N");  in stmmac_dma_cap_show()
6472 		   priv->dma_cap.dbgmem ? "Y" : "N");  in stmmac_dma_cap_show()
6474 		   priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0);  in stmmac_dma_cap_show()
6485 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_device_event()  local
6492 		if (priv->dbgfs_dir)  in stmmac_device_event()
6493 			priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,  in stmmac_device_event()
6494 							 priv->dbgfs_dir,  in stmmac_device_event()
6509 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_init_fs()  local
6514 	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);  in stmmac_init_fs()
6517 	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,  in stmmac_init_fs()
6521 	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,  in stmmac_init_fs()
6529 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_exit_fs()  local
6531 	debugfs_remove_recursive(priv->dbgfs_dir);  in stmmac_exit_fs()
6559 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)  in stmmac_vlan_update()  argument
6566 	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {  in stmmac_vlan_update()
6573 	if (!priv->dma_cap.vlhash) {  in stmmac_vlan_update()
6581 	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);  in stmmac_vlan_update()
6586 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_vlan_rx_add_vid()  local
6590 	ret = pm_runtime_resume_and_get(priv->device);  in stmmac_vlan_rx_add_vid()
6597 	set_bit(vid, priv->active_vlans);  in stmmac_vlan_rx_add_vid()
6598 	ret = stmmac_vlan_update(priv, is_double);  in stmmac_vlan_rx_add_vid()
6600 		clear_bit(vid, priv->active_vlans);  in stmmac_vlan_rx_add_vid()
6604 	if (priv->hw->num_vlan) {  in stmmac_vlan_rx_add_vid()
6605 		ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);  in stmmac_vlan_rx_add_vid()
6610 	pm_runtime_put(priv->device);  in stmmac_vlan_rx_add_vid()
6617 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_vlan_rx_kill_vid()  local
6621 	ret = pm_runtime_resume_and_get(priv->device);  in stmmac_vlan_rx_kill_vid()
6628 	clear_bit(vid, priv->active_vlans);  in stmmac_vlan_rx_kill_vid()
6630 	if (priv->hw->num_vlan) {  in stmmac_vlan_rx_kill_vid()
6631 		ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);  in stmmac_vlan_rx_kill_vid()
6636 	ret = stmmac_vlan_update(priv, is_double);  in stmmac_vlan_rx_kill_vid()
6639 	pm_runtime_put(priv->device);  in stmmac_vlan_rx_kill_vid()
6646 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_bpf()  local
6650 		return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack);  in stmmac_bpf()
6652 		return stmmac_xdp_setup_pool(priv, bpf->xsk.pool,  in stmmac_bpf()
6662 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_xmit()  local
6668 	if (unlikely(test_bit(STMMAC_DOWN, &priv->state)))  in stmmac_xdp_xmit()
6674 	queue = stmmac_xdp_get_tx_queue(priv, cpu);  in stmmac_xdp_xmit()
6675 	nq = netdev_get_tx_queue(priv->dev, queue);  in stmmac_xdp_xmit()
6684 		res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true);  in stmmac_xdp_xmit()
6692 		stmmac_flush_tx_descriptors(priv, queue);  in stmmac_xdp_xmit()
6693 		stmmac_tx_timer_arm(priv, queue);  in stmmac_xdp_xmit()
6701 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_disable_rx_queue()  argument
6703 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_disable_rx_queue()
6707 	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0);  in stmmac_disable_rx_queue()
6710 	stmmac_stop_rx_dma(priv, queue);  in stmmac_disable_rx_queue()
6711 	__free_dma_rx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_disable_rx_queue()
6714 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_enable_rx_queue()  argument
6716 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_enable_rx_queue()
6717 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_enable_rx_queue()
6722 	ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_rx_queue()
6724 		netdev_err(priv->dev, "Failed to alloc RX desc.\n");  in stmmac_enable_rx_queue()
6728 	ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL);  in stmmac_enable_rx_queue()
6730 		__free_dma_rx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_rx_queue()
6731 		netdev_err(priv->dev, "Failed to init RX desc.\n");  in stmmac_enable_rx_queue()
6735 	stmmac_reset_rx_queue(priv, queue);  in stmmac_enable_rx_queue()
6736 	stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue);  in stmmac_enable_rx_queue()
6738 	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_enable_rx_queue()
6743 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,  in stmmac_enable_rx_queue()
6748 		stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_enable_rx_queue()
6752 		stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_enable_rx_queue()
6753 				      priv->dma_conf.dma_buf_sz,  in stmmac_enable_rx_queue()
6757 	stmmac_start_rx_dma(priv, queue);  in stmmac_enable_rx_queue()
6760 	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0);  in stmmac_enable_rx_queue()
6764 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_disable_tx_queue()  argument
6766 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_disable_tx_queue()
6770 	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1);  in stmmac_disable_tx_queue()
6773 	stmmac_stop_tx_dma(priv, queue);  in stmmac_disable_tx_queue()
6774 	__free_dma_tx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_disable_tx_queue()
6777 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_enable_tx_queue()  argument
6779 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_enable_tx_queue()
6780 	struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_enable_tx_queue()
6784 	ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6786 		netdev_err(priv->dev, "Failed to alloc TX desc.\n");  in stmmac_enable_tx_queue()
6790 	ret = __init_dma_tx_desc_rings(priv,  &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6792 		__free_dma_tx_desc_resources(priv, &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6793 		netdev_err(priv->dev, "Failed to init TX desc.\n");  in stmmac_enable_tx_queue()
6797 	stmmac_reset_tx_queue(priv, queue);  in stmmac_enable_tx_queue()
6798 	stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue);  in stmmac_enable_tx_queue()
6800 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_enable_tx_queue()
6804 		stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index);  in stmmac_enable_tx_queue()
6807 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,  in stmmac_enable_tx_queue()
6810 	stmmac_start_tx_dma(priv, queue);  in stmmac_enable_tx_queue()
6813 	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1);  in stmmac_enable_tx_queue()
6819 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_release()  local
6826 	stmmac_disable_all_queues(priv);  in stmmac_xdp_release()
6828 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_xdp_release()
6829 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_xdp_release()
6835 	stmmac_stop_all_dma(priv);  in stmmac_xdp_release()
6838 	free_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_xdp_release()
6841 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_xdp_release()
6852 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_open()  local
6853 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_xdp_open()
6854 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_xdp_open()
6863 	ret = alloc_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_xdp_open()
6870 	ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL);  in stmmac_xdp_open()
6877 	stmmac_reset_queues_param(priv);  in stmmac_xdp_open()
6881 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);  in stmmac_xdp_open()
6882 		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);  in stmmac_xdp_open()
6886 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;  in stmmac_xdp_open()
6890 		rx_q = &priv->dma_conf.rx_queue[chan];  in stmmac_xdp_open()
6892 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_xdp_open()
6898 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,  in stmmac_xdp_open()
6903 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_xdp_open()
6907 			stmmac_set_dma_bfsize(priv, priv->ioaddr,  in stmmac_xdp_open()
6908 					      priv->dma_conf.dma_buf_sz,  in stmmac_xdp_open()
6912 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);  in stmmac_xdp_open()
6917 		tx_q = &priv->dma_conf.tx_queue[chan];  in stmmac_xdp_open()
6919 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,  in stmmac_xdp_open()
6923 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,  in stmmac_xdp_open()
6931 	stmmac_mac_set(priv, priv->ioaddr, true);  in stmmac_xdp_open()
6934 	stmmac_start_all_dma(priv);  in stmmac_xdp_open()
6941 	stmmac_enable_all_queues(priv);  in stmmac_xdp_open()
6944 	stmmac_enable_all_dma_irq(priv);  in stmmac_xdp_open()
6949 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_xdp_open()
6950 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_xdp_open()
6954 	free_dma_desc_resources(priv, &priv->dma_conf);  in stmmac_xdp_open()
6961 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xsk_wakeup()  local
6966 	if (test_bit(STMMAC_DOWN, &priv->state) ||  in stmmac_xsk_wakeup()
6967 	    !netif_carrier_ok(priv->dev))  in stmmac_xsk_wakeup()
6970 	if (!stmmac_xdp_is_enabled(priv))  in stmmac_xsk_wakeup()
6973 	if (queue >= priv->plat->rx_queues_to_use ||  in stmmac_xsk_wakeup()
6974 	    queue >= priv->plat->tx_queues_to_use)  in stmmac_xsk_wakeup()
6977 	rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_xsk_wakeup()
6978 	tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_xsk_wakeup()
6979 	ch = &priv->channel[queue];  in stmmac_xsk_wakeup()
6997 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_get_stats64()  local
6998 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_get_stats64()
6999 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_get_stats64()
7004 		struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q];  in stmmac_get_stats64()
7022 		struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q];  in stmmac_get_stats64()
7036 	stats->rx_dropped = priv->xstats.rx_dropped;  in stmmac_get_stats64()
7037 	stats->rx_errors = priv->xstats.rx_errors;  in stmmac_get_stats64()
7038 	stats->tx_dropped = priv->xstats.tx_dropped;  in stmmac_get_stats64()
7039 	stats->tx_errors = priv->xstats.tx_errors;  in stmmac_get_stats64()
7040 	stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier;  in stmmac_get_stats64()
7041 	stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision;  in stmmac_get_stats64()
7042 	stats->rx_length_errors = priv->xstats.rx_length;  in stmmac_get_stats64()
7043 	stats->rx_crc_errors = priv->xstats.rx_crc_errors;  in stmmac_get_stats64()
7044 	stats->rx_over_errors = priv->xstats.rx_overflow_cntr;  in stmmac_get_stats64()
7045 	stats->rx_missed_errors = priv->xstats.rx_missed_cntr;  in stmmac_get_stats64()
7069 static void stmmac_reset_subtask(struct stmmac_priv *priv)  in stmmac_reset_subtask()  argument
7071 	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))  in stmmac_reset_subtask()
7073 	if (test_bit(STMMAC_DOWN, &priv->state))  in stmmac_reset_subtask()
7076 	netdev_err(priv->dev, "Reset adapter.\n");  in stmmac_reset_subtask()
7079 	netif_trans_update(priv->dev);  in stmmac_reset_subtask()
7080 	while (test_and_set_bit(STMMAC_RESETING, &priv->state))  in stmmac_reset_subtask()
7083 	set_bit(STMMAC_DOWN, &priv->state);  in stmmac_reset_subtask()
7084 	dev_close(priv->dev);  in stmmac_reset_subtask()
7085 	dev_open(priv->dev, NULL);  in stmmac_reset_subtask()
7086 	clear_bit(STMMAC_DOWN, &priv->state);  in stmmac_reset_subtask()
7087 	clear_bit(STMMAC_RESETING, &priv->state);  in stmmac_reset_subtask()
7093 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,  in stmmac_service_task()  local
7096 	stmmac_reset_subtask(priv);  in stmmac_service_task()
7097 	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);  in stmmac_service_task()
7108 static int stmmac_hw_init(struct stmmac_priv *priv)  in stmmac_hw_init()  argument
7113 	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I)  in stmmac_hw_init()
7115 	priv->chain_mode = chain_mode;  in stmmac_hw_init()
7118 	ret = stmmac_hwif_init(priv);  in stmmac_hw_init()
7123 	priv->hw_cap_support = stmmac_get_hw_features(priv);  in stmmac_hw_init()
7124 	if (priv->hw_cap_support) {  in stmmac_hw_init()
7125 		dev_info(priv->device, "DMA HW capability register supported\n");  in stmmac_hw_init()
7132 		priv->plat->enh_desc = priv->dma_cap.enh_desc;  in stmmac_hw_init()
7133 		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up &&  in stmmac_hw_init()
7134 				!(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL);  in stmmac_hw_init()
7135 		priv->hw->pmt = priv->plat->pmt;  in stmmac_hw_init()
7136 		if (priv->dma_cap.hash_tb_sz) {  in stmmac_hw_init()
7137 			priv->hw->multicast_filter_bins =  in stmmac_hw_init()
7138 					(BIT(priv->dma_cap.hash_tb_sz) << 5);  in stmmac_hw_init()
7139 			priv->hw->mcast_bits_log2 =  in stmmac_hw_init()
7140 					ilog2(priv->hw->multicast_filter_bins);  in stmmac_hw_init()
7144 		if (priv->plat->force_thresh_dma_mode)  in stmmac_hw_init()
7145 			priv->plat->tx_coe = 0;  in stmmac_hw_init()
7147 			priv->plat->tx_coe = priv->dma_cap.tx_coe;  in stmmac_hw_init()
7150 		priv->plat->rx_coe = priv->dma_cap.rx_coe;  in stmmac_hw_init()
7152 		if (priv->dma_cap.rx_coe_type2)  in stmmac_hw_init()
7153 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;  in stmmac_hw_init()
7154 		else if (priv->dma_cap.rx_coe_type1)  in stmmac_hw_init()
7155 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;  in stmmac_hw_init()
7158 		dev_info(priv->device, "No HW DMA feature register supported\n");  in stmmac_hw_init()
7161 	if (priv->plat->rx_coe) {  in stmmac_hw_init()
7162 		priv->hw->rx_csum = priv->plat->rx_coe;  in stmmac_hw_init()
7163 		dev_info(priv->device, "RX Checksum Offload Engine supported\n");  in stmmac_hw_init()
7164 		if (priv->synopsys_id < DWMAC_CORE_4_00)  in stmmac_hw_init()
7165 			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);  in stmmac_hw_init()
7167 	if (priv->plat->tx_coe)  in stmmac_hw_init()
7168 		dev_info(priv->device, "TX Checksum insertion supported\n");  in stmmac_hw_init()
7170 	if (priv->plat->pmt) {  in stmmac_hw_init()
7171 		dev_info(priv->device, "Wake-Up On Lan supported\n");  in stmmac_hw_init()
7172 		device_set_wakeup_capable(priv->device, 1);  in stmmac_hw_init()
7175 	if (priv->dma_cap.tsoen)  in stmmac_hw_init()
7176 		dev_info(priv->device, "TSO supported\n");  in stmmac_hw_init()
7178 	priv->hw->vlan_fail_q_en =  in stmmac_hw_init()
7179 		(priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN);  in stmmac_hw_init()
7180 	priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;  in stmmac_hw_init()
7183 	if (priv->hwif_quirks) {  in stmmac_hw_init()
7184 		ret = priv->hwif_quirks(priv);  in stmmac_hw_init()
7194 	if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||  in stmmac_hw_init()
7195 	    (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {  in stmmac_hw_init()
7196 		priv->use_riwt = 1;  in stmmac_hw_init()
7197 		dev_info(priv->device,  in stmmac_hw_init()
7206 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_napi_add()  local
7209 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);  in stmmac_napi_add()
7212 		struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_napi_add()
7214 		ch->priv_data = priv;  in stmmac_napi_add()
7218 		if (queue < priv->plat->rx_queues_to_use) {  in stmmac_napi_add()
7221 		if (queue < priv->plat->tx_queues_to_use) {  in stmmac_napi_add()
7225 		if (queue < priv->plat->rx_queues_to_use &&  in stmmac_napi_add()
7226 		    queue < priv->plat->tx_queues_to_use) {  in stmmac_napi_add()
7235 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_napi_del()  local
7238 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);  in stmmac_napi_del()
7241 		struct stmmac_channel *ch = &priv->channel[queue];  in stmmac_napi_del()
7243 		if (queue < priv->plat->rx_queues_to_use)  in stmmac_napi_del()
7245 		if (queue < priv->plat->tx_queues_to_use)  in stmmac_napi_del()
7247 		if (queue < priv->plat->rx_queues_to_use &&  in stmmac_napi_del()
7248 		    queue < priv->plat->tx_queues_to_use) {  in stmmac_napi_del()
7256 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_reinit_queues()  local
7264 	priv->plat->rx_queues_to_use = rx_cnt;  in stmmac_reinit_queues()
7265 	priv->plat->tx_queues_to_use = tx_cnt;  in stmmac_reinit_queues()
7267 		for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)  in stmmac_reinit_queues()
7268 			priv->rss.table[i] = ethtool_rxfh_indir_default(i,  in stmmac_reinit_queues()
7281 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_reinit_ringparam()  local
7287 	priv->dma_conf.dma_rx_size = rx_size;  in stmmac_reinit_ringparam()
7288 	priv->dma_conf.dma_tx_size = tx_size;  in stmmac_reinit_ringparam()
7300 	struct stmmac_priv *priv = ctx->priv;  in stmmac_xdp_rx_timestamp()  local
7305 	if (!priv->hwts_rx_en)  in stmmac_xdp_rx_timestamp()
7309 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)  in stmmac_xdp_rx_timestamp()
7313 	if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) {  in stmmac_xdp_rx_timestamp()
7314 		stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns);  in stmmac_xdp_rx_timestamp()
7315 		ns -= priv->plat->cdc_error_adj;  in stmmac_xdp_rx_timestamp()
7342 	struct stmmac_priv *priv;  in stmmac_dvr_probe()  local
7353 	priv = netdev_priv(ndev);  in stmmac_dvr_probe()
7354 	priv->device = device;  in stmmac_dvr_probe()
7355 	priv->dev = ndev;  in stmmac_dvr_probe()
7358 		u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp);  in stmmac_dvr_probe()
7360 		u64_stats_init(&priv->xstats.txq_stats[i].q_syncp);  in stmmac_dvr_probe()
7361 		u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp);  in stmmac_dvr_probe()
7364 	priv->xstats.pcpu_stats =  in stmmac_dvr_probe()
7366 	if (!priv->xstats.pcpu_stats)  in stmmac_dvr_probe()
7370 	priv->pause = pause;  in stmmac_dvr_probe()
7371 	priv->plat = plat_dat;  in stmmac_dvr_probe()
7372 	priv->ioaddr = res->addr;  in stmmac_dvr_probe()
7373 	priv->dev->base_addr = (unsigned long)res->addr;  in stmmac_dvr_probe()
7374 	priv->plat->dma_cfg->multi_msi_en =  in stmmac_dvr_probe()
7375 		(priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN);  in stmmac_dvr_probe()
7377 	priv->dev->irq = res->irq;  in stmmac_dvr_probe()
7378 	priv->wol_irq = res->wol_irq;  in stmmac_dvr_probe()
7379 	priv->lpi_irq = res->lpi_irq;  in stmmac_dvr_probe()
7380 	priv->sfty_irq = res->sfty_irq;  in stmmac_dvr_probe()
7381 	priv->sfty_ce_irq = res->sfty_ce_irq;  in stmmac_dvr_probe()
7382 	priv->sfty_ue_irq = res->sfty_ue_irq;  in stmmac_dvr_probe()
7384 		priv->rx_irq[i] = res->rx_irq[i];  in stmmac_dvr_probe()
7386 		priv->tx_irq[i] = res->tx_irq[i];  in stmmac_dvr_probe()
7389 		eth_hw_addr_set(priv->dev, res->mac);  in stmmac_dvr_probe()
7391 	dev_set_drvdata(device, priv->dev);  in stmmac_dvr_probe()
7396 	priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL);  in stmmac_dvr_probe()
7397 	if (!priv->af_xdp_zc_qps)  in stmmac_dvr_probe()
7401 	priv->wq = create_singlethread_workqueue("stmmac_wq");  in stmmac_dvr_probe()
7402 	if (!priv->wq) {  in stmmac_dvr_probe()
7403 		dev_err(priv->device, "failed to create workqueue\n");  in stmmac_dvr_probe()
7408 	INIT_WORK(&priv->service_task, stmmac_service_task);  in stmmac_dvr_probe()
7414 		priv->plat->phy_addr = phyaddr;  in stmmac_dvr_probe()
7416 	if (priv->plat->stmmac_rst) {  in stmmac_dvr_probe()
7417 		ret = reset_control_assert(priv->plat->stmmac_rst);  in stmmac_dvr_probe()
7418 		reset_control_deassert(priv->plat->stmmac_rst);  in stmmac_dvr_probe()
7423 			reset_control_reset(priv->plat->stmmac_rst);  in stmmac_dvr_probe()
7426 	ret = reset_control_deassert(priv->plat->stmmac_ahb_rst);  in stmmac_dvr_probe()
7428 		dev_err(priv->device, "unable to bring out of ahb reset: %pe\n",  in stmmac_dvr_probe()
7435 	ret = stmmac_hw_init(priv);  in stmmac_dvr_probe()
7441 	if (priv->synopsys_id < DWMAC_CORE_5_20)  in stmmac_dvr_probe()
7442 		priv->plat->dma_cfg->dche = false;  in stmmac_dvr_probe()
7444 	stmmac_check_ether_addr(priv);  in stmmac_dvr_probe()
7456 	ret = stmmac_tc_init(priv, priv);  in stmmac_dvr_probe()
7461 	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {  in stmmac_dvr_probe()
7463 		if (priv->plat->has_gmac4)  in stmmac_dvr_probe()
7465 		priv->tso = true;  in stmmac_dvr_probe()
7466 		dev_info(priv->device, "TSO feature enabled\n");  in stmmac_dvr_probe()
7469 	if (priv->dma_cap.sphen &&  in stmmac_dvr_probe()
7470 	    !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) {  in stmmac_dvr_probe()
7472 		priv->sph_cap = true;  in stmmac_dvr_probe()
7473 		priv->sph = priv->sph_cap;  in stmmac_dvr_probe()
7474 		dev_info(priv->device, "SPH feature enabled\n");  in stmmac_dvr_probe()
7482 	if (priv->plat->host_dma_width)  in stmmac_dvr_probe()
7483 		priv->dma_cap.host_dma_width = priv->plat->host_dma_width;  in stmmac_dvr_probe()
7485 		priv->dma_cap.host_dma_width = priv->dma_cap.addr64;  in stmmac_dvr_probe()
7487 	if (priv->dma_cap.host_dma_width) {  in stmmac_dvr_probe()
7489 				DMA_BIT_MASK(priv->dma_cap.host_dma_width));  in stmmac_dvr_probe()
7491 			dev_info(priv->device, "Using %d/%d bits DMA host/device width\n",  in stmmac_dvr_probe()
7492 				 priv->dma_cap.host_dma_width, priv->dma_cap.addr64);  in stmmac_dvr_probe()
7499 				priv->plat->dma_cfg->eame = true;  in stmmac_dvr_probe()
7503 				dev_err(priv->device, "Failed to set DMA Mask\n");  in stmmac_dvr_probe()
7507 			priv->dma_cap.host_dma_width = 32;  in stmmac_dvr_probe()
7516 	if (priv->plat->has_gmac4) {  in stmmac_dvr_probe()
7518 		priv->hw->hw_vlan_en = true;  in stmmac_dvr_probe()
7520 	if (priv->dma_cap.vlhash) {  in stmmac_dvr_probe()
7524 	if (priv->dma_cap.vlins) {  in stmmac_dvr_probe()
7526 		if (priv->dma_cap.dvlan)  in stmmac_dvr_probe()
7530 	priv->msg_enable = netif_msg_init(debug, default_msg_level);  in stmmac_dvr_probe()
7532 	priv->xstats.threshold = tc;  in stmmac_dvr_probe()
7535 	rxq = priv->plat->rx_queues_to_use;  in stmmac_dvr_probe()
7536 	netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));  in stmmac_dvr_probe()
7537 	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)  in stmmac_dvr_probe()
7538 		priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);  in stmmac_dvr_probe()
7540 	if (priv->dma_cap.rssen && priv->plat->rss_en)  in stmmac_dvr_probe()
7547 	if (priv->plat->has_xgmac)  in stmmac_dvr_probe()
7549 	else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))  in stmmac_dvr_probe()
7556 	if ((priv->plat->maxmtu < ndev->max_mtu) &&  in stmmac_dvr_probe()
7557 	    (priv->plat->maxmtu >= ndev->min_mtu))  in stmmac_dvr_probe()
7558 		ndev->max_mtu = priv->plat->maxmtu;  in stmmac_dvr_probe()
7559 	else if (priv->plat->maxmtu < ndev->min_mtu)  in stmmac_dvr_probe()
7560 		dev_warn(priv->device,  in stmmac_dvr_probe()
7562 			 __func__, priv->plat->maxmtu);  in stmmac_dvr_probe()
7565 		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */  in stmmac_dvr_probe()
7572 	mutex_init(&priv->lock);  in stmmac_dvr_probe()
7574 	stmmac_fpe_init(priv);  in stmmac_dvr_probe()
7582 	if (priv->plat->clk_csr >= 0)  in stmmac_dvr_probe()
7583 		priv->clk_csr = priv->plat->clk_csr;  in stmmac_dvr_probe()
7585 		stmmac_clk_csr_set(priv);  in stmmac_dvr_probe()
7587 	stmmac_check_pcs_mode(priv);  in stmmac_dvr_probe()
7596 		dev_err_probe(priv->device, ret,  in stmmac_dvr_probe()
7598 			      priv->plat->bus_id);  in stmmac_dvr_probe()
7602 	if (priv->plat->speed_mode_2500)  in stmmac_dvr_probe()
7603 		priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv);  in stmmac_dvr_probe()
7609 	ret = stmmac_phy_setup(priv);  in stmmac_dvr_probe()
7617 		dev_err(priv->device, "%s: ERROR %i registering the device\n",  in stmmac_dvr_probe()
7626 	if (priv->plat->dump_debug_regs)  in stmmac_dvr_probe()
7627 		priv->plat->dump_debug_regs(priv->plat->bsp_priv);  in stmmac_dvr_probe()
7637 	phylink_destroy(priv->phylink);  in stmmac_dvr_probe()
7645 	destroy_workqueue(priv->wq);  in stmmac_dvr_probe()
7647 	bitmap_free(priv->af_xdp_zc_qps);  in stmmac_dvr_probe()
7662 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_dvr_remove()  local
7664 	netdev_info(priv->dev, "%s: removing driver", __func__);  in stmmac_dvr_remove()
7668 	stmmac_stop_all_dma(priv);  in stmmac_dvr_remove()
7669 	stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_dvr_remove()
7675 	phylink_destroy(priv->phylink);  in stmmac_dvr_remove()
7676 	if (priv->plat->stmmac_rst)  in stmmac_dvr_remove()
7677 		reset_control_assert(priv->plat->stmmac_rst);  in stmmac_dvr_remove()
7678 	reset_control_assert(priv->plat->stmmac_ahb_rst);  in stmmac_dvr_remove()
7683 	destroy_workqueue(priv->wq);  in stmmac_dvr_remove()
7684 	mutex_destroy(&priv->lock);  in stmmac_dvr_remove()
7685 	bitmap_free(priv->af_xdp_zc_qps);  in stmmac_dvr_remove()
7702 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_suspend()  local
7708 	mutex_lock(&priv->lock);  in stmmac_suspend()
7712 	stmmac_disable_all_queues(priv);  in stmmac_suspend()
7714 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)  in stmmac_suspend()
7715 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);  in stmmac_suspend()
7717 	if (priv->eee_enabled) {  in stmmac_suspend()
7718 		priv->tx_path_in_lpi_mode = false;  in stmmac_suspend()
7719 		del_timer_sync(&priv->eee_ctrl_timer);  in stmmac_suspend()
7723 	stmmac_stop_all_dma(priv);  in stmmac_suspend()
7725 	if (priv->plat->serdes_powerdown)  in stmmac_suspend()
7726 		priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);  in stmmac_suspend()
7729 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_suspend()
7730 		stmmac_pmt(priv, priv->hw, priv->wolopts);  in stmmac_suspend()
7731 		priv->irq_wake = 1;  in stmmac_suspend()
7733 		stmmac_mac_set(priv, priv->ioaddr, false);  in stmmac_suspend()
7734 		pinctrl_pm_select_sleep_state(priv->device);  in stmmac_suspend()
7737 	mutex_unlock(&priv->lock);  in stmmac_suspend()
7740 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_suspend()
7741 		phylink_suspend(priv->phylink, true);  in stmmac_suspend()
7743 		if (device_may_wakeup(priv->device))  in stmmac_suspend()
7744 			phylink_speed_down(priv->phylink, false);  in stmmac_suspend()
7745 		phylink_suspend(priv->phylink, false);  in stmmac_suspend()
7749 	if (stmmac_fpe_supported(priv))  in stmmac_suspend()
7750 		timer_shutdown_sync(&priv->fpe_cfg.verify_timer);  in stmmac_suspend()
7752 	priv->speed = SPEED_UNKNOWN;  in stmmac_suspend()
7757 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_reset_rx_queue()  argument
7759 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];  in stmmac_reset_rx_queue()
7765 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_reset_tx_queue()  argument
7767 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];  in stmmac_reset_tx_queue()
7773 	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));  in stmmac_reset_tx_queue()
7780 static void stmmac_reset_queues_param(struct stmmac_priv *priv)  in stmmac_reset_queues_param()  argument
7782 	u32 rx_cnt = priv->plat->rx_queues_to_use;  in stmmac_reset_queues_param()
7783 	u32 tx_cnt = priv->plat->tx_queues_to_use;  in stmmac_reset_queues_param()
7787 		stmmac_reset_rx_queue(priv, queue);  in stmmac_reset_queues_param()
7790 		stmmac_reset_tx_queue(priv, queue);  in stmmac_reset_queues_param()
7802 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_resume()  local
7814 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_resume()
7815 		mutex_lock(&priv->lock);  in stmmac_resume()
7816 		stmmac_pmt(priv, priv->hw, 0);  in stmmac_resume()
7817 		mutex_unlock(&priv->lock);  in stmmac_resume()
7818 		priv->irq_wake = 0;  in stmmac_resume()
7820 		pinctrl_pm_select_default_state(priv->device);  in stmmac_resume()
7822 		if (priv->mii)  in stmmac_resume()
7823 			stmmac_mdio_reset(priv->mii);  in stmmac_resume()
7826 	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&  in stmmac_resume()
7827 	    priv->plat->serdes_powerup) {  in stmmac_resume()
7828 		ret = priv->plat->serdes_powerup(ndev,  in stmmac_resume()
7829 						 priv->plat->bsp_priv);  in stmmac_resume()
7836 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {  in stmmac_resume()
7837 		phylink_resume(priv->phylink);  in stmmac_resume()
7839 		phylink_resume(priv->phylink);  in stmmac_resume()
7840 		if (device_may_wakeup(priv->device))  in stmmac_resume()
7841 			phylink_speed_up(priv->phylink);  in stmmac_resume()
7846 	mutex_lock(&priv->lock);  in stmmac_resume()
7848 	stmmac_reset_queues_param(priv);  in stmmac_resume()
7850 	stmmac_free_tx_skbufs(priv);  in stmmac_resume()
7851 	stmmac_clear_descriptors(priv, &priv->dma_conf);  in stmmac_resume()
7854 	stmmac_init_coalesce(priv);  in stmmac_resume()
7857 	stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);  in stmmac_resume()
7859 	stmmac_enable_all_queues(priv);  in stmmac_resume()
7860 	stmmac_enable_all_dma_irq(priv);  in stmmac_resume()
7862 	mutex_unlock(&priv->lock);  in stmmac_resume()