1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2004-2006 Atmel Corporation
10 #include <linux/clk-provider.h>
25 #include <linux/dma-mapping.h>
40 #include <linux/firmware/xlnx-zynqmp.h>
58 				 * (bp)->rx_ring_size)
64 				 * (bp)->tx_ring_size)
67 #define MACB_TX_WAKEUP_THRESH(bp)	(3 * (bp)->tx_ring_size / 4)
78 …MAX_TX_LEN		((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN -…
94  * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
133 	switch (bp->hw_dma_cap) {  in macb_dma_desc_get_size()
158 	switch (bp->hw_dma_cap) {  in macb_adj_dma_desc_idx()
184 	return index & (bp->tx_ring_size - 1);  in macb_tx_ring_wrap()
190 	index = macb_tx_ring_wrap(queue->bp, index);  in macb_tx_desc()
191 	index = macb_adj_dma_desc_idx(queue->bp, index);  in macb_tx_desc()
192 	return &queue->tx_ring[index];  in macb_tx_desc()
198 	return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];  in macb_tx_skb()
205 	offset = macb_tx_ring_wrap(queue->bp, index) *  in macb_tx_dma()
206 			macb_dma_desc_get_size(queue->bp);  in macb_tx_dma()
208 	return queue->tx_ring_dma + offset;  in macb_tx_dma()
213 	return index & (bp->rx_ring_size - 1);  in macb_rx_ring_wrap()
218 	index = macb_rx_ring_wrap(queue->bp, index);  in macb_rx_desc()
219 	index = macb_adj_dma_desc_idx(queue->bp, index);  in macb_rx_desc()
220 	return &queue->rx_ring[index];  in macb_rx_desc()
225 	return queue->rx_buffers + queue->bp->rx_buffer_size *  in macb_rx_buffer()
226 	       macb_rx_ring_wrap(queue->bp, index);  in macb_rx_buffer()
232 	return __raw_readl(bp->regs + offset);  in hw_readl_native()
237 	__raw_writel(value, bp->regs + offset);  in hw_writel_native()
242 	return readl_relaxed(bp->regs + offset);  in hw_readl()
247 	writel_relaxed(value, bp->regs + offset);  in hw_writel()
284 	bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));  in macb_set_hwaddr()
286 	top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));  in macb_set_hwaddr()
323 			eth_hw_addr_set(bp->dev, addr);  in macb_get_hwaddr()
328 	dev_info(&bp->pdev->dev, "invalid hw address, using random\n");  in macb_get_hwaddr()
329 	eth_hw_addr_random(bp->dev);  in macb_get_hwaddr()
342 	struct macb *bp = bus->priv;  in macb_mdio_read_c22()
345 	status = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_mdio_read_c22()
366 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_read_c22()
367 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_read_c22()
375 	struct macb *bp = bus->priv;  in macb_mdio_read_c45()
378 	status = pm_runtime_get_sync(&bp->pdev->dev);  in macb_mdio_read_c45()
380 		pm_runtime_put_noidle(&bp->pdev->dev);  in macb_mdio_read_c45()
412 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_read_c45()
413 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_read_c45()
421 	struct macb *bp = bus->priv;  in macb_mdio_write_c22()
424 	status = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_mdio_write_c22()
444 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_write_c22()
445 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_write_c22()
454 	struct macb *bp = bus->priv;  in macb_mdio_write_c45()
457 	status = pm_runtime_get_sync(&bp->pdev->dev);  in macb_mdio_write_c45()
459 		pm_runtime_put_noidle(&bp->pdev->dev);  in macb_mdio_write_c45()
490 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_write_c45()
491 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_write_c45()
501 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_init_buffers()
502 		queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));  in macb_init_buffers()
504 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_init_buffers()
506 				     upper_32_bits(queue->rx_ring_dma));  in macb_init_buffers()
508 		queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));  in macb_init_buffers()
510 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_init_buffers()
512 				     upper_32_bits(queue->tx_ring_dma));  in macb_init_buffers()
518  * macb_set_tx_clk() - Set a clock to a new frequency
526 	if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG))  in macb_set_tx_clk()
530 	if (bp->phy_interface == PHY_INTERFACE_MODE_MII)  in macb_set_tx_clk()
547 	rate_rounded = clk_round_rate(bp->tx_clk, rate);  in macb_set_tx_clk()
554 	ferr = abs(rate_rounded - rate);  in macb_set_tx_clk()
557 		netdev_warn(bp->dev,  in macb_set_tx_clk()
561 	if (clk_set_rate(bp->tx_clk, rate_rounded))  in macb_set_tx_clk()
562 		netdev_err(bp->dev, "adjusting tx_clk failed.\n");  in macb_set_tx_clk()
586 	state->speed = SPEED_10000;  in macb_usx_pcs_get_state()
587 	state->duplex = 1;  in macb_usx_pcs_get_state()
588 	state->an_complete = 1;  in macb_usx_pcs_get_state()
591 	state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK));  in macb_usx_pcs_get_state()
594 		state->pause = MLO_PAUSE_RX;  in macb_usx_pcs_get_state()
614 	state->link = 0;  in macb_pcs_get_state()
646 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_config()
652 	spin_lock_irqsave(&bp->lock, flags);  in macb_mac_config()
657 	if (bp->caps & MACB_CAPS_MACB_IS_EMAC) {  in macb_mac_config()
658 		if (state->interface == PHY_INTERFACE_MODE_RMII)  in macb_mac_config()
664 		if (state->interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mac_config()
666 		} else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {  in macb_mac_config()
669 		} else if (bp->caps & MACB_CAPS_MIIONRGMII &&  in macb_mac_config()
670 			   bp->phy_interface == PHY_INTERFACE_MODE_MII) {  in macb_mac_config()
686 	if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mac_config()
698 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_mac_config()
704 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_link_down()
710 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))  in macb_mac_link_down()
711 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_mac_link_down()
713 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));  in macb_mac_link_down()
728 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_link_up()
735 	spin_lock_irqsave(&bp->lock, flags);  in macb_mac_link_up()
747 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {  in macb_mac_link_up()
762 		bp->macbgem_ops.mog_init_rings(bp);  in macb_mac_link_up()
765 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_mac_link_up()
767 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));  in macb_mac_link_up()
772 	if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER)  in macb_mac_link_up()
776 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_mac_link_up()
778 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))  in macb_mac_link_up()
794 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_select_pcs()
798 		return &bp->phylink_usx_pcs;  in macb_mac_select_pcs()
800 		return &bp->phylink_sgmii_pcs;  in macb_mac_select_pcs()
814 	dn = of_parse_phandle(dn, "phy-handle", 0);  in macb_phy_handle_exists()
821 	struct device_node *dn = bp->pdev->dev.of_node;  in macb_phylink_connect()
822 	struct net_device *dev = bp->dev;  in macb_phylink_connect()
827 		ret = phylink_of_phy_connect(bp->phylink, dn, 0);  in macb_phylink_connect()
830 		phydev = phy_find_first(bp->mii_bus);  in macb_phylink_connect()
833 			return -ENXIO;  in macb_phylink_connect()
837 		ret = phylink_connect_phy(bp->phylink, phydev);  in macb_phylink_connect()
845 	phylink_start(bp->phylink);  in macb_phylink_connect()
853 	struct net_device *ndev = to_net_dev(config->dev);  in macb_get_pcs_fixed_state()
856 	state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0;  in macb_get_pcs_fixed_state()
864 	bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops;  in macb_mii_probe()
865 	bp->phylink_sgmii_pcs.neg_mode = true;  in macb_mii_probe()
866 	bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops;  in macb_mii_probe()
867 	bp->phylink_usx_pcs.neg_mode = true;  in macb_mii_probe()
869 	bp->phylink_config.dev = &dev->dev;  in macb_mii_probe()
870 	bp->phylink_config.type = PHYLINK_NETDEV;  in macb_mii_probe()
871 	bp->phylink_config.mac_managed_pm = true;  in macb_mii_probe()
873 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mii_probe()
874 		bp->phylink_config.poll_fixed_state = true;  in macb_mii_probe()
875 		bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;  in macb_mii_probe()
878 	bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |  in macb_mii_probe()
882 		  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
884 		  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
887 	if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) {  in macb_mii_probe()
888 		bp->phylink_config.mac_capabilities |= MAC_1000FD;  in macb_mii_probe()
889 		if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))  in macb_mii_probe()
890 			bp->phylink_config.mac_capabilities |= MAC_1000HD;  in macb_mii_probe()
893 			  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
894 		phy_interface_set_rgmii(bp->phylink_config.supported_interfaces);  in macb_mii_probe()
896 		if (bp->caps & MACB_CAPS_PCS)  in macb_mii_probe()
898 				  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
900 		if (bp->caps & MACB_CAPS_HIGH_SPEED) {  in macb_mii_probe()
902 				  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
903 			bp->phylink_config.mac_capabilities |= MAC_10000FD;  in macb_mii_probe()
907 	bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,  in macb_mii_probe()
908 				     bp->phy_interface, &macb_phylink_ops);  in macb_mii_probe()
909 	if (IS_ERR(bp->phylink)) {  in macb_mii_probe()
911 			   PTR_ERR(bp->phylink));  in macb_mii_probe()
912 		return PTR_ERR(bp->phylink);  in macb_mii_probe()
920 	struct device_node *child, *np = bp->pdev->dev.of_node;  in macb_mdiobus_register()
926 		return of_mdiobus_register(bp->mii_bus, mdio_np);  in macb_mdiobus_register()
940 			return of_mdiobus_register(bp->mii_bus, np);  in macb_mdiobus_register()
943 	return mdiobus_register(bp->mii_bus);  in macb_mdiobus_register()
948 	struct device_node *mdio_np, *np = bp->pdev->dev.of_node;  in macb_mii_init()
949 	int err = -ENXIO;  in macb_mii_init()
951 	/* With fixed-link, we don't need to register the MDIO bus,  in macb_mii_init()
957 		return macb_mii_probe(bp->dev);  in macb_mii_init()
962 	bp->mii_bus = mdiobus_alloc();  in macb_mii_init()
963 	if (!bp->mii_bus) {  in macb_mii_init()
964 		err = -ENOMEM;  in macb_mii_init()
968 	bp->mii_bus->name = "MACB_mii_bus";  in macb_mii_init()
969 	bp->mii_bus->read = &macb_mdio_read_c22;  in macb_mii_init()
970 	bp->mii_bus->write = &macb_mdio_write_c22;  in macb_mii_init()
971 	bp->mii_bus->read_c45 = &macb_mdio_read_c45;  in macb_mii_init()
972 	bp->mii_bus->write_c45 = &macb_mdio_write_c45;  in macb_mii_init()
973 	snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",  in macb_mii_init()
974 		 bp->pdev->name, bp->pdev->id);  in macb_mii_init()
975 	bp->mii_bus->priv = bp;  in macb_mii_init()
976 	bp->mii_bus->parent = &bp->pdev->dev;  in macb_mii_init()
978 	dev_set_drvdata(&bp->dev->dev, bp->mii_bus);  in macb_mii_init()
984 	err = macb_mii_probe(bp->dev);  in macb_mii_init()
991 	mdiobus_unregister(bp->mii_bus);  in macb_mii_init()
993 	mdiobus_free(bp->mii_bus);  in macb_mii_init()
1002 	u32 *p = &bp->hw_stats.macb.rx_pause_frames;  in macb_update_stats()
1003 	u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;  in macb_update_stats()
1006 	WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);  in macb_update_stats()
1009 		*p += bp->macb_reg_readl(bp, offset);  in macb_update_stats()
1029 	return -ETIMEDOUT;  in macb_halt_tx()
1034 	if (tx_skb->mapping) {  in macb_tx_unmap()
1035 		if (tx_skb->mapped_as_page)  in macb_tx_unmap()
1036 			dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,  in macb_tx_unmap()
1037 				       tx_skb->size, DMA_TO_DEVICE);  in macb_tx_unmap()
1039 			dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,  in macb_tx_unmap()
1040 					 tx_skb->size, DMA_TO_DEVICE);  in macb_tx_unmap()
1041 		tx_skb->mapping = 0;  in macb_tx_unmap()
1044 	if (tx_skb->skb) {  in macb_tx_unmap()
1045 		napi_consume_skb(tx_skb->skb, budget);  in macb_tx_unmap()
1046 		tx_skb->skb = NULL;  in macb_tx_unmap()
1055 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_set_addr()
1057 		desc_64->addrh = upper_32_bits(addr);  in macb_set_addr()
1065 	desc->addr = lower_32_bits(addr);  in macb_set_addr()
1074 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_get_addr()
1076 		addr = ((u64)(desc_64->addrh) << 32);  in macb_get_addr()
1079 	addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));  in macb_get_addr()
1081 	if (bp->hw_dma_cap & HW_DMA_CAP_PTP)  in macb_get_addr()
1092 	struct macb		*bp = queue->bp;  in macb_tx_error_task()
1099 	netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",  in macb_tx_error_task()
1100 		    (unsigned int)(queue - bp->queues),  in macb_tx_error_task()
1101 		    queue->tx_tail, queue->tx_head);  in macb_tx_error_task()
1109 	napi_disable(&queue->napi_tx);  in macb_tx_error_task()
1110 	spin_lock_irqsave(&bp->lock, flags);  in macb_tx_error_task()
1113 	netif_tx_stop_all_queues(bp->dev);  in macb_tx_error_task()
1120 		netdev_err(bp->dev, "BUG: halt tx timed out\n");  in macb_tx_error_task()
1128 	for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {  in macb_tx_error_task()
1132 		ctrl = desc->ctrl;  in macb_tx_error_task()
1134 		skb = tx_skb->skb;  in macb_tx_error_task()
1142 				skb = tx_skb->skb;  in macb_tx_error_task()
1149 				netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",  in macb_tx_error_task()
1151 					    skb->data);  in macb_tx_error_task()
1152 				bp->dev->stats.tx_packets++;  in macb_tx_error_task()
1153 				queue->stats.tx_packets++;  in macb_tx_error_task()
1154 				bp->dev->stats.tx_bytes += skb->len;  in macb_tx_error_task()
1155 				queue->stats.tx_bytes += skb->len;  in macb_tx_error_task()
1158 			/* "Buffers exhausted mid-frame" errors may only happen  in macb_tx_error_task()
1163 				netdev_err(bp->dev,  in macb_tx_error_task()
1164 					   "BUG: TX buffers exhausted mid-frame\n");  in macb_tx_error_task()
1166 			desc->ctrl = ctrl | MACB_BIT(TX_USED);  in macb_tx_error_task()
1175 	desc->ctrl = MACB_BIT(TX_USED);  in macb_tx_error_task()
1181 	queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));  in macb_tx_error_task()
1183 	if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_tx_error_task()
1184 		queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));  in macb_tx_error_task()
1187 	queue->tx_head = 0;  in macb_tx_error_task()
1188 	queue->tx_tail = 0;  in macb_tx_error_task()
1198 	netif_tx_start_all_queues(bp->dev);  in macb_tx_error_task()
1201 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_tx_error_task()
1202 	napi_enable(&queue->napi_tx);  in macb_tx_error_task()
1212 	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))  in ptp_one_step_sync()
1224 	if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP)  in ptp_one_step_sync()
1237 	struct macb *bp = queue->bp;  in macb_tx_complete()
1238 	u16 queue_index = queue - bp->queues;  in macb_tx_complete()
1243 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_complete()
1244 	head = queue->tx_head;  in macb_tx_complete()
1245 	for (tail = queue->tx_tail; tail != head && packets < budget; tail++) {  in macb_tx_complete()
1256 		ctrl = desc->ctrl;  in macb_tx_complete()
1267 			skb = tx_skb->skb;  in macb_tx_complete()
1271 				if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&  in macb_tx_complete()
1275 				netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",  in macb_tx_complete()
1277 					    skb->data);  in macb_tx_complete()
1278 				bp->dev->stats.tx_packets++;  in macb_tx_complete()
1279 				queue->stats.tx_packets++;  in macb_tx_complete()
1280 				bp->dev->stats.tx_bytes += skb->len;  in macb_tx_complete()
1281 				queue->stats.tx_bytes += skb->len;  in macb_tx_complete()
1285 			/* Now we can safely release resources */  in macb_tx_complete()
1297 	queue->tx_tail = tail;  in macb_tx_complete()
1298 	if (__netif_subqueue_stopped(bp->dev, queue_index) &&  in macb_tx_complete()
1299 	    CIRC_CNT(queue->tx_head, queue->tx_tail,  in macb_tx_complete()
1300 		     bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))  in macb_tx_complete()
1301 		netif_wake_subqueue(bp->dev, queue_index);  in macb_tx_complete()
1302 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_complete()
1312 	struct macb *bp = queue->bp;  in gem_rx_refill()
1315 	while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,  in gem_rx_refill()
1316 			bp->rx_ring_size) > 0) {  in gem_rx_refill()
1317 		entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);  in gem_rx_refill()
1324 		if (!queue->rx_skbuff[entry]) {  in gem_rx_refill()
1326 			skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);  in gem_rx_refill()
1328 				netdev_err(bp->dev,  in gem_rx_refill()
1334 			paddr = dma_map_single(&bp->pdev->dev, skb->data,  in gem_rx_refill()
1335 					       bp->rx_buffer_size,  in gem_rx_refill()
1337 			if (dma_mapping_error(&bp->pdev->dev, paddr)) {  in gem_rx_refill()
1342 			queue->rx_skbuff[entry] = skb;  in gem_rx_refill()
1344 			if (entry == bp->rx_ring_size - 1)  in gem_rx_refill()
1346 			desc->ctrl = 0;  in gem_rx_refill()
1356 			desc->ctrl = 0;  in gem_rx_refill()
1358 			desc->addr &= ~MACB_BIT(RX_USED);  in gem_rx_refill()
1360 		queue->rx_prepared_head++;  in gem_rx_refill()
1366 	netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",  in gem_rx_refill()
1367 			queue, queue->rx_prepared_head, queue->rx_tail);  in gem_rx_refill()
1379 		desc->addr &= ~MACB_BIT(RX_USED);  in discard_partial_frame()
1394 	struct macb *bp = queue->bp;  in gem_rx()
1406 		entry = macb_rx_ring_wrap(bp, queue->rx_tail);  in gem_rx()
1412 		rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;  in gem_rx()
1418 		/* Ensure ctrl is at least as up-to-date as rxused */  in gem_rx()
1421 		ctrl = desc->ctrl;  in gem_rx()
1423 		queue->rx_tail++;  in gem_rx()
1427 			netdev_err(bp->dev,  in gem_rx()
1429 			bp->dev->stats.rx_dropped++;  in gem_rx()
1430 			queue->stats.rx_dropped++;  in gem_rx()
1433 		skb = queue->rx_skbuff[entry];  in gem_rx()
1435 			netdev_err(bp->dev,  in gem_rx()
1437 			bp->dev->stats.rx_dropped++;  in gem_rx()
1438 			queue->stats.rx_dropped++;  in gem_rx()
1442 		queue->rx_skbuff[entry] = NULL;  in gem_rx()
1443 		len = ctrl & bp->rx_frm_len_mask;  in gem_rx()
1445 		netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);  in gem_rx()
1448 		dma_unmap_single(&bp->pdev->dev, addr,  in gem_rx()
1449 				 bp->rx_buffer_size, DMA_FROM_DEVICE);  in gem_rx()
1451 		skb->protocol = eth_type_trans(skb, bp->dev);  in gem_rx()
1453 		if (bp->dev->features & NETIF_F_RXCSUM &&  in gem_rx()
1454 		    !(bp->dev->flags & IFF_PROMISC) &&  in gem_rx()
1456 			skb->ip_summed = CHECKSUM_UNNECESSARY;  in gem_rx()
1458 		bp->dev->stats.rx_packets++;  in gem_rx()
1459 		queue->stats.rx_packets++;  in gem_rx()
1460 		bp->dev->stats.rx_bytes += skb->len;  in gem_rx()
1461 		queue->stats.rx_bytes += skb->len;  in gem_rx()
1466 		netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",  in gem_rx()
1467 			    skb->len, skb->csum);  in gem_rx()
1471 			       skb->data, 32, true);  in gem_rx()
1490 	struct macb *bp = queue->bp;  in macb_rx_frame()
1493 	len = desc->ctrl & bp->rx_frm_len_mask;  in macb_rx_frame()
1495 	netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",  in macb_rx_frame()
1501 	 * payload word-aligned.  in macb_rx_frame()
1507 	skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);  in macb_rx_frame()
1509 		bp->dev->stats.rx_dropped++;  in macb_rx_frame()
1512 			desc->addr &= ~MACB_BIT(RX_USED);  in macb_rx_frame()
1529 		unsigned int frag_len = bp->rx_buffer_size;  in macb_rx_frame()
1534 				return -1;  in macb_rx_frame()
1536 			frag_len = len - offset;  in macb_rx_frame()
1541 		offset += bp->rx_buffer_size;  in macb_rx_frame()
1543 		desc->addr &= ~MACB_BIT(RX_USED);  in macb_rx_frame()
1553 	skb->protocol = eth_type_trans(skb, bp->dev);  in macb_rx_frame()
1555 	bp->dev->stats.rx_packets++;  in macb_rx_frame()
1556 	bp->dev->stats.rx_bytes += skb->len;  in macb_rx_frame()
1557 	netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",  in macb_rx_frame()
1558 		    skb->len, skb->csum);  in macb_rx_frame()
1566 	struct macb *bp = queue->bp;  in macb_init_rx_ring()
1571 	addr = queue->rx_buffers_dma;  in macb_init_rx_ring()
1572 	for (i = 0; i < bp->rx_ring_size; i++) {  in macb_init_rx_ring()
1575 		desc->ctrl = 0;  in macb_init_rx_ring()
1576 		addr += bp->rx_buffer_size;  in macb_init_rx_ring()
1578 	desc->addr |= MACB_BIT(RX_WRAP);  in macb_init_rx_ring()
1579 	queue->rx_tail = 0;  in macb_init_rx_ring()
1585 	struct macb *bp = queue->bp;  in macb_rx()
1589 	int first_frag = -1;  in macb_rx()
1591 	for (tail = queue->rx_tail; budget > 0; tail++) {  in macb_rx()
1598 		if (!(desc->addr & MACB_BIT(RX_USED)))  in macb_rx()
1601 		/* Ensure ctrl is at least as up-to-date as addr */  in macb_rx()
1604 		ctrl = desc->ctrl;  in macb_rx()
1607 			if (first_frag != -1)  in macb_rx()
1615 			if (unlikely(first_frag == -1)) {  in macb_rx()
1621 			first_frag = -1;  in macb_rx()
1628 				budget--;  in macb_rx()
1637 		netdev_err(bp->dev, "RX queue corruption: reset it\n");  in macb_rx()
1639 		spin_lock_irqsave(&bp->lock, flags);  in macb_rx()
1645 		queue_writel(queue, RBQP, queue->rx_ring_dma);  in macb_rx()
1649 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_rx()
1653 	if (first_frag != -1)  in macb_rx()
1654 		queue->rx_tail = first_frag;  in macb_rx()
1656 		queue->rx_tail = tail;  in macb_rx()
1663 	struct macb *bp = queue->bp;  in macb_rx_pending()
1667 	entry = macb_rx_ring_wrap(bp, queue->rx_tail);  in macb_rx_pending()
1673 	return (desc->addr & MACB_BIT(RX_USED)) != 0;  in macb_rx_pending()
1679 	struct macb *bp = queue->bp;  in macb_rx_poll()
1682 	work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);  in macb_rx_poll()
1684 	netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n",  in macb_rx_poll()
1685 		    (unsigned int)(queue - bp->queues), work_done, budget);  in macb_rx_poll()
1688 		queue_writel(queue, IER, bp->rx_intr_mask);  in macb_rx_poll()
1694 		 * interrupts are re-enabled.  in macb_rx_poll()
1695 		 * Check for this case here to avoid losing a wakeup. This can  in macb_rx_poll()
1701 			queue_writel(queue, IDR, bp->rx_intr_mask);  in macb_rx_poll()
1702 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_rx_poll()
1704 			netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n");  in macb_rx_poll()
1716 	struct macb *bp = queue->bp;  in macb_tx_restart()
1719 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_restart()
1721 	if (queue->tx_head == queue->tx_tail)  in macb_tx_restart()
1726 	head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head));  in macb_tx_restart()
1731 	spin_lock_irq(&bp->lock);  in macb_tx_restart()
1733 	spin_unlock_irq(&bp->lock);  in macb_tx_restart()
1736 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_restart()
1743 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_complete_pending()
1744 	if (queue->tx_head != queue->tx_tail) {  in macb_tx_complete_pending()
1748 		if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED))  in macb_tx_complete_pending()
1751 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_complete_pending()
1758 	struct macb *bp = queue->bp;  in macb_tx_poll()
1764 	if (queue->txubr_pending) {  in macb_tx_poll()
1765 		queue->txubr_pending = false;  in macb_tx_poll()
1766 		netdev_vdbg(bp->dev, "poll: tx restart\n");  in macb_tx_poll()
1770 	netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n",  in macb_tx_poll()
1771 		    (unsigned int)(queue - bp->queues), work_done, budget);  in macb_tx_poll()
1780 		 * interrupts are re-enabled.  in macb_tx_poll()
1781 		 * Check for this case here to avoid losing a wakeup. This can  in macb_tx_poll()
1788 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_tx_poll()
1790 			netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n");  in macb_tx_poll()
1801 	struct net_device *dev = bp->dev;  in macb_hresp_error_task()
1806 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_hresp_error_task()
1807 		queue_writel(queue, IDR, bp->rx_intr_mask |  in macb_hresp_error_task()
1818 	bp->macbgem_ops.mog_init_rings(bp);  in macb_hresp_error_task()
1824 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_hresp_error_task()
1826 			     bp->rx_intr_mask |  in macb_hresp_error_task()
1840 	struct macb *bp = queue->bp;  in macb_wol_interrupt()
1848 	spin_lock(&bp->lock);  in macb_wol_interrupt()
1853 		netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n",  in macb_wol_interrupt()
1854 			    (unsigned int)(queue - bp->queues),  in macb_wol_interrupt()
1856 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_wol_interrupt()
1858 		pm_wakeup_event(&bp->pdev->dev, 0);  in macb_wol_interrupt()
1861 	spin_unlock(&bp->lock);  in macb_wol_interrupt()
1869 	struct macb *bp = queue->bp;  in gem_wol_interrupt()
1877 	spin_lock(&bp->lock);  in gem_wol_interrupt()
1882 		netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n",  in gem_wol_interrupt()
1883 			    (unsigned int)(queue - bp->queues),  in gem_wol_interrupt()
1885 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in gem_wol_interrupt()
1887 		pm_wakeup_event(&bp->pdev->dev, 0);  in gem_wol_interrupt()
1890 	spin_unlock(&bp->lock);  in gem_wol_interrupt()
1898 	struct macb *bp = queue->bp;  in macb_interrupt()
1899 	struct net_device *dev = bp->dev;  in macb_interrupt()
1907 	spin_lock(&bp->lock);  in macb_interrupt()
1912 			queue_writel(queue, IDR, -1);  in macb_interrupt()
1913 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1914 				queue_writel(queue, ISR, -1);  in macb_interrupt()
1918 		netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",  in macb_interrupt()
1919 			    (unsigned int)(queue - bp->queues),  in macb_interrupt()
1922 		if (status & bp->rx_intr_mask) {  in macb_interrupt()
1929 			queue_writel(queue, IDR, bp->rx_intr_mask);  in macb_interrupt()
1930 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1933 			if (napi_schedule_prep(&queue->napi_rx)) {  in macb_interrupt()
1934 				netdev_vdbg(bp->dev, "scheduling RX softirq\n");  in macb_interrupt()
1935 				__napi_schedule(&queue->napi_rx);  in macb_interrupt()
1942 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1947 				queue->txubr_pending = true;  in macb_interrupt()
1948 				wmb(); // ensure softirq can see update  in macb_interrupt()
1951 			if (napi_schedule_prep(&queue->napi_tx)) {  in macb_interrupt()
1952 				netdev_vdbg(bp->dev, "scheduling TX softirq\n");  in macb_interrupt()
1953 				__napi_schedule(&queue->napi_tx);  in macb_interrupt()
1959 			schedule_work(&queue->tx_error_task);  in macb_interrupt()
1961 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1968 		 * add that if/when we get our hands on a full-blown MII PHY.  in macb_interrupt()
1971 		/* There is a hardware issue under heavy load where DMA can  in macb_interrupt()
1973 		 * interrupts but it can be cleared by re-enabling RX. See  in macb_interrupt()
1984 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1991 				bp->hw_stats.gem.rx_overruns++;  in macb_interrupt()
1993 				bp->hw_stats.macb.rx_overruns++;  in macb_interrupt()
1995 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
2000 			queue_work(system_bh_wq, &bp->hresp_err_bh_work);  in macb_interrupt()
2003 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
2009 	spin_unlock(&bp->lock);  in macb_interrupt()
2015 /* Polling receive - used by netconsole and other diagnostic tools
2026 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_poll_controller()
2027 		macb_interrupt(dev->irq, queue);  in macb_poll_controller()
2038 	unsigned int len, entry, i, tx_head = queue->tx_head;  in macb_tx_map()
2042 	unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;  in macb_tx_map()
2047 	if (skb_shinfo(skb)->gso_size != 0) {  in macb_tx_map()
2048 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)  in macb_tx_map()
2049 			/* UDP - UFO */  in macb_tx_map()
2052 			/* TCP - TSO */  in macb_tx_map()
2056 	/* First, map non-paged data */  in macb_tx_map()
2065 		tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2067 		mapping = dma_map_single(&bp->pdev->dev,  in macb_tx_map()
2068 					 skb->data + offset,  in macb_tx_map()
2070 		if (dma_mapping_error(&bp->pdev->dev, mapping))  in macb_tx_map()
2074 		tx_skb->skb = NULL;  in macb_tx_map()
2075 		tx_skb->mapping = mapping;  in macb_tx_map()
2076 		tx_skb->size = size;  in macb_tx_map()
2077 		tx_skb->mapped_as_page = false;  in macb_tx_map()
2079 		len -= size;  in macb_tx_map()
2084 		size = min(len, bp->max_tx_length);  in macb_tx_map()
2089 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];  in macb_tx_map()
2094 			size = min(len, bp->max_tx_length);  in macb_tx_map()
2096 			tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2098 			mapping = skb_frag_dma_map(&bp->pdev->dev, frag,  in macb_tx_map()
2100 			if (dma_mapping_error(&bp->pdev->dev, mapping))  in macb_tx_map()
2104 			tx_skb->skb = NULL;  in macb_tx_map()
2105 			tx_skb->mapping = mapping;  in macb_tx_map()
2106 			tx_skb->size = size;  in macb_tx_map()
2107 			tx_skb->mapped_as_page = true;  in macb_tx_map()
2109 			len -= size;  in macb_tx_map()
2118 		netdev_err(bp->dev, "BUG! empty skb!\n");  in macb_tx_map()
2123 	tx_skb->skb = skb;  in macb_tx_map()
2136 	desc->ctrl = ctrl;  in macb_tx_map()
2141 			mss_mfs = skb_shinfo(skb)->gso_size +  in macb_tx_map()
2145 			mss_mfs = skb_shinfo(skb)->gso_size;  in macb_tx_map()
2147 			 * can be set only for TSO  in macb_tx_map()
2154 		i--;  in macb_tx_map()
2156 		tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2159 		ctrl = (u32)tx_skb->size;  in macb_tx_map()
2164 		if (unlikely(entry == (bp->tx_ring_size - 1)))  in macb_tx_map()
2168 		if (i == queue->tx_head) {  in macb_tx_map()
2171 			if ((bp->dev->features & NETIF_F_HW_CSUM) &&  in macb_tx_map()
2172 			    skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl &&  in macb_tx_map()
2182 		macb_set_addr(bp, desc, tx_skb->mapping);  in macb_tx_map()
2183 		/* desc->addr must be visible to hardware before clearing  in macb_tx_map()
2184 		 * 'TX_USED' bit in desc->ctrl.  in macb_tx_map()
2187 		desc->ctrl = ctrl;  in macb_tx_map()
2188 	} while (i != queue->tx_head);  in macb_tx_map()
2190 	queue->tx_head = tx_head;  in macb_tx_map()
2195 	netdev_err(bp->dev, "TX DMA map failed\n");  in macb_tx_map()
2197 	for (i = queue->tx_head; i != tx_head; i++) {  in macb_tx_map()
2216 	if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP))  in macb_features_check()
2226 	if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))  in macb_features_check()
2229 	nr_frags = skb_shinfo(skb)->nr_frags;  in macb_features_check()
2231 	nr_frags--;  in macb_features_check()
2233 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];  in macb_features_check()
2244 	if (skb->ip_summed != CHECKSUM_PARTIAL)  in macb_clear_csum()
2247 	/* make sure we can modify the header */  in macb_clear_csum()
2249 		return -1;  in macb_clear_csum()
2252 	 * This is required - at least for Zynq, which otherwise calculates  in macb_clear_csum()
2255 	*(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;  in macb_clear_csum()
2263 	int padlen = ETH_ZLEN - (*skb)->len;  in macb_pad_and_fcs()
2268 	if (!(ndev->features & NETIF_F_HW_CSUM) ||  in macb_pad_and_fcs()
2269 	    !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||  in macb_pad_and_fcs()
2270 	    skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb))  in macb_pad_and_fcs()
2288 			return -ENOMEM;  in macb_pad_and_fcs()
2295 		skb_put_zero(*skb, padlen - ETH_FCS_LEN);  in macb_pad_and_fcs()
2299 	fcs = crc32_le(~0, (*skb)->data, (*skb)->len);  in macb_pad_and_fcs()
2314 	struct macb_queue *queue = &bp->queues[queue_index];  in macb_start_xmit()
2331 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&  in macb_start_xmit()
2332 	    (bp->hw_dma_cap & HW_DMA_CAP_PTP))  in macb_start_xmit()
2333 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;  in macb_start_xmit()
2336 	is_lso = (skb_shinfo(skb)->gso_size != 0);  in macb_start_xmit()
2340 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)  in macb_start_xmit()
2346 			netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");  in macb_start_xmit()
2351 		hdrlen = min(skb_headlen(skb), bp->max_tx_length);  in macb_start_xmit()
2354 	netdev_vdbg(bp->dev,  in macb_start_xmit()
2356 		    queue_index, skb->len, skb->head, skb->data,  in macb_start_xmit()
2359 		       skb->data, 16, true);  in macb_start_xmit()
2368 		desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;  in macb_start_xmit()
2370 		desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);  in macb_start_xmit()
2371 	nr_frags = skb_shinfo(skb)->nr_frags;  in macb_start_xmit()
2373 		frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);  in macb_start_xmit()
2374 		desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);  in macb_start_xmit()
2377 	spin_lock_bh(&queue->tx_ptr_lock);  in macb_start_xmit()
2380 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail,  in macb_start_xmit()
2381 		       bp->tx_ring_size) < desc_cnt) {  in macb_start_xmit()
2383 		netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",  in macb_start_xmit()
2384 			   queue->tx_head, queue->tx_tail);  in macb_start_xmit()
2399 	spin_lock_irq(&bp->lock);  in macb_start_xmit()
2401 	spin_unlock_irq(&bp->lock);  in macb_start_xmit()
2403 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)  in macb_start_xmit()
2407 	spin_unlock_bh(&queue->tx_ptr_lock);  in macb_start_xmit()
2415 		bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;  in macb_init_rx_buffer_size()
2417 		bp->rx_buffer_size = size;  in macb_init_rx_buffer_size()
2419 		if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {  in macb_init_rx_buffer_size()
2420 			netdev_dbg(bp->dev,  in macb_init_rx_buffer_size()
2423 			bp->rx_buffer_size =  in macb_init_rx_buffer_size()
2424 				roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);  in macb_init_rx_buffer_size()
2428 	netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",  in macb_init_rx_buffer_size()
2429 		   bp->dev->mtu, bp->rx_buffer_size);  in macb_init_rx_buffer_size()
2441 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_free_rx_buffers()
2442 		if (!queue->rx_skbuff)  in gem_free_rx_buffers()
2445 		for (i = 0; i < bp->rx_ring_size; i++) {  in gem_free_rx_buffers()
2446 			skb = queue->rx_skbuff[i];  in gem_free_rx_buffers()
2454 			dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,  in gem_free_rx_buffers()
2460 		kfree(queue->rx_skbuff);  in gem_free_rx_buffers()
2461 		queue->rx_skbuff = NULL;  in gem_free_rx_buffers()
2467 	struct macb_queue *queue = &bp->queues[0];  in macb_free_rx_buffers()
2469 	if (queue->rx_buffers) {  in macb_free_rx_buffers()
2470 		dma_free_coherent(&bp->pdev->dev,  in macb_free_rx_buffers()
2471 				  bp->rx_ring_size * bp->rx_buffer_size,  in macb_free_rx_buffers()
2472 				  queue->rx_buffers, queue->rx_buffers_dma);  in macb_free_rx_buffers()
2473 		queue->rx_buffers = NULL;  in macb_free_rx_buffers()
2483 	if (bp->rx_ring_tieoff) {  in macb_free_consistent()
2484 		dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp),  in macb_free_consistent()
2485 				  bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma);  in macb_free_consistent()
2486 		bp->rx_ring_tieoff = NULL;  in macb_free_consistent()
2489 	bp->macbgem_ops.mog_free_rx_buffers(bp);  in macb_free_consistent()
2491 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_free_consistent()
2492 		kfree(queue->tx_skb);  in macb_free_consistent()
2493 		queue->tx_skb = NULL;  in macb_free_consistent()
2494 		if (queue->tx_ring) {  in macb_free_consistent()
2495 			size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;  in macb_free_consistent()
2496 			dma_free_coherent(&bp->pdev->dev, size,  in macb_free_consistent()
2497 					  queue->tx_ring, queue->tx_ring_dma);  in macb_free_consistent()
2498 			queue->tx_ring = NULL;  in macb_free_consistent()
2500 		if (queue->rx_ring) {  in macb_free_consistent()
2501 			size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;  in macb_free_consistent()
2502 			dma_free_coherent(&bp->pdev->dev, size,  in macb_free_consistent()
2503 					  queue->rx_ring, queue->rx_ring_dma);  in macb_free_consistent()
2504 			queue->rx_ring = NULL;  in macb_free_consistent()
2515 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_alloc_rx_buffers()
2516 		size = bp->rx_ring_size * sizeof(struct sk_buff *);  in gem_alloc_rx_buffers()
2517 		queue->rx_skbuff = kzalloc(size, GFP_KERNEL);  in gem_alloc_rx_buffers()
2518 		if (!queue->rx_skbuff)  in gem_alloc_rx_buffers()
2519 			return -ENOMEM;  in gem_alloc_rx_buffers()
2521 			netdev_dbg(bp->dev,  in gem_alloc_rx_buffers()
2523 				   bp->rx_ring_size, queue->rx_skbuff);  in gem_alloc_rx_buffers()
2530 	struct macb_queue *queue = &bp->queues[0];  in macb_alloc_rx_buffers()
2533 	size = bp->rx_ring_size * bp->rx_buffer_size;  in macb_alloc_rx_buffers()
2534 	queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_rx_buffers()
2535 					    &queue->rx_buffers_dma, GFP_KERNEL);  in macb_alloc_rx_buffers()
2536 	if (!queue->rx_buffers)  in macb_alloc_rx_buffers()
2537 		return -ENOMEM;  in macb_alloc_rx_buffers()
2539 	netdev_dbg(bp->dev,  in macb_alloc_rx_buffers()
2541 		   size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);  in macb_alloc_rx_buffers()
2551 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_alloc_consistent()
2552 		size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;  in macb_alloc_consistent()
2553 		queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_consistent()
2554 						    &queue->tx_ring_dma,  in macb_alloc_consistent()
2556 		if (!queue->tx_ring)  in macb_alloc_consistent()
2558 		netdev_dbg(bp->dev,  in macb_alloc_consistent()
2560 			   q, size, (unsigned long)queue->tx_ring_dma,  in macb_alloc_consistent()
2561 			   queue->tx_ring);  in macb_alloc_consistent()
2563 		size = bp->tx_ring_size * sizeof(struct macb_tx_skb);  in macb_alloc_consistent()
2564 		queue->tx_skb = kmalloc(size, GFP_KERNEL);  in macb_alloc_consistent()
2565 		if (!queue->tx_skb)  in macb_alloc_consistent()
2568 		size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;  in macb_alloc_consistent()
2569 		queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_consistent()
2570 						 &queue->rx_ring_dma, GFP_KERNEL);  in macb_alloc_consistent()
2571 		if (!queue->rx_ring)  in macb_alloc_consistent()
2573 		netdev_dbg(bp->dev,  in macb_alloc_consistent()
2575 			   size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);  in macb_alloc_consistent()
2577 	if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))  in macb_alloc_consistent()
2581 	if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) {  in macb_alloc_consistent()
2582 		bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev,  in macb_alloc_consistent()
2584 							&bp->rx_ring_tieoff_dma,  in macb_alloc_consistent()
2586 		if (!bp->rx_ring_tieoff)  in macb_alloc_consistent()
2594 	return -ENOMEM;  in macb_alloc_consistent()
2599 	struct macb_dma_desc *desc = bp->rx_ring_tieoff;  in macb_init_tieoff()
2601 	if (bp->caps & MACB_CAPS_QUEUE_DISABLE)  in macb_init_tieoff()
2607 	desc->ctrl = 0;  in macb_init_tieoff()
2617 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_init_rings()
2618 		for (i = 0; i < bp->tx_ring_size; i++) {  in gem_init_rings()
2621 			desc->ctrl = MACB_BIT(TX_USED);  in gem_init_rings()
2623 		desc->ctrl |= MACB_BIT(TX_WRAP);  in gem_init_rings()
2624 		queue->tx_head = 0;  in gem_init_rings()
2625 		queue->tx_tail = 0;  in gem_init_rings()
2627 		queue->rx_tail = 0;  in gem_init_rings()
2628 		queue->rx_prepared_head = 0;  in gem_init_rings()
2641 	macb_init_rx_ring(&bp->queues[0]);  in macb_init_rings()
2643 	for (i = 0; i < bp->tx_ring_size; i++) {  in macb_init_rings()
2644 		desc = macb_tx_desc(&bp->queues[0], i);  in macb_init_rings()
2646 		desc->ctrl = MACB_BIT(TX_USED);  in macb_init_rings()
2648 	bp->queues[0].tx_head = 0;  in macb_init_rings()
2649 	bp->queues[0].tx_tail = 0;  in macb_init_rings()
2650 	desc->ctrl |= MACB_BIT(TX_WRAP);  in macb_init_rings()
2672 	macb_writel(bp, TSR, -1);  in macb_reset_hw()
2673 	macb_writel(bp, RSR, -1);  in macb_reset_hw()
2679 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_reset_hw()
2680 		queue_writel(queue, IDR, -1);  in macb_reset_hw()
2682 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_reset_hw()
2683 			queue_writel(queue, ISR, -1);  in macb_reset_hw()
2690 	unsigned long pclk_hz = clk_get_rate(bp->pclk);  in gem_mdc_clk_div()
2720 	pclk_hz = clk_get_rate(bp->pclk);  in macb_mdc_clk_div()
2754  * - use the correct receive buffer size
2755  * - set best burst length for DMA operations
2757  * - set both rx/tx packet buffers to full memory size
2767 	buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;  in macb_configure_dma()
2769 		dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);  in macb_configure_dma()
2770 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_configure_dma()
2776 		if (bp->dma_burst_length)  in macb_configure_dma()
2777 			dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);  in macb_configure_dma()
2778 		dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);  in macb_configure_dma()
2781 		if (bp->native_io)  in macb_configure_dma()
2786 		if (bp->dev->features & NETIF_F_HW_CSUM)  in macb_configure_dma()
2793 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_configure_dma()
2797 		if (bp->hw_dma_cap & HW_DMA_CAP_PTP)  in macb_configure_dma()
2800 		netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",  in macb_configure_dma()
2816 	if (bp->caps & MACB_CAPS_JUMBO)  in macb_init_hw()
2820 	if (bp->dev->flags & IFF_PROMISC)  in macb_init_hw()
2822 	else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)  in macb_init_hw()
2824 	if (!(bp->dev->flags & IFF_BROADCAST))  in macb_init_hw()
2828 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)  in macb_init_hw()
2829 		gem_writel(bp, JML, bp->jumbo_max_len);  in macb_init_hw()
2830 	bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;  in macb_init_hw()
2831 	if (bp->caps & MACB_CAPS_JUMBO)  in macb_init_hw()
2832 		bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;  in macb_init_hw()
2837 	if (bp->rx_watermark)  in macb_init_hw()
2838 		gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU)));  in macb_init_hw()
2897 /* Add multicast addresses to the internal multicast-hash table. */
2909 		bitnr = hash_get_index(ha->addr);  in macb_sethashtable()
2925 	if (dev->flags & IFF_PROMISC) {  in macb_set_rx_mode()
2937 		if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)  in macb_set_rx_mode()
2941 	if (dev->flags & IFF_ALLMULTI) {  in macb_set_rx_mode()
2943 		macb_or_gem_writel(bp, HRB, -1);  in macb_set_rx_mode()
2944 		macb_or_gem_writel(bp, HRT, -1);  in macb_set_rx_mode()
2950 	} else if (dev->flags & (~IFF_ALLMULTI)) {  in macb_set_rx_mode()
2962 	size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;  in macb_open()
2968 	netdev_dbg(bp->dev, "open\n");  in macb_open()
2970 	err = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_open()
2984 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_open()
2985 		napi_enable(&queue->napi_rx);  in macb_open()
2986 		napi_enable(&queue->napi_tx);  in macb_open()
2991 	err = phy_power_on(bp->sgmii_phy);  in macb_open()
3001 	if (bp->ptp_info)  in macb_open()
3002 		bp->ptp_info->ptp_init(dev);  in macb_open()
3007 	phy_power_off(bp->sgmii_phy);  in macb_open()
3011 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_open()
3012 		napi_disable(&queue->napi_rx);  in macb_open()
3013 		napi_disable(&queue->napi_tx);  in macb_open()
3017 	pm_runtime_put_sync(&bp->pdev->dev);  in macb_open()
3030 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_close()
3031 		napi_disable(&queue->napi_rx);  in macb_close()
3032 		napi_disable(&queue->napi_tx);  in macb_close()
3035 	phylink_stop(bp->phylink);  in macb_close()
3036 	phylink_disconnect_phy(bp->phylink);  in macb_close()
3038 	phy_power_off(bp->sgmii_phy);  in macb_close()
3040 	spin_lock_irqsave(&bp->lock, flags);  in macb_close()
3043 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_close()
3047 	if (bp->ptp_info)  in macb_close()
3048 		bp->ptp_info->ptp_remove(dev);  in macb_close()
3050 	pm_runtime_put(&bp->pdev->dev);  in macb_close()
3058 		return -EBUSY;  in macb_change_mtu()
3060 	WRITE_ONCE(dev->mtu, new_mtu);  in macb_change_mtu()
3083 	u32 *p = &bp->hw_stats.gem.tx_octets_31_0;  in gem_update_stats()
3087 		u64 val = bp->macb_reg_readl(bp, offset);  in gem_update_stats()
3089 		bp->ethtool_stats[i] += val;  in gem_update_stats()
3094 			val = bp->macb_reg_readl(bp, offset + 4);  in gem_update_stats()
3095 			bp->ethtool_stats[i] += ((u64)val) << 32;  in gem_update_stats()
3101 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in gem_update_stats()
3102 		for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)  in gem_update_stats()
3103 			bp->ethtool_stats[idx++] = *stat;  in gem_update_stats()
3108 	struct gem_stats *hwstat = &bp->hw_stats.gem;  in gem_get_stats()
3109 	struct net_device_stats *nstat = &bp->dev->stats;  in gem_get_stats()
3111 	if (!netif_running(bp->dev))  in gem_get_stats()
3116 	nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +  in gem_get_stats()
3117 			    hwstat->rx_alignment_errors +  in gem_get_stats()
3118 			    hwstat->rx_resource_errors +  in gem_get_stats()
3119 			    hwstat->rx_overruns +  in gem_get_stats()
3120 			    hwstat->rx_oversize_frames +  in gem_get_stats()
3121 			    hwstat->rx_jabbers +  in gem_get_stats()
3122 			    hwstat->rx_undersized_frames +  in gem_get_stats()
3123 			    hwstat->rx_length_field_frame_errors);  in gem_get_stats()
3124 	nstat->tx_errors = (hwstat->tx_late_collisions +  in gem_get_stats()
3125 			    hwstat->tx_excessive_collisions +  in gem_get_stats()
3126 			    hwstat->tx_underrun +  in gem_get_stats()
3127 			    hwstat->tx_carrier_sense_errors);  in gem_get_stats()
3128 	nstat->multicast = hwstat->rx_multicast_frames;  in gem_get_stats()
3129 	nstat->collisions = (hwstat->tx_single_collision_frames +  in gem_get_stats()
3130 			     hwstat->tx_multiple_collision_frames +  in gem_get_stats()
3131 			     hwstat->tx_excessive_collisions);  in gem_get_stats()
3132 	nstat->rx_length_errors = (hwstat->rx_oversize_frames +  in gem_get_stats()
3133 				   hwstat->rx_jabbers +  in gem_get_stats()
3134 				   hwstat->rx_undersized_frames +  in gem_get_stats()
3135 				   hwstat->rx_length_field_frame_errors);  in gem_get_stats()
3136 	nstat->rx_over_errors = hwstat->rx_resource_errors;  in gem_get_stats()
3137 	nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;  in gem_get_stats()
3138 	nstat->rx_frame_errors = hwstat->rx_alignment_errors;  in gem_get_stats()
3139 	nstat->rx_fifo_errors = hwstat->rx_overruns;  in gem_get_stats()
3140 	nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;  in gem_get_stats()
3141 	nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;  in gem_get_stats()
3142 	nstat->tx_fifo_errors = hwstat->tx_underrun;  in gem_get_stats()
3154 	memcpy(data, &bp->ethtool_stats, sizeof(u64)  in gem_get_ethtool_stats()
3164 		return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;  in gem_get_sset_count()
3166 		return -EOPNOTSUPP;  in gem_get_sset_count()
3184 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_get_ethtool_strings()
3198 	struct net_device_stats *nstat = &bp->dev->stats;  in macb_get_stats()
3199 	struct macb_stats *hwstat = &bp->hw_stats.macb;  in macb_get_stats()
3208 	nstat->rx_errors = (hwstat->rx_fcs_errors +  in macb_get_stats()
3209 			    hwstat->rx_align_errors +  in macb_get_stats()
3210 			    hwstat->rx_resource_errors +  in macb_get_stats()
3211 			    hwstat->rx_overruns +  in macb_get_stats()
3212 			    hwstat->rx_oversize_pkts +  in macb_get_stats()
3213 			    hwstat->rx_jabbers +  in macb_get_stats()
3214 			    hwstat->rx_undersize_pkts +  in macb_get_stats()
3215 			    hwstat->rx_length_mismatch);  in macb_get_stats()
3216 	nstat->tx_errors = (hwstat->tx_late_cols +  in macb_get_stats()
3217 			    hwstat->tx_excessive_cols +  in macb_get_stats()
3218 			    hwstat->tx_underruns +  in macb_get_stats()
3219 			    hwstat->tx_carrier_errors +  in macb_get_stats()
3220 			    hwstat->sqe_test_errors);  in macb_get_stats()
3221 	nstat->collisions = (hwstat->tx_single_cols +  in macb_get_stats()
3222 			     hwstat->tx_multiple_cols +  in macb_get_stats()
3223 			     hwstat->tx_excessive_cols);  in macb_get_stats()
3224 	nstat->rx_length_errors = (hwstat->rx_oversize_pkts +  in macb_get_stats()
3225 				   hwstat->rx_jabbers +  in macb_get_stats()
3226 				   hwstat->rx_undersize_pkts +  in macb_get_stats()
3227 				   hwstat->rx_length_mismatch);  in macb_get_stats()
3228 	nstat->rx_over_errors = hwstat->rx_resource_errors +  in macb_get_stats()
3229 				   hwstat->rx_overruns;  in macb_get_stats()
3230 	nstat->rx_crc_errors = hwstat->rx_fcs_errors;  in macb_get_stats()
3231 	nstat->rx_frame_errors = hwstat->rx_align_errors;  in macb_get_stats()
3232 	nstat->rx_fifo_errors = hwstat->rx_overruns;  in macb_get_stats()
3234 	nstat->tx_aborted_errors = hwstat->tx_excessive_cols;  in macb_get_stats()
3235 	nstat->tx_carrier_errors = hwstat->tx_carrier_errors;  in macb_get_stats()
3236 	nstat->tx_fifo_errors = hwstat->tx_underruns;  in macb_get_stats()
3254 	regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))  in macb_get_regs()
3257 	tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);  in macb_get_regs()
3258 	head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);  in macb_get_regs()
3271 	regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);  in macb_get_regs()
3272 	regs_buff[11] = macb_tx_dma(&bp->queues[0], head);  in macb_get_regs()
3274 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_get_regs()
3284 	phylink_ethtool_get_wol(bp->phylink, wol);  in macb_get_wol()
3285 	wol->supported |= (WAKE_MAGIC | WAKE_ARP);  in macb_get_wol()
3288 	wol->wolopts |= bp->wolopts;  in macb_get_wol()
3297 	ret = phylink_ethtool_set_wol(bp->phylink, wol);  in macb_set_wol()
3299 	if (ret && ret != -EOPNOTSUPP)  in macb_set_wol()
3302 	bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0;  in macb_set_wol()
3303 	bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0;  in macb_set_wol()
3304 	bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0;  in macb_set_wol()
3306 	device_set_wakeup_enable(&bp->pdev->dev, bp->wol);  in macb_set_wol()
3316 	return phylink_ethtool_ksettings_get(bp->phylink, kset);  in macb_get_link_ksettings()
3324 	return phylink_ethtool_ksettings_set(bp->phylink, kset);  in macb_set_link_ksettings()
3334 	ring->rx_max_pending = MAX_RX_RING_SIZE;  in macb_get_ringparam()
3335 	ring->tx_max_pending = MAX_TX_RING_SIZE;  in macb_get_ringparam()
3337 	ring->rx_pending = bp->rx_ring_size;  in macb_get_ringparam()
3338 	ring->tx_pending = bp->tx_ring_size;  in macb_get_ringparam()
3350 	if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))  in macb_set_ringparam()
3351 		return -EINVAL;  in macb_set_ringparam()
3353 	new_rx_size = clamp_t(u32, ring->rx_pending,  in macb_set_ringparam()
3357 	new_tx_size = clamp_t(u32, ring->tx_pending,  in macb_set_ringparam()
3361 	if ((new_tx_size == bp->tx_ring_size) &&  in macb_set_ringparam()
3362 	    (new_rx_size == bp->rx_ring_size)) {  in macb_set_ringparam()
3367 	if (netif_running(bp->dev)) {  in macb_set_ringparam()
3369 		macb_close(bp->dev);  in macb_set_ringparam()
3372 	bp->rx_ring_size = new_rx_size;  in macb_set_ringparam()
3373 	bp->tx_ring_size = new_tx_size;  in macb_set_ringparam()
3376 		macb_open(bp->dev);  in macb_set_ringparam()
3387 	tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");  in gem_get_tsu_rate()
3391 	else if (!IS_ERR(bp->pclk)) {  in gem_get_tsu_rate()
3392 		tsu_clk = bp->pclk;  in gem_get_tsu_rate()
3395 		return -ENOTSUPP;  in gem_get_tsu_rate()
3409 	if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {  in gem_get_ts_info()
3414 	info->so_timestamping =  in gem_get_ts_info()
3419 	info->tx_types =  in gem_get_ts_info()
3423 	info->rx_filters =  in gem_get_ts_info()
3427 	if (bp->ptp_clock)  in gem_get_ts_info()
3428 		info->phc_index = ptp_clock_index(bp->ptp_clock);  in gem_get_ts_info()
3449 	if (bp->ptp_info)  in macb_get_ts_info()
3450 		return bp->ptp_info->get_ts_info(netdev, info);  in macb_get_ts_info()
3457 	struct net_device *netdev = bp->dev;  in gem_enable_flow_filters()
3462 	if (!(netdev->features & NETIF_F_NTUPLE))  in gem_enable_flow_filters()
3467 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_enable_flow_filters()
3468 		struct ethtool_rx_flow_spec *fs = &item->fs;  in gem_enable_flow_filters()
3471 		if (fs->location >= num_t2_scr)  in gem_enable_flow_filters()
3474 		t2_scr = gem_readl_n(bp, SCRT2, fs->location);  in gem_enable_flow_filters()
3480 		tp4sp_m = &(fs->m_u.tcp_ip4_spec);  in gem_enable_flow_filters()
3482 		if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))  in gem_enable_flow_filters()
3487 		if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))  in gem_enable_flow_filters()
3492 		if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))  in gem_enable_flow_filters()
3497 		gem_writel_n(bp, SCRT2, fs->location, t2_scr);  in gem_enable_flow_filters()
3504 	uint16_t index = fs->location;  in gem_prog_cmp_regs()
3513 	tp4sp_v = &(fs->h_u.tcp_ip4_spec);  in gem_prog_cmp_regs()
3514 	tp4sp_m = &(fs->m_u.tcp_ip4_spec);  in gem_prog_cmp_regs()
3517 	if (tp4sp_m->ip4src == 0xFFFFFFFF) {  in gem_prog_cmp_regs()
3518 		/* 1st compare reg - IP source address */  in gem_prog_cmp_regs()
3521 		w0 = tp4sp_v->ip4src;  in gem_prog_cmp_regs()
3522 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3531 	if (tp4sp_m->ip4dst == 0xFFFFFFFF) {  in gem_prog_cmp_regs()
3532 		/* 2nd compare reg - IP destination address */  in gem_prog_cmp_regs()
3535 		w0 = tp4sp_v->ip4dst;  in gem_prog_cmp_regs()
3536 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3545 	if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {  in gem_prog_cmp_regs()
3546 		/* 3rd compare reg - source port, destination port */  in gem_prog_cmp_regs()
3550 		if (tp4sp_m->psrc == tp4sp_m->pdst) {  in gem_prog_cmp_regs()
3551 			w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);  in gem_prog_cmp_regs()
3552 			w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);  in gem_prog_cmp_regs()
3553 			w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3557 			w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */  in gem_prog_cmp_regs()
3559 			if (tp4sp_m->psrc == 0xFFFF) { /* src port */  in gem_prog_cmp_regs()
3560 				w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);  in gem_prog_cmp_regs()
3563 				w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);  in gem_prog_cmp_regs()
3573 	t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);  in gem_prog_cmp_regs()
3588 	struct ethtool_rx_flow_spec *fs = &cmd->fs;  in gem_add_flow_filter()
3591 	int ret = -EINVAL;  in gem_add_flow_filter()
3596 		return -ENOMEM;  in gem_add_flow_filter()
3597 	memcpy(&newfs->fs, fs, sizeof(newfs->fs));  in gem_add_flow_filter()
3601 			fs->flow_type, (int)fs->ring_cookie, fs->location,  in gem_add_flow_filter()
3602 			htonl(fs->h_u.tcp_ip4_spec.ip4src),  in gem_add_flow_filter()
3603 			htonl(fs->h_u.tcp_ip4_spec.ip4dst),  in gem_add_flow_filter()
3604 			be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),  in gem_add_flow_filter()
3605 			be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));  in gem_add_flow_filter()
3607 	spin_lock_irqsave(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3610 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_add_flow_filter()
3611 		if (item->fs.location > newfs->fs.location) {  in gem_add_flow_filter()
3612 			list_add_tail(&newfs->list, &item->list);  in gem_add_flow_filter()
3615 		} else if (item->fs.location == fs->location) {  in gem_add_flow_filter()
3617 					fs->location);  in gem_add_flow_filter()
3618 			ret = -EBUSY;  in gem_add_flow_filter()
3623 		list_add_tail(&newfs->list, &bp->rx_fs_list.list);  in gem_add_flow_filter()
3626 	bp->rx_fs_list.count++;  in gem_add_flow_filter()
3630 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3634 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3647 	spin_lock_irqsave(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3649 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_del_flow_filter()
3650 		if (item->fs.location == cmd->fs.location) {  in gem_del_flow_filter()
3652 			fs = &(item->fs);  in gem_del_flow_filter()
3655 					fs->flow_type, (int)fs->ring_cookie, fs->location,  in gem_del_flow_filter()
3656 					htonl(fs->h_u.tcp_ip4_spec.ip4src),  in gem_del_flow_filter()
3657 					htonl(fs->h_u.tcp_ip4_spec.ip4dst),  in gem_del_flow_filter()
3658 					be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),  in gem_del_flow_filter()
3659 					be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));  in gem_del_flow_filter()
3661 			gem_writel_n(bp, SCRT2, fs->location, 0);  in gem_del_flow_filter()
3663 			list_del(&item->list);  in gem_del_flow_filter()
3664 			bp->rx_fs_list.count--;  in gem_del_flow_filter()
3665 			spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3671 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3672 	return -EINVAL;  in gem_del_flow_filter()
3681 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_get_flow_entry()
3682 		if (item->fs.location == cmd->fs.location) {  in gem_get_flow_entry()
3683 			memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));  in gem_get_flow_entry()
3687 	return -EINVAL;  in gem_get_flow_entry()
3697 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_get_all_flow_entries()
3698 		if (cnt == cmd->rule_cnt)  in gem_get_all_flow_entries()
3699 			return -EMSGSIZE;  in gem_get_all_flow_entries()
3700 		rule_locs[cnt] = item->fs.location;  in gem_get_all_flow_entries()
3703 	cmd->data = bp->max_tuples;  in gem_get_all_flow_entries()
3704 	cmd->rule_cnt = cnt;  in gem_get_all_flow_entries()
3715 	switch (cmd->cmd) {  in gem_get_rxnfc()
3717 		cmd->data = bp->num_queues;  in gem_get_rxnfc()
3720 		cmd->rule_cnt = bp->rx_fs_list.count;  in gem_get_rxnfc()
3730 			  "Command parameter %d is not supported\n", cmd->cmd);  in gem_get_rxnfc()
3731 		ret = -EOPNOTSUPP;  in gem_get_rxnfc()
3742 	switch (cmd->cmd) {  in gem_set_rxnfc()
3744 		if ((cmd->fs.location >= bp->max_tuples)  in gem_set_rxnfc()
3745 				|| (cmd->fs.ring_cookie >= bp->num_queues)) {  in gem_set_rxnfc()
3746 			ret = -EINVAL;  in gem_set_rxnfc()
3756 			  "Command parameter %d is not supported\n", cmd->cmd);  in gem_set_rxnfc()
3757 		ret = -EOPNOTSUPP;  in gem_set_rxnfc()
3799 		return -EINVAL;  in macb_ioctl()
3801 	return phylink_mii_ioctl(bp->phylink, rq, cmd);  in macb_ioctl()
3810 		return -EINVAL;  in macb_hwtstamp_get()
3812 	if (!bp->ptp_info)  in macb_hwtstamp_get()
3813 		return -EOPNOTSUPP;  in macb_hwtstamp_get()
3815 	return bp->ptp_info->get_hwtst(dev, cfg);  in macb_hwtstamp_get()
3825 		return -EINVAL;  in macb_hwtstamp_set()
3827 	if (!bp->ptp_info)  in macb_hwtstamp_set()
3828 		return -EOPNOTSUPP;  in macb_hwtstamp_set()
3830 	return bp->ptp_info->set_hwtst(dev, cfg, extack);  in macb_hwtstamp_set()
3853 	struct net_device *netdev = bp->dev;  in macb_set_rxcsum_feature()
3860 	if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))  in macb_set_rxcsum_feature()
3881 	netdev_features_t changed = features ^ netdev->features;  in macb_set_features()
3900 	struct net_device *netdev = bp->dev;  in macb_restore_features()
3901 	netdev_features_t features = netdev->features;  in macb_restore_features()
3911 	list_for_each_entry(item, &bp->rx_fs_list.list, list)  in macb_restore_features()
3912 		gem_prog_cmp_regs(bp, &item->fs);  in macb_restore_features()
3945 		bp->caps = dt_conf->caps;  in macb_configure_caps()
3947 	if (hw_is_gem(bp->regs, bp->native_io)) {  in macb_configure_caps()
3948 		bp->caps |= MACB_CAPS_MACB_IS_GEM;  in macb_configure_caps()
3952 			bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;  in macb_configure_caps()
3954 			bp->caps |= MACB_CAPS_PCS;  in macb_configure_caps()
3957 			bp->caps |= MACB_CAPS_HIGH_SPEED;  in macb_configure_caps()
3960 			bp->caps |= MACB_CAPS_FIFO_MODE;  in macb_configure_caps()
3963 				dev_err(&bp->pdev->dev,  in macb_configure_caps()
3967 				bp->hw_dma_cap |= HW_DMA_CAP_PTP;  in macb_configure_caps()
3968 				bp->ptp_info = &gem_ptp_info;  in macb_configure_caps()
3974 	dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);  in macb_configure_caps()
4020 	pdata = dev_get_platdata(&pdev->dev);  in macb_clk_init()
4022 		*pclk = pdata->pclk;  in macb_clk_init()
4023 		*hclk = pdata->hclk;  in macb_clk_init()
4025 		*pclk = devm_clk_get(&pdev->dev, "pclk");  in macb_clk_init()
4026 		*hclk = devm_clk_get(&pdev->dev, "hclk");  in macb_clk_init()
4030 		return dev_err_probe(&pdev->dev,  in macb_clk_init()
4031 				     IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV,  in macb_clk_init()
4035 		return dev_err_probe(&pdev->dev,  in macb_clk_init()
4036 				     IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV,  in macb_clk_init()
4039 	*tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");  in macb_clk_init()
4043 	*rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");  in macb_clk_init()
4047 	*tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");  in macb_clk_init()
4053 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);  in macb_clk_init()
4059 		dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);  in macb_clk_init()
4065 		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);  in macb_clk_init()
4071 		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);  in macb_clk_init()
4077 		dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);  in macb_clk_init()
4107 	bp->tx_ring_size = DEFAULT_TX_RING_SIZE;  in macb_init()
4108 	bp->rx_ring_size = DEFAULT_RX_RING_SIZE;  in macb_init()
4115 		if (!(bp->queue_mask & (1 << hw_q)))  in macb_init()
4118 		queue = &bp->queues[q];  in macb_init()
4119 		queue->bp = bp;  in macb_init()
4120 		spin_lock_init(&queue->tx_ptr_lock);  in macb_init()
4121 		netif_napi_add(dev, &queue->napi_rx, macb_rx_poll);  in macb_init()
4122 		netif_napi_add(dev, &queue->napi_tx, macb_tx_poll);  in macb_init()
4124 			queue->ISR  = GEM_ISR(hw_q - 1);  in macb_init()
4125 			queue->IER  = GEM_IER(hw_q - 1);  in macb_init()
4126 			queue->IDR  = GEM_IDR(hw_q - 1);  in macb_init()
4127 			queue->IMR  = GEM_IMR(hw_q - 1);  in macb_init()
4128 			queue->TBQP = GEM_TBQP(hw_q - 1);  in macb_init()
4129 			queue->RBQP = GEM_RBQP(hw_q - 1);  in macb_init()
4130 			queue->RBQS = GEM_RBQS(hw_q - 1);  in macb_init()
4132 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_init()
4133 				queue->TBQPH = GEM_TBQPH(hw_q - 1);  in macb_init()
4134 				queue->RBQPH = GEM_RBQPH(hw_q - 1);  in macb_init()
4139 			queue->ISR  = MACB_ISR;  in macb_init()
4140 			queue->IER  = MACB_IER;  in macb_init()
4141 			queue->IDR  = MACB_IDR;  in macb_init()
4142 			queue->IMR  = MACB_IMR;  in macb_init()
4143 			queue->TBQP = MACB_TBQP;  in macb_init()
4144 			queue->RBQP = MACB_RBQP;  in macb_init()
4146 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_init()
4147 				queue->TBQPH = MACB_TBQPH;  in macb_init()
4148 				queue->RBQPH = MACB_RBQPH;  in macb_init()
4158 		queue->irq = platform_get_irq(pdev, q);  in macb_init()
4159 		err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,  in macb_init()
4160 				       IRQF_SHARED, dev->name, queue);  in macb_init()
4162 			dev_err(&pdev->dev,  in macb_init()
4164 				queue->irq, err);  in macb_init()
4168 		INIT_WORK(&queue->tx_error_task, macb_tx_error_task);  in macb_init()
4172 	dev->netdev_ops = &macb_netdev_ops;  in macb_init()
4176 		bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;  in macb_init()
4177 		bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;  in macb_init()
4178 		bp->macbgem_ops.mog_init_rings = gem_init_rings;  in macb_init()
4179 		bp->macbgem_ops.mog_rx = gem_rx;  in macb_init()
4180 		dev->ethtool_ops = &gem_ethtool_ops;  in macb_init()
4182 		bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;  in macb_init()
4183 		bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;  in macb_init()
4184 		bp->macbgem_ops.mog_init_rings = macb_init_rings;  in macb_init()
4185 		bp->macbgem_ops.mog_rx = macb_rx;  in macb_init()
4186 		dev->ethtool_ops = &macb_ethtool_ops;  in macb_init()
4191 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;  in macb_init()
4194 	dev->hw_features = NETIF_F_SG;  in macb_init()
4198 		dev->hw_features |= MACB_NETIF_LSO;  in macb_init()
4201 	if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))  in macb_init()
4202 		dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;  in macb_init()
4203 	if (bp->caps & MACB_CAPS_SG_DISABLED)  in macb_init()
4204 		dev->hw_features &= ~NETIF_F_SG;  in macb_init()
4205 	dev->features = dev->hw_features;  in macb_init()
4209 	 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs  in macb_init()
4212 	bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),  in macb_init()
4214 	INIT_LIST_HEAD(&bp->rx_fs_list.list);  in macb_init()
4215 	if (bp->max_tuples > 0) {  in macb_init()
4223 			dev->hw_features |= NETIF_F_NTUPLE;  in macb_init()
4225 			bp->rx_fs_list.count = 0;  in macb_init()
4226 			spin_lock_init(&bp->rx_fs_lock);  in macb_init()
4228 			bp->max_tuples = 0;  in macb_init()
4231 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {  in macb_init()
4233 		if (phy_interface_mode_is_rgmii(bp->phy_interface))  in macb_init()
4234 			val = bp->usrio->rgmii;  in macb_init()
4235 		else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&  in macb_init()
4236 			 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))  in macb_init()
4237 			val = bp->usrio->rmii;  in macb_init()
4238 		else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))  in macb_init()
4239 			val = bp->usrio->mii;  in macb_init()
4241 		if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)  in macb_init()
4242 			val |= bp->usrio->refclk;  in macb_init()
4250 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)  in macb_init()
4274 	struct macb_queue *q = &lp->queues[0];  in at91ether_alloc_coherent()
4276 	q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4279 					 &q->rx_ring_dma, GFP_KERNEL);  in at91ether_alloc_coherent()
4280 	if (!q->rx_ring)  in at91ether_alloc_coherent()
4281 		return -ENOMEM;  in at91ether_alloc_coherent()
4283 	q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4286 					    &q->rx_buffers_dma, GFP_KERNEL);  in at91ether_alloc_coherent()
4287 	if (!q->rx_buffers) {  in at91ether_alloc_coherent()
4288 		dma_free_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4291 				  q->rx_ring, q->rx_ring_dma);  in at91ether_alloc_coherent()
4292 		q->rx_ring = NULL;  in at91ether_alloc_coherent()
4293 		return -ENOMEM;  in at91ether_alloc_coherent()
4301 	struct macb_queue *q = &lp->queues[0];  in at91ether_free_coherent()
4303 	if (q->rx_ring) {  in at91ether_free_coherent()
4304 		dma_free_coherent(&lp->pdev->dev,  in at91ether_free_coherent()
4307 				  q->rx_ring, q->rx_ring_dma);  in at91ether_free_coherent()
4308 		q->rx_ring = NULL;  in at91ether_free_coherent()
4311 	if (q->rx_buffers) {  in at91ether_free_coherent()
4312 		dma_free_coherent(&lp->pdev->dev,  in at91ether_free_coherent()
4315 				  q->rx_buffers, q->rx_buffers_dma);  in at91ether_free_coherent()
4316 		q->rx_buffers = NULL;  in at91ether_free_coherent()
4323 	struct macb_queue *q = &lp->queues[0];  in at91ether_start()
4333 	addr = q->rx_buffers_dma;  in at91ether_start()
4337 		desc->ctrl = 0;  in at91ether_start()
4342 	desc->addr |= MACB_BIT(RX_WRAP);  in at91ether_start()
4345 	q->rx_tail = 0;  in at91ether_start()
4348 	macb_writel(lp, RBQP, q->rx_ring_dma);  in at91ether_start()
4394 	ret = pm_runtime_resume_and_get(&lp->pdev->dev);  in at91ether_open()
4419 	pm_runtime_put_sync(&lp->pdev->dev);  in at91ether_open()
4430 	phylink_stop(lp->phylink);  in at91ether_close()
4431 	phylink_disconnect_phy(lp->phylink);  in at91ether_close()
4435 	return pm_runtime_put(&lp->pdev->dev);  in at91ether_close()
4450 		lp->rm9200_txq[desc].skb = skb;  in at91ether_start_xmit()
4451 		lp->rm9200_txq[desc].size = skb->len;  in at91ether_start_xmit()
4452 		lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data,  in at91ether_start_xmit()
4453 							      skb->len, DMA_TO_DEVICE);  in at91ether_start_xmit()
4454 		if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) {  in at91ether_start_xmit()
4456 			dev->stats.tx_dropped++;  in at91ether_start_xmit()
4462 		macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping);  in at91ether_start_xmit()
4464 		macb_writel(lp, TCR, skb->len);  in at91ether_start_xmit()
4480 	struct macb_queue *q = &lp->queues[0];  in at91ether_rx()
4486 	desc = macb_rx_desc(q, q->rx_tail);  in at91ether_rx()
4487 	while (desc->addr & MACB_BIT(RX_USED)) {  in at91ether_rx()
4488 		p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;  in at91ether_rx()
4489 		pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);  in at91ether_rx()
4495 			skb->protocol = eth_type_trans(skb, dev);  in at91ether_rx()
4496 			dev->stats.rx_packets++;  in at91ether_rx()
4497 			dev->stats.rx_bytes += pktlen;  in at91ether_rx()
4500 			dev->stats.rx_dropped++;  in at91ether_rx()
4503 		if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))  in at91ether_rx()
4504 			dev->stats.multicast++;  in at91ether_rx()
4507 		desc->addr &= ~MACB_BIT(RX_USED);  in at91ether_rx()
4510 		if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)  in at91ether_rx()
4511 			q->rx_tail = 0;  in at91ether_rx()
4513 			q->rx_tail++;  in at91ether_rx()
4515 		desc = macb_rx_desc(q, q->rx_tail);  in at91ether_rx()
4540 			dev->stats.tx_errors++;  in at91ether_interrupt()
4543 		if (lp->rm9200_txq[desc].skb) {  in at91ether_interrupt()
4544 			dev_consume_skb_irq(lp->rm9200_txq[desc].skb);  in at91ether_interrupt()
4545 			lp->rm9200_txq[desc].skb = NULL;  in at91ether_interrupt()
4546 			dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping,  in at91ether_interrupt()
4547 					 lp->rm9200_txq[desc].size, DMA_TO_DEVICE);  in at91ether_interrupt()
4548 			dev->stats.tx_packets++;  in at91ether_interrupt()
4549 			dev->stats.tx_bytes += lp->rm9200_txq[desc].size;  in at91ether_interrupt()
4554 	/* Work-around for EMAC Errata section 41.3.1 */  in at91ether_interrupt()
4574 	at91ether_interrupt(dev->irq, dev);  in at91ether_poll_controller()
4606 	*pclk = devm_clk_get(&pdev->dev, "ether_clk");  in at91ether_clk_init()
4612 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);  in at91ether_clk_init()
4625 	bp->queues[0].bp = bp;  in at91ether_init()
4627 	dev->netdev_ops = &at91ether_netdev_ops;  in at91ether_init()
4628 	dev->ethtool_ops = &macb_ethtool_ops;  in at91ether_init()
4630 	err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,  in at91ether_init()
4631 			       0, dev->name, dev);  in at91ether_init()
4645 	return mgmt->rate;  in fu540_macb_tx_recalc_rate()
4678 		iowrite32(1, mgmt->reg);  in fu540_macb_tx_set_rate()
4680 		iowrite32(0, mgmt->reg);  in fu540_macb_tx_set_rate()
4681 	mgmt->rate = rate;  in fu540_macb_tx_set_rate()
4703 	mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);  in fu540_c000_clk_init()
4705 		err = -ENOMEM;  in fu540_c000_clk_init()
4709 	init.name = "sifive-gemgxl-mgmt";  in fu540_c000_clk_init()
4714 	mgmt->rate = 0;  in fu540_c000_clk_init()
4715 	mgmt->hw.init = &init;  in fu540_c000_clk_init()
4717 	*tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);  in fu540_c000_clk_init()
4725 		dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);  in fu540_c000_clk_init()
4729 		dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);  in fu540_c000_clk_init()
4742 	mgmt->reg = devm_platform_ioremap_resource(pdev, 1);  in fu540_c000_init()
4743 	if (IS_ERR(mgmt->reg))  in fu540_c000_init()
4744 		return PTR_ERR(mgmt->reg);  in fu540_c000_init()
4755 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {  in init_reset_optional()
4757 		bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL);  in init_reset_optional()
4759 		if (IS_ERR(bp->sgmii_phy))  in init_reset_optional()
4760 			return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy),  in init_reset_optional()
4763 		ret = phy_init(bp->sgmii_phy);  in init_reset_optional()
4765 			return dev_err_probe(&pdev->dev, ret,  in init_reset_optional()
4772 			ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains",  in init_reset_optional()
4775 				dev_err(&pdev->dev, "Failed to read power management information\n");  in init_reset_optional()
4790 	ret = device_reset_optional(&pdev->dev);  in init_reset_optional()
4792 		phy_exit(bp->sgmii_phy);  in init_reset_optional()
4793 		return dev_err_probe(&pdev->dev, ret, "failed to reset controller");  in init_reset_optional()
4800 		phy_exit(bp->sgmii_phy);  in init_reset_optional()
4958 	{ .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
4960 	{ .compatible = "cdns,np4-macb", .data = &np4_config },
4961 	{ .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
4963 	{ .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
4964 	{ .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
4965 	{ .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },
4966 	{ .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
4967 	{ .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
4968 	{ .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
4969 	{ .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
4971 	{ .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */
4972 	{ .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */
4973 	{ .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
4974 	{ .compatible = "microchip,mpfs-macb", .data = &mpfs_config },
4975 	{ .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
4976 	{ .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
4977 	{ .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config},
4978 	{ .compatible = "xlnx,zynq-gem", .data = &zynq_config },
4979 	{ .compatible = "xlnx,versal-gem", .data = &versal_config},
5001 			struct clk **) = macb_config->clk_init;  in macb_probe()
5002 	int (*init)(struct platform_device *) = macb_config->init;  in macb_probe()
5003 	struct device_node *np = pdev->dev.of_node;  in macb_probe()
5024 		if (match && match->data) {  in macb_probe()
5025 			macb_config = match->data;  in macb_probe()
5026 			clk_init = macb_config->clk_init;  in macb_probe()
5027 			init = macb_config->init;  in macb_probe()
5035 	pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);  in macb_probe()
5036 	pm_runtime_use_autosuspend(&pdev->dev);  in macb_probe()
5037 	pm_runtime_get_noresume(&pdev->dev);  in macb_probe()
5038 	pm_runtime_set_active(&pdev->dev);  in macb_probe()
5039 	pm_runtime_enable(&pdev->dev);  in macb_probe()
5045 		err = -ENOMEM;  in macb_probe()
5049 	dev->base_addr = regs->start;  in macb_probe()
5051 	SET_NETDEV_DEV(dev, &pdev->dev);  in macb_probe()
5054 	bp->pdev = pdev;  in macb_probe()
5055 	bp->dev = dev;  in macb_probe()
5056 	bp->regs = mem;  in macb_probe()
5057 	bp->native_io = native_io;  in macb_probe()
5059 		bp->macb_reg_readl = hw_readl_native;  in macb_probe()
5060 		bp->macb_reg_writel = hw_writel_native;  in macb_probe()
5062 		bp->macb_reg_readl = hw_readl;  in macb_probe()
5063 		bp->macb_reg_writel = hw_writel;  in macb_probe()
5065 	bp->num_queues = num_queues;  in macb_probe()
5066 	bp->queue_mask = queue_mask;  in macb_probe()
5068 		bp->dma_burst_length = macb_config->dma_burst_length;  in macb_probe()
5069 	bp->pclk = pclk;  in macb_probe()
5070 	bp->hclk = hclk;  in macb_probe()
5071 	bp->tx_clk = tx_clk;  in macb_probe()
5072 	bp->rx_clk = rx_clk;  in macb_probe()
5073 	bp->tsu_clk = tsu_clk;  in macb_probe()
5075 		bp->jumbo_max_len = macb_config->jumbo_max_len;  in macb_probe()
5077 	if (!hw_is_gem(bp->regs, bp->native_io))  in macb_probe()
5078 		bp->max_tx_length = MACB_MAX_TX_LEN;  in macb_probe()
5079 	else if (macb_config->max_tx_length)  in macb_probe()
5080 		bp->max_tx_length = macb_config->max_tx_length;  in macb_probe()
5082 		bp->max_tx_length = GEM_MAX_TX_LEN;  in macb_probe()
5084 	bp->wol = 0;  in macb_probe()
5085 	device_set_wakeup_capable(&pdev->dev, 1);  in macb_probe()
5087 	bp->usrio = macb_config->usrio;  in macb_probe()
5093 		err = of_property_read_u32(bp->pdev->dev.of_node,  in macb_probe()
5094 					   "cdns,rx-watermark",  in macb_probe()
5095 					   &bp->rx_watermark);  in macb_probe()
5101 			wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1;  in macb_probe()
5102 			if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) {  in macb_probe()
5103 				dev_info(&bp->pdev->dev, "Invalid watermark value\n");  in macb_probe()
5104 				bp->rx_watermark = 0;  in macb_probe()
5108 	spin_lock_init(&bp->lock);  in macb_probe()
5115 		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));  in macb_probe()
5116 		bp->hw_dma_cap |= HW_DMA_CAP_64B;  in macb_probe()
5121 	dev->irq = platform_get_irq(pdev, 0);  in macb_probe()
5122 	if (dev->irq < 0) {  in macb_probe()
5123 		err = dev->irq;  in macb_probe()
5127 	/* MTU range: 68 - 1518 or 10240 */  in macb_probe()
5128 	dev->min_mtu = GEM_MTU_MIN_SIZE;  in macb_probe()
5129 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)  in macb_probe()
5130 		dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN;  in macb_probe()
5132 		dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN;  in macb_probe()
5134 	if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {  in macb_probe()
5137 			bp->rx_bd_rd_prefetch = (2 << (val - 1)) *  in macb_probe()
5142 			bp->tx_bd_rd_prefetch = (2 << (val - 1)) *  in macb_probe()
5146 	bp->rx_intr_mask = MACB_RX_INT_FLAGS;  in macb_probe()
5147 	if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)  in macb_probe()
5148 		bp->rx_intr_mask |= MACB_BIT(RXUBR);  in macb_probe()
5150 	err = of_get_ethdev_address(np, bp->dev);  in macb_probe()
5151 	if (err == -EPROBE_DEFER)  in macb_probe()
5159 		bp->phy_interface = PHY_INTERFACE_MODE_MII;  in macb_probe()
5161 		bp->phy_interface = interface;  in macb_probe()
5176 		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");  in macb_probe()
5180 	INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task);  in macb_probe()
5184 		    dev->base_addr, dev->irq, dev->dev_addr);  in macb_probe()
5186 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_probe()
5187 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_probe()
5192 	mdiobus_unregister(bp->mii_bus);  in macb_probe()
5193 	mdiobus_free(bp->mii_bus);  in macb_probe()
5196 	phy_exit(bp->sgmii_phy);  in macb_probe()
5203 	pm_runtime_disable(&pdev->dev);  in macb_probe()
5204 	pm_runtime_set_suspended(&pdev->dev);  in macb_probe()
5205 	pm_runtime_dont_use_autosuspend(&pdev->dev);  in macb_probe()
5219 		phy_exit(bp->sgmii_phy);  in macb_remove()
5220 		mdiobus_unregister(bp->mii_bus);  in macb_remove()
5221 		mdiobus_free(bp->mii_bus);  in macb_remove()
5224 		cancel_work_sync(&bp->hresp_err_bh_work);  in macb_remove()
5225 		pm_runtime_disable(&pdev->dev);  in macb_remove()
5226 		pm_runtime_dont_use_autosuspend(&pdev->dev);  in macb_remove()
5227 		if (!pm_runtime_suspended(&pdev->dev)) {  in macb_remove()
5228 			macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk,  in macb_remove()
5229 					  bp->rx_clk, bp->tsu_clk);  in macb_remove()
5230 			pm_runtime_set_suspended(&pdev->dev);  in macb_remove()
5232 		phylink_destroy(bp->phylink);  in macb_remove()
5249 	if (!device_may_wakeup(&bp->dev->dev))  in macb_suspend()
5250 		phy_exit(bp->sgmii_phy);  in macb_suspend()
5255 	if (bp->wol & MACB_WOL_ENABLED) {  in macb_suspend()
5257 		idev = __in_dev_get_rcu(bp->dev);  in macb_suspend()
5259 			ifa = rcu_dereference(idev->ifa_list);  in macb_suspend()
5260 		if ((bp->wolopts & WAKE_ARP) && !ifa) {  in macb_suspend()
5262 			return -EOPNOTSUPP;  in macb_suspend()
5264 		spin_lock_irqsave(&bp->lock, flags);  in macb_suspend()
5271 		for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_suspend()
5274 			if (bp->caps & MACB_CAPS_QUEUE_DISABLE) {  in macb_suspend()
5279 					     lower_32_bits(bp->rx_ring_tieoff_dma));  in macb_suspend()
5282 					     upper_32_bits(bp->rx_ring_tieoff_dma));  in macb_suspend()
5286 			queue_writel(queue, IDR, -1);  in macb_suspend()
5288 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_suspend()
5289 				queue_writel(queue, ISR, -1);  in macb_suspend()
5294 		macb_writel(bp, TSR, -1);  in macb_suspend()
5295 		macb_writel(bp, RSR, -1);  in macb_suspend()
5297 		tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0;  in macb_suspend()
5298 		if (bp->wolopts & WAKE_ARP) {  in macb_suspend()
5301 			tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local));  in macb_suspend()
5307 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);  in macb_suspend()
5309 			err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt,  in macb_suspend()
5310 					       IRQF_SHARED, netdev->name, bp->queues);  in macb_suspend()
5314 					bp->queues[0].irq, err);  in macb_suspend()
5315 				spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5318 			queue_writel(bp->queues, IER, GEM_BIT(WOL));  in macb_suspend()
5321 			err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt,  in macb_suspend()
5322 					       IRQF_SHARED, netdev->name, bp->queues);  in macb_suspend()
5326 					bp->queues[0].irq, err);  in macb_suspend()
5327 				spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5330 			queue_writel(bp->queues, IER, MACB_BIT(WOL));  in macb_suspend()
5333 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5335 		enable_irq_wake(bp->queues[0].irq);  in macb_suspend()
5339 	for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_suspend()
5341 		napi_disable(&queue->napi_rx);  in macb_suspend()
5342 		napi_disable(&queue->napi_tx);  in macb_suspend()
5345 	if (!(bp->wol & MACB_WOL_ENABLED)) {  in macb_suspend()
5347 		phylink_stop(bp->phylink);  in macb_suspend()
5349 		spin_lock_irqsave(&bp->lock, flags);  in macb_suspend()
5351 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5354 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_suspend()
5355 		bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);  in macb_suspend()
5357 	if (netdev->hw_features & NETIF_F_NTUPLE)  in macb_suspend()
5358 		bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);  in macb_suspend()
5360 	if (bp->ptp_info)  in macb_suspend()
5361 		bp->ptp_info->ptp_remove(netdev);  in macb_suspend()
5377 	if (!device_may_wakeup(&bp->dev->dev))  in macb_resume()
5378 		phy_init(bp->sgmii_phy);  in macb_resume()
5386 	if (bp->wol & MACB_WOL_ENABLED) {  in macb_resume()
5387 		spin_lock_irqsave(&bp->lock, flags);  in macb_resume()
5390 			queue_writel(bp->queues, IDR, GEM_BIT(WOL));  in macb_resume()
5393 			queue_writel(bp->queues, IDR, MACB_BIT(WOL));  in macb_resume()
5397 		queue_readl(bp->queues, ISR);  in macb_resume()
5398 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_resume()
5399 			queue_writel(bp->queues, ISR, -1);  in macb_resume()
5401 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);  in macb_resume()
5402 		err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt,  in macb_resume()
5403 				       IRQF_SHARED, netdev->name, bp->queues);  in macb_resume()
5407 				bp->queues[0].irq, err);  in macb_resume()
5408 			spin_unlock_irqrestore(&bp->lock, flags);  in macb_resume()
5411 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_resume()
5413 		disable_irq_wake(bp->queues[0].irq);  in macb_resume()
5419 		phylink_stop(bp->phylink);  in macb_resume()
5423 	for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_resume()
5425 		napi_enable(&queue->napi_rx);  in macb_resume()
5426 		napi_enable(&queue->napi_tx);  in macb_resume()
5429 	if (netdev->hw_features & NETIF_F_NTUPLE)  in macb_resume()
5430 		gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);  in macb_resume()
5432 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_resume()
5433 		macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);  in macb_resume()
5441 	phylink_start(bp->phylink);  in macb_resume()
5445 	if (bp->ptp_info)  in macb_resume()
5446 		bp->ptp_info->ptp_init(netdev);  in macb_resume()
5457 		macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk);  in macb_runtime_suspend()
5458 	else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK))  in macb_runtime_suspend()
5459 		macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk);  in macb_runtime_suspend()
5470 		clk_prepare_enable(bp->pclk);  in macb_runtime_resume()
5471 		clk_prepare_enable(bp->hclk);  in macb_runtime_resume()
5472 		clk_prepare_enable(bp->tx_clk);  in macb_runtime_resume()
5473 		clk_prepare_enable(bp->rx_clk);  in macb_runtime_resume()
5474 		clk_prepare_enable(bp->tsu_clk);  in macb_runtime_resume()
5475 	} else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) {  in macb_runtime_resume()
5476 		clk_prepare_enable(bp->tsu_clk);  in macb_runtime_resume()