Lines Matching +full:rate +full:- +full:lp +full:- +full:ms
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2004-2006 Atmel Corporation
10 #include <linux/clk-provider.h>
23 #include <linux/dma-mapping.h>
37 #include <linux/firmware/xlnx-zynqmp.h>
44 unsigned long rate; member
55 * (bp)->rx_ring_size)
61 * (bp)->tx_ring_size)
64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4)
75 …MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN -…
91 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
94 #define MACB_PM_TIMEOUT 100 /* ms */
130 switch (bp->hw_dma_cap) { in macb_dma_desc_get_size()
155 switch (bp->hw_dma_cap) { in macb_adj_dma_desc_idx()
181 return index & (bp->tx_ring_size - 1); in macb_tx_ring_wrap()
187 index = macb_tx_ring_wrap(queue->bp, index); in macb_tx_desc()
188 index = macb_adj_dma_desc_idx(queue->bp, index); in macb_tx_desc()
189 return &queue->tx_ring[index]; in macb_tx_desc()
195 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; in macb_tx_skb()
202 offset = macb_tx_ring_wrap(queue->bp, index) * in macb_tx_dma()
203 macb_dma_desc_get_size(queue->bp); in macb_tx_dma()
205 return queue->tx_ring_dma + offset; in macb_tx_dma()
210 return index & (bp->rx_ring_size - 1); in macb_rx_ring_wrap()
215 index = macb_rx_ring_wrap(queue->bp, index); in macb_rx_desc()
216 index = macb_adj_dma_desc_idx(queue->bp, index); in macb_rx_desc()
217 return &queue->rx_ring[index]; in macb_rx_desc()
222 return queue->rx_buffers + queue->bp->rx_buffer_size * in macb_rx_buffer()
223 macb_rx_ring_wrap(queue->bp, index); in macb_rx_buffer()
229 return __raw_readl(bp->regs + offset); in hw_readl_native()
234 __raw_writel(value, bp->regs + offset); in hw_writel_native()
239 return readl_relaxed(bp->regs + offset); in hw_readl()
244 writel_relaxed(value, bp->regs + offset); in hw_writel()
281 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); in macb_set_hwaddr()
283 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); in macb_set_hwaddr()
320 eth_hw_addr_set(bp->dev, addr); in macb_get_hwaddr()
325 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); in macb_get_hwaddr()
326 eth_hw_addr_random(bp->dev); in macb_get_hwaddr()
339 struct macb *bp = bus->priv; in macb_mdio_read_c22()
342 status = pm_runtime_resume_and_get(&bp->pdev->dev); in macb_mdio_read_c22()
363 pm_runtime_mark_last_busy(&bp->pdev->dev); in macb_mdio_read_c22()
364 pm_runtime_put_autosuspend(&bp->pdev->dev); in macb_mdio_read_c22()
372 struct macb *bp = bus->priv; in macb_mdio_read_c45()
375 status = pm_runtime_get_sync(&bp->pdev->dev); in macb_mdio_read_c45()
377 pm_runtime_put_noidle(&bp->pdev->dev); in macb_mdio_read_c45()
409 pm_runtime_mark_last_busy(&bp->pdev->dev); in macb_mdio_read_c45()
410 pm_runtime_put_autosuspend(&bp->pdev->dev); in macb_mdio_read_c45()
418 struct macb *bp = bus->priv; in macb_mdio_write_c22()
421 status = pm_runtime_resume_and_get(&bp->pdev->dev); in macb_mdio_write_c22()
441 pm_runtime_mark_last_busy(&bp->pdev->dev); in macb_mdio_write_c22()
442 pm_runtime_put_autosuspend(&bp->pdev->dev); in macb_mdio_write_c22()
451 struct macb *bp = bus->priv; in macb_mdio_write_c45()
454 status = pm_runtime_get_sync(&bp->pdev->dev); in macb_mdio_write_c45()
456 pm_runtime_put_noidle(&bp->pdev->dev); in macb_mdio_write_c45()
487 pm_runtime_mark_last_busy(&bp->pdev->dev); in macb_mdio_write_c45()
488 pm_runtime_put_autosuspend(&bp->pdev->dev); in macb_mdio_write_c45()
498 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_init_buffers()
499 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); in macb_init_buffers()
501 if (bp->hw_dma_cap & HW_DMA_CAP_64B) in macb_init_buffers()
503 upper_32_bits(queue->rx_ring_dma)); in macb_init_buffers()
505 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); in macb_init_buffers()
507 if (bp->hw_dma_cap & HW_DMA_CAP_64B) in macb_init_buffers()
509 upper_32_bits(queue->tx_ring_dma)); in macb_init_buffers()
515 * macb_set_tx_clk() - Set a clock to a new frequency
521 long ferr, rate, rate_rounded; in macb_set_tx_clk() local
523 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) in macb_set_tx_clk()
527 if (bp->phy_interface == PHY_INTERFACE_MODE_MII) in macb_set_tx_clk()
530 rate = rgmii_clock(speed); in macb_set_tx_clk()
531 if (rate < 0) in macb_set_tx_clk()
534 rate_rounded = clk_round_rate(bp->tx_clk, rate); in macb_set_tx_clk()
541 ferr = abs(rate_rounded - rate); in macb_set_tx_clk()
542 ferr = DIV_ROUND_UP(ferr, rate / 100000); in macb_set_tx_clk()
544 netdev_warn(bp->dev, in macb_set_tx_clk()
546 rate); in macb_set_tx_clk()
548 if (clk_set_rate(bp->tx_clk, rate_rounded)) in macb_set_tx_clk()
549 netdev_err(bp->dev, "adjusting tx_clk failed.\n"); in macb_set_tx_clk()
574 state->speed = SPEED_10000; in macb_usx_pcs_get_state()
575 state->duplex = 1; in macb_usx_pcs_get_state()
576 state->an_complete = 1; in macb_usx_pcs_get_state()
579 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); in macb_usx_pcs_get_state()
582 state->pause = MLO_PAUSE_RX; in macb_usx_pcs_get_state()
602 state->link = 0; in macb_pcs_get_state()
634 struct net_device *ndev = to_net_dev(config->dev); in macb_mac_config()
640 spin_lock_irqsave(&bp->lock, flags); in macb_mac_config()
645 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { in macb_mac_config()
646 if (state->interface == PHY_INTERFACE_MODE_RMII) in macb_mac_config()
652 if (state->interface == PHY_INTERFACE_MODE_SGMII) { in macb_mac_config()
654 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) { in macb_mac_config()
657 } else if (bp->caps & MACB_CAPS_MIIONRGMII && in macb_mac_config()
658 bp->phy_interface == PHY_INTERFACE_MODE_MII) { in macb_mac_config()
674 if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) { in macb_mac_config()
686 spin_unlock_irqrestore(&bp->lock, flags); in macb_mac_config()
692 struct net_device *ndev = to_net_dev(config->dev); in macb_mac_link_down()
698 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) in macb_mac_link_down()
699 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) in macb_mac_link_down()
701 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); in macb_mac_link_down()
716 struct net_device *ndev = to_net_dev(config->dev); in macb_mac_link_up()
723 spin_lock_irqsave(&bp->lock, flags); in macb_mac_link_up()
735 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { in macb_mac_link_up()
750 bp->macbgem_ops.mog_init_rings(bp); in macb_mac_link_up()
753 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) in macb_mac_link_up()
755 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); in macb_mac_link_up()
760 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER) in macb_mac_link_up()
764 spin_unlock_irqrestore(&bp->lock, flags); in macb_mac_link_up()
766 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) in macb_mac_link_up()
782 struct net_device *ndev = to_net_dev(config->dev); in macb_mac_select_pcs()
786 return &bp->phylink_usx_pcs; in macb_mac_select_pcs()
788 return &bp->phylink_sgmii_pcs; in macb_mac_select_pcs()
802 dn = of_parse_phandle(dn, "phy-handle", 0); in macb_phy_handle_exists()
809 struct device_node *dn = bp->pdev->dev.of_node; in macb_phylink_connect()
810 struct net_device *dev = bp->dev; in macb_phylink_connect()
815 ret = phylink_of_phy_connect(bp->phylink, dn, 0); in macb_phylink_connect()
818 phydev = phy_find_first(bp->mii_bus); in macb_phylink_connect()
821 return -ENXIO; in macb_phylink_connect()
825 ret = phylink_connect_phy(bp->phylink, phydev); in macb_phylink_connect()
833 phylink_start(bp->phylink); in macb_phylink_connect()
841 struct net_device *ndev = to_net_dev(config->dev); in macb_get_pcs_fixed_state()
844 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; in macb_get_pcs_fixed_state()
852 bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; in macb_mii_probe()
853 bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; in macb_mii_probe()
855 bp->phylink_config.dev = &dev->dev; in macb_mii_probe()
856 bp->phylink_config.type = PHYLINK_NETDEV; in macb_mii_probe()
857 bp->phylink_config.mac_managed_pm = true; in macb_mii_probe()
859 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { in macb_mii_probe()
860 bp->phylink_config.poll_fixed_state = true; in macb_mii_probe()
861 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; in macb_mii_probe()
864 bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | in macb_mii_probe()
868 bp->phylink_config.supported_interfaces); in macb_mii_probe()
870 bp->phylink_config.supported_interfaces); in macb_mii_probe()
873 if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { in macb_mii_probe()
874 bp->phylink_config.mac_capabilities |= MAC_1000FD; in macb_mii_probe()
875 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) in macb_mii_probe()
876 bp->phylink_config.mac_capabilities |= MAC_1000HD; in macb_mii_probe()
879 bp->phylink_config.supported_interfaces); in macb_mii_probe()
880 phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); in macb_mii_probe()
882 if (bp->caps & MACB_CAPS_PCS) in macb_mii_probe()
884 bp->phylink_config.supported_interfaces); in macb_mii_probe()
886 if (bp->caps & MACB_CAPS_HIGH_SPEED) { in macb_mii_probe()
888 bp->phylink_config.supported_interfaces); in macb_mii_probe()
889 bp->phylink_config.mac_capabilities |= MAC_10000FD; in macb_mii_probe()
893 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, in macb_mii_probe()
894 bp->phy_interface, &macb_phylink_ops); in macb_mii_probe()
895 if (IS_ERR(bp->phylink)) { in macb_mii_probe()
897 PTR_ERR(bp->phylink)); in macb_mii_probe()
898 return PTR_ERR(bp->phylink); in macb_mii_probe()
906 struct device_node *child, *np = bp->pdev->dev.of_node; in macb_mdiobus_register()
912 return of_mdiobus_register(bp->mii_bus, mdio_np); in macb_mdiobus_register()
926 return of_mdiobus_register(bp->mii_bus, np); in macb_mdiobus_register()
929 return mdiobus_register(bp->mii_bus); in macb_mdiobus_register()
934 struct device_node *mdio_np, *np = bp->pdev->dev.of_node; in macb_mii_init()
935 int err = -ENXIO; in macb_mii_init()
937 /* With fixed-link, we don't need to register the MDIO bus, in macb_mii_init()
943 return macb_mii_probe(bp->dev); in macb_mii_init()
948 bp->mii_bus = mdiobus_alloc(); in macb_mii_init()
949 if (!bp->mii_bus) { in macb_mii_init()
950 err = -ENOMEM; in macb_mii_init()
954 bp->mii_bus->name = "MACB_mii_bus"; in macb_mii_init()
955 bp->mii_bus->read = &macb_mdio_read_c22; in macb_mii_init()
956 bp->mii_bus->write = &macb_mdio_write_c22; in macb_mii_init()
957 bp->mii_bus->read_c45 = &macb_mdio_read_c45; in macb_mii_init()
958 bp->mii_bus->write_c45 = &macb_mdio_write_c45; in macb_mii_init()
959 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", in macb_mii_init()
960 bp->pdev->name, bp->pdev->id); in macb_mii_init()
961 bp->mii_bus->priv = bp; in macb_mii_init()
962 bp->mii_bus->parent = &bp->pdev->dev; in macb_mii_init()
964 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); in macb_mii_init()
970 err = macb_mii_probe(bp->dev); in macb_mii_init()
977 mdiobus_unregister(bp->mii_bus); in macb_mii_init()
979 mdiobus_free(bp->mii_bus); in macb_mii_init()
988 u64 *p = &bp->hw_stats.macb.rx_pause_frames; in macb_update_stats()
989 u64 *end = &bp->hw_stats.macb.tx_pause_frames + 1; in macb_update_stats()
992 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); in macb_update_stats()
995 *p += bp->macb_reg_readl(bp, offset); in macb_update_stats()
1015 return -ETIMEDOUT; in macb_halt_tx()
1020 if (tx_skb->mapping) { in macb_tx_unmap()
1021 if (tx_skb->mapped_as_page) in macb_tx_unmap()
1022 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, in macb_tx_unmap()
1023 tx_skb->size, DMA_TO_DEVICE); in macb_tx_unmap()
1025 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, in macb_tx_unmap()
1026 tx_skb->size, DMA_TO_DEVICE); in macb_tx_unmap()
1027 tx_skb->mapping = 0; in macb_tx_unmap()
1030 if (tx_skb->skb) { in macb_tx_unmap()
1031 napi_consume_skb(tx_skb->skb, budget); in macb_tx_unmap()
1032 tx_skb->skb = NULL; in macb_tx_unmap()
1041 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { in macb_set_addr()
1043 desc_64->addrh = upper_32_bits(addr); in macb_set_addr()
1051 desc->addr = lower_32_bits(addr); in macb_set_addr()
1060 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { in macb_get_addr()
1062 addr = ((u64)(desc_64->addrh) << 32); in macb_get_addr()
1065 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); in macb_get_addr()
1067 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) in macb_get_addr()
1078 struct macb *bp = queue->bp; in macb_tx_error_task()
1088 queue_index = queue - bp->queues; in macb_tx_error_task()
1089 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", in macb_tx_error_task()
1090 queue_index, queue->tx_tail, queue->tx_head); in macb_tx_error_task()
1098 napi_disable(&queue->napi_tx); in macb_tx_error_task()
1099 spin_lock_irqsave(&bp->lock, flags); in macb_tx_error_task()
1102 netif_tx_stop_all_queues(bp->dev); in macb_tx_error_task()
1109 netdev_err(bp->dev, "BUG: halt tx timed out\n"); in macb_tx_error_task()
1117 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { in macb_tx_error_task()
1121 ctrl = desc->ctrl; in macb_tx_error_task()
1123 skb = tx_skb->skb; in macb_tx_error_task()
1131 skb = tx_skb->skb; in macb_tx_error_task()
1138 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", in macb_tx_error_task()
1140 skb->data); in macb_tx_error_task()
1141 bp->dev->stats.tx_packets++; in macb_tx_error_task()
1142 queue->stats.tx_packets++; in macb_tx_error_task()
1144 bp->dev->stats.tx_bytes += skb->len; in macb_tx_error_task()
1145 queue->stats.tx_bytes += skb->len; in macb_tx_error_task()
1146 bytes += skb->len; in macb_tx_error_task()
1149 /* "Buffers exhausted mid-frame" errors may only happen in macb_tx_error_task()
1154 netdev_err(bp->dev, in macb_tx_error_task()
1155 "BUG: TX buffers exhausted mid-frame\n"); in macb_tx_error_task()
1157 desc->ctrl = ctrl | MACB_BIT(TX_USED); in macb_tx_error_task()
1163 netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), in macb_tx_error_task()
1169 desc->ctrl = MACB_BIT(TX_USED); in macb_tx_error_task()
1175 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); in macb_tx_error_task()
1177 if (bp->hw_dma_cap & HW_DMA_CAP_64B) in macb_tx_error_task()
1178 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); in macb_tx_error_task()
1181 queue->tx_head = 0; in macb_tx_error_task()
1182 queue->tx_tail = 0; in macb_tx_error_task()
1192 netif_tx_start_all_queues(bp->dev); in macb_tx_error_task()
1195 spin_unlock_irqrestore(&bp->lock, flags); in macb_tx_error_task()
1196 napi_enable(&queue->napi_tx); in macb_tx_error_task()
1206 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) in ptp_one_step_sync()
1218 if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) in ptp_one_step_sync()
1231 struct macb *bp = queue->bp; in macb_tx_complete()
1232 u16 queue_index = queue - bp->queues; in macb_tx_complete()
1238 spin_lock(&queue->tx_ptr_lock); in macb_tx_complete()
1239 head = queue->tx_head; in macb_tx_complete()
1240 for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { in macb_tx_complete()
1251 ctrl = desc->ctrl; in macb_tx_complete()
1262 skb = tx_skb->skb; in macb_tx_complete()
1266 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && in macb_tx_complete()
1270 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", in macb_tx_complete()
1272 skb->data); in macb_tx_complete()
1273 bp->dev->stats.tx_packets++; in macb_tx_complete()
1274 queue->stats.tx_packets++; in macb_tx_complete()
1275 bp->dev->stats.tx_bytes += skb->len; in macb_tx_complete()
1276 queue->stats.tx_bytes += skb->len; in macb_tx_complete()
1278 bytes += skb->len; in macb_tx_complete()
1293 netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), in macb_tx_complete()
1296 queue->tx_tail = tail; in macb_tx_complete()
1297 if (__netif_subqueue_stopped(bp->dev, queue_index) && in macb_tx_complete()
1298 CIRC_CNT(queue->tx_head, queue->tx_tail, in macb_tx_complete()
1299 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) in macb_tx_complete()
1300 netif_wake_subqueue(bp->dev, queue_index); in macb_tx_complete()
1301 spin_unlock(&queue->tx_ptr_lock); in macb_tx_complete()
1311 struct macb *bp = queue->bp; in gem_rx_refill()
1314 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, in gem_rx_refill()
1315 bp->rx_ring_size) > 0) { in gem_rx_refill()
1316 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); in gem_rx_refill()
1323 if (!queue->rx_skbuff[entry]) { in gem_rx_refill()
1325 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); in gem_rx_refill()
1327 netdev_err(bp->dev, in gem_rx_refill()
1333 paddr = dma_map_single(&bp->pdev->dev, skb->data, in gem_rx_refill()
1334 bp->rx_buffer_size, in gem_rx_refill()
1336 if (dma_mapping_error(&bp->pdev->dev, paddr)) { in gem_rx_refill()
1341 queue->rx_skbuff[entry] = skb; in gem_rx_refill()
1343 if (entry == bp->rx_ring_size - 1) in gem_rx_refill()
1345 desc->ctrl = 0; in gem_rx_refill()
1355 desc->ctrl = 0; in gem_rx_refill()
1357 desc->addr &= ~MACB_BIT(RX_USED); in gem_rx_refill()
1359 queue->rx_prepared_head++; in gem_rx_refill()
1365 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", in gem_rx_refill()
1366 queue, queue->rx_prepared_head, queue->rx_tail); in gem_rx_refill()
1378 desc->addr &= ~MACB_BIT(RX_USED); in discard_partial_frame()
1393 struct macb *bp = queue->bp; in gem_rx()
1405 entry = macb_rx_ring_wrap(bp, queue->rx_tail); in gem_rx()
1411 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; in gem_rx()
1417 /* Ensure ctrl is at least as up-to-date as rxused */ in gem_rx()
1420 ctrl = desc->ctrl; in gem_rx()
1422 queue->rx_tail++; in gem_rx()
1426 netdev_err(bp->dev, in gem_rx()
1428 bp->dev->stats.rx_dropped++; in gem_rx()
1429 queue->stats.rx_dropped++; in gem_rx()
1432 skb = queue->rx_skbuff[entry]; in gem_rx()
1434 netdev_err(bp->dev, in gem_rx()
1436 bp->dev->stats.rx_dropped++; in gem_rx()
1437 queue->stats.rx_dropped++; in gem_rx()
1441 queue->rx_skbuff[entry] = NULL; in gem_rx()
1442 len = ctrl & bp->rx_frm_len_mask; in gem_rx()
1444 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); in gem_rx()
1447 dma_unmap_single(&bp->pdev->dev, addr, in gem_rx()
1448 bp->rx_buffer_size, DMA_FROM_DEVICE); in gem_rx()
1450 skb->protocol = eth_type_trans(skb, bp->dev); in gem_rx()
1452 if (bp->dev->features & NETIF_F_RXCSUM && in gem_rx()
1453 !(bp->dev->flags & IFF_PROMISC) && in gem_rx()
1455 skb->ip_summed = CHECKSUM_UNNECESSARY; in gem_rx()
1457 bp->dev->stats.rx_packets++; in gem_rx()
1458 queue->stats.rx_packets++; in gem_rx()
1459 bp->dev->stats.rx_bytes += skb->len; in gem_rx()
1460 queue->stats.rx_bytes += skb->len; in gem_rx()
1465 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", in gem_rx()
1466 skb->len, skb->csum); in gem_rx()
1470 skb->data, 32, true); in gem_rx()
1489 struct macb *bp = queue->bp; in macb_rx_frame()
1492 len = desc->ctrl & bp->rx_frm_len_mask; in macb_rx_frame()
1494 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", in macb_rx_frame()
1500 * payload word-aligned. in macb_rx_frame()
1506 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); in macb_rx_frame()
1508 bp->dev->stats.rx_dropped++; in macb_rx_frame()
1511 desc->addr &= ~MACB_BIT(RX_USED); in macb_rx_frame()
1528 unsigned int frag_len = bp->rx_buffer_size; in macb_rx_frame()
1533 return -1; in macb_rx_frame()
1535 frag_len = len - offset; in macb_rx_frame()
1540 offset += bp->rx_buffer_size; in macb_rx_frame()
1542 desc->addr &= ~MACB_BIT(RX_USED); in macb_rx_frame()
1552 skb->protocol = eth_type_trans(skb, bp->dev); in macb_rx_frame()
1554 bp->dev->stats.rx_packets++; in macb_rx_frame()
1555 bp->dev->stats.rx_bytes += skb->len; in macb_rx_frame()
1556 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", in macb_rx_frame()
1557 skb->len, skb->csum); in macb_rx_frame()
1565 struct macb *bp = queue->bp; in macb_init_rx_ring()
1570 addr = queue->rx_buffers_dma; in macb_init_rx_ring()
1571 for (i = 0; i < bp->rx_ring_size; i++) { in macb_init_rx_ring()
1574 desc->ctrl = 0; in macb_init_rx_ring()
1575 addr += bp->rx_buffer_size; in macb_init_rx_ring()
1577 desc->addr |= MACB_BIT(RX_WRAP); in macb_init_rx_ring()
1578 queue->rx_tail = 0; in macb_init_rx_ring()
1584 struct macb *bp = queue->bp; in macb_rx()
1588 int first_frag = -1; in macb_rx()
1590 for (tail = queue->rx_tail; budget > 0; tail++) { in macb_rx()
1597 if (!(desc->addr & MACB_BIT(RX_USED))) in macb_rx()
1600 /* Ensure ctrl is at least as up-to-date as addr */ in macb_rx()
1603 ctrl = desc->ctrl; in macb_rx()
1606 if (first_frag != -1) in macb_rx()
1614 if (unlikely(first_frag == -1)) { in macb_rx()
1620 first_frag = -1; in macb_rx()
1627 budget--; in macb_rx()
1636 netdev_err(bp->dev, "RX queue corruption: reset it\n"); in macb_rx()
1638 spin_lock_irqsave(&bp->lock, flags); in macb_rx()
1644 queue_writel(queue, RBQP, queue->rx_ring_dma); in macb_rx()
1648 spin_unlock_irqrestore(&bp->lock, flags); in macb_rx()
1652 if (first_frag != -1) in macb_rx()
1653 queue->rx_tail = first_frag; in macb_rx()
1655 queue->rx_tail = tail; in macb_rx()
1662 struct macb *bp = queue->bp; in macb_rx_pending()
1666 entry = macb_rx_ring_wrap(bp, queue->rx_tail); in macb_rx_pending()
1672 return (desc->addr & MACB_BIT(RX_USED)) != 0; in macb_rx_pending()
1678 struct macb *bp = queue->bp; in macb_rx_poll()
1681 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); in macb_rx_poll()
1683 netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n", in macb_rx_poll()
1684 (unsigned int)(queue - bp->queues), work_done, budget); in macb_rx_poll()
1687 queue_writel(queue, IER, bp->rx_intr_mask); in macb_rx_poll()
1693 * interrupts are re-enabled. in macb_rx_poll()
1700 queue_writel(queue, IDR, bp->rx_intr_mask); in macb_rx_poll()
1701 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_rx_poll()
1703 netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); in macb_rx_poll()
1715 struct macb *bp = queue->bp; in macb_tx_restart()
1718 spin_lock(&queue->tx_ptr_lock); in macb_tx_restart()
1720 if (queue->tx_head == queue->tx_tail) in macb_tx_restart()
1725 head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); in macb_tx_restart()
1730 spin_lock_irq(&bp->lock); in macb_tx_restart()
1732 spin_unlock_irq(&bp->lock); in macb_tx_restart()
1735 spin_unlock(&queue->tx_ptr_lock); in macb_tx_restart()
1742 spin_lock(&queue->tx_ptr_lock); in macb_tx_complete_pending()
1743 if (queue->tx_head != queue->tx_tail) { in macb_tx_complete_pending()
1747 if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) in macb_tx_complete_pending()
1750 spin_unlock(&queue->tx_ptr_lock); in macb_tx_complete_pending()
1757 struct macb *bp = queue->bp; in macb_tx_poll()
1763 if (queue->txubr_pending) { in macb_tx_poll()
1764 queue->txubr_pending = false; in macb_tx_poll()
1765 netdev_vdbg(bp->dev, "poll: tx restart\n"); in macb_tx_poll()
1769 netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", in macb_tx_poll()
1770 (unsigned int)(queue - bp->queues), work_done, budget); in macb_tx_poll()
1779 * interrupts are re-enabled. in macb_tx_poll()
1787 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_tx_poll()
1789 netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); in macb_tx_poll()
1800 struct net_device *dev = bp->dev; in macb_hresp_error_task()
1805 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_hresp_error_task()
1806 queue_writel(queue, IDR, bp->rx_intr_mask | in macb_hresp_error_task()
1817 bp->macbgem_ops.mog_init_rings(bp); in macb_hresp_error_task()
1823 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) in macb_hresp_error_task()
1825 bp->rx_intr_mask | in macb_hresp_error_task()
1839 struct macb *bp = queue->bp; in macb_wol_interrupt()
1847 spin_lock(&bp->lock); in macb_wol_interrupt()
1852 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", in macb_wol_interrupt()
1853 (unsigned int)(queue - bp->queues), in macb_wol_interrupt()
1855 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_wol_interrupt()
1857 pm_wakeup_event(&bp->pdev->dev, 0); in macb_wol_interrupt()
1860 spin_unlock(&bp->lock); in macb_wol_interrupt()
1868 struct macb *bp = queue->bp; in gem_wol_interrupt()
1876 spin_lock(&bp->lock); in gem_wol_interrupt()
1881 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", in gem_wol_interrupt()
1882 (unsigned int)(queue - bp->queues), in gem_wol_interrupt()
1884 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in gem_wol_interrupt()
1886 pm_wakeup_event(&bp->pdev->dev, 0); in gem_wol_interrupt()
1889 spin_unlock(&bp->lock); in gem_wol_interrupt()
1897 struct macb *bp = queue->bp; in macb_interrupt()
1898 struct net_device *dev = bp->dev; in macb_interrupt()
1906 spin_lock(&bp->lock); in macb_interrupt()
1911 queue_writel(queue, IDR, -1); in macb_interrupt()
1912 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_interrupt()
1913 queue_writel(queue, ISR, -1); in macb_interrupt()
1917 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", in macb_interrupt()
1918 (unsigned int)(queue - bp->queues), in macb_interrupt()
1921 if (status & bp->rx_intr_mask) { in macb_interrupt()
1928 queue_writel(queue, IDR, bp->rx_intr_mask); in macb_interrupt()
1929 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_interrupt()
1932 if (napi_schedule_prep(&queue->napi_rx)) { in macb_interrupt()
1933 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); in macb_interrupt()
1934 __napi_schedule(&queue->napi_rx); in macb_interrupt()
1941 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_interrupt()
1946 queue->txubr_pending = true; in macb_interrupt()
1950 if (napi_schedule_prep(&queue->napi_tx)) { in macb_interrupt()
1951 netdev_vdbg(bp->dev, "scheduling TX softirq\n"); in macb_interrupt()
1952 __napi_schedule(&queue->napi_tx); in macb_interrupt()
1958 schedule_work(&queue->tx_error_task); in macb_interrupt()
1960 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_interrupt()
1967 * add that if/when we get our hands on a full-blown MII PHY. in macb_interrupt()
1972 * interrupts but it can be cleared by re-enabling RX. See in macb_interrupt()
1983 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_interrupt()
1989 spin_lock(&bp->stats_lock); in macb_interrupt()
1991 bp->hw_stats.gem.rx_overruns++; in macb_interrupt()
1993 bp->hw_stats.macb.rx_overruns++; in macb_interrupt()
1994 spin_unlock(&bp->stats_lock); in macb_interrupt()
1996 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_interrupt()
2001 queue_work(system_bh_wq, &bp->hresp_err_bh_work); in macb_interrupt()
2004 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_interrupt()
2010 spin_unlock(&bp->lock); in macb_interrupt()
2016 /* Polling receive - used by netconsole and other diagnostic tools
2027 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) in macb_poll_controller()
2028 macb_interrupt(dev->irq, queue); in macb_poll_controller()
2039 unsigned int len, entry, i, tx_head = queue->tx_head; in macb_tx_map()
2043 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; in macb_tx_map()
2048 if (skb_shinfo(skb)->gso_size != 0) { in macb_tx_map()
2049 if (ip_hdr(skb)->protocol == IPPROTO_UDP) in macb_tx_map()
2050 /* UDP - UFO */ in macb_tx_map()
2053 /* TCP - TSO */ in macb_tx_map()
2057 /* First, map non-paged data */ in macb_tx_map()
2066 tx_skb = &queue->tx_skb[entry]; in macb_tx_map()
2068 mapping = dma_map_single(&bp->pdev->dev, in macb_tx_map()
2069 skb->data + offset, in macb_tx_map()
2071 if (dma_mapping_error(&bp->pdev->dev, mapping)) in macb_tx_map()
2075 tx_skb->skb = NULL; in macb_tx_map()
2076 tx_skb->mapping = mapping; in macb_tx_map()
2077 tx_skb->size = size; in macb_tx_map()
2078 tx_skb->mapped_as_page = false; in macb_tx_map()
2080 len -= size; in macb_tx_map()
2085 size = min(len, bp->max_tx_length); in macb_tx_map()
2090 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; in macb_tx_map()
2095 size = min(len, bp->max_tx_length); in macb_tx_map()
2097 tx_skb = &queue->tx_skb[entry]; in macb_tx_map()
2099 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, in macb_tx_map()
2101 if (dma_mapping_error(&bp->pdev->dev, mapping)) in macb_tx_map()
2105 tx_skb->skb = NULL; in macb_tx_map()
2106 tx_skb->mapping = mapping; in macb_tx_map()
2107 tx_skb->size = size; in macb_tx_map()
2108 tx_skb->mapped_as_page = true; in macb_tx_map()
2110 len -= size; in macb_tx_map()
2119 netdev_err(bp->dev, "BUG! empty skb!\n"); in macb_tx_map()
2124 tx_skb->skb = skb; in macb_tx_map()
2137 desc->ctrl = ctrl; in macb_tx_map()
2142 mss_mfs = skb_shinfo(skb)->gso_size + in macb_tx_map()
2146 mss_mfs = skb_shinfo(skb)->gso_size; in macb_tx_map()
2155 i--; in macb_tx_map()
2157 tx_skb = &queue->tx_skb[entry]; in macb_tx_map()
2160 ctrl = (u32)tx_skb->size; in macb_tx_map()
2165 if (unlikely(entry == (bp->tx_ring_size - 1))) in macb_tx_map()
2169 if (i == queue->tx_head) { in macb_tx_map()
2172 if ((bp->dev->features & NETIF_F_HW_CSUM) && in macb_tx_map()
2173 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && in macb_tx_map()
2183 macb_set_addr(bp, desc, tx_skb->mapping); in macb_tx_map()
2184 /* desc->addr must be visible to hardware before clearing in macb_tx_map()
2185 * 'TX_USED' bit in desc->ctrl. in macb_tx_map()
2188 desc->ctrl = ctrl; in macb_tx_map()
2189 } while (i != queue->tx_head); in macb_tx_map()
2191 queue->tx_head = tx_head; in macb_tx_map()
2196 netdev_err(bp->dev, "TX DMA map failed\n"); in macb_tx_map()
2198 for (i = queue->tx_head; i != tx_head; i++) { in macb_tx_map()
2217 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) in macb_features_check()
2227 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) in macb_features_check()
2230 nr_frags = skb_shinfo(skb)->nr_frags; in macb_features_check()
2232 nr_frags--; in macb_features_check()
2234 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; in macb_features_check()
2245 if (skb->ip_summed != CHECKSUM_PARTIAL) in macb_clear_csum()
2250 return -1; in macb_clear_csum()
2253 * This is required - at least for Zynq, which otherwise calculates in macb_clear_csum()
2256 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; in macb_clear_csum()
2264 int padlen = ETH_ZLEN - (*skb)->len; in macb_pad_and_fcs()
2269 if (!(ndev->features & NETIF_F_HW_CSUM) || in macb_pad_and_fcs()
2270 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || in macb_pad_and_fcs()
2271 skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb)) in macb_pad_and_fcs()
2289 return -ENOMEM; in macb_pad_and_fcs()
2296 skb_put_zero(*skb, padlen - ETH_FCS_LEN); in macb_pad_and_fcs()
2300 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); in macb_pad_and_fcs()
2315 struct macb_queue *queue = &bp->queues[queue_index]; in macb_start_xmit()
2332 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && in macb_start_xmit()
2333 (bp->hw_dma_cap & HW_DMA_CAP_PTP)) in macb_start_xmit()
2334 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in macb_start_xmit()
2337 is_lso = (skb_shinfo(skb)->gso_size != 0); in macb_start_xmit()
2341 if (ip_hdr(skb)->protocol == IPPROTO_UDP) in macb_start_xmit()
2347 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); in macb_start_xmit()
2352 hdrlen = min(skb_headlen(skb), bp->max_tx_length); in macb_start_xmit()
2355 netdev_vdbg(bp->dev, in macb_start_xmit()
2357 queue_index, skb->len, skb->head, skb->data, in macb_start_xmit()
2360 skb->data, 16, true); in macb_start_xmit()
2369 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; in macb_start_xmit()
2371 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); in macb_start_xmit()
2372 nr_frags = skb_shinfo(skb)->nr_frags; in macb_start_xmit()
2374 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); in macb_start_xmit()
2375 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); in macb_start_xmit()
2378 spin_lock_bh(&queue->tx_ptr_lock); in macb_start_xmit()
2381 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, in macb_start_xmit()
2382 bp->tx_ring_size) < desc_cnt) { in macb_start_xmit()
2384 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", in macb_start_xmit()
2385 queue->tx_head, queue->tx_tail); in macb_start_xmit()
2399 netdev_tx_sent_queue(netdev_get_tx_queue(bp->dev, queue_index), in macb_start_xmit()
2400 skb->len); in macb_start_xmit()
2402 spin_lock_irq(&bp->lock); in macb_start_xmit()
2404 spin_unlock_irq(&bp->lock); in macb_start_xmit()
2406 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) in macb_start_xmit()
2410 spin_unlock_bh(&queue->tx_ptr_lock); in macb_start_xmit()
2418 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; in macb_init_rx_buffer_size()
2420 bp->rx_buffer_size = size; in macb_init_rx_buffer_size()
2422 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { in macb_init_rx_buffer_size()
2423 netdev_dbg(bp->dev, in macb_init_rx_buffer_size()
2426 bp->rx_buffer_size = in macb_init_rx_buffer_size()
2427 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); in macb_init_rx_buffer_size()
2431 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", in macb_init_rx_buffer_size()
2432 bp->dev->mtu, bp->rx_buffer_size); in macb_init_rx_buffer_size()
2444 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in gem_free_rx_buffers()
2445 if (!queue->rx_skbuff) in gem_free_rx_buffers()
2448 for (i = 0; i < bp->rx_ring_size; i++) { in gem_free_rx_buffers()
2449 skb = queue->rx_skbuff[i]; in gem_free_rx_buffers()
2457 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, in gem_free_rx_buffers()
2463 kfree(queue->rx_skbuff); in gem_free_rx_buffers()
2464 queue->rx_skbuff = NULL; in gem_free_rx_buffers()
2470 struct macb_queue *queue = &bp->queues[0]; in macb_free_rx_buffers()
2472 if (queue->rx_buffers) { in macb_free_rx_buffers()
2473 dma_free_coherent(&bp->pdev->dev, in macb_free_rx_buffers()
2474 bp->rx_ring_size * bp->rx_buffer_size, in macb_free_rx_buffers()
2475 queue->rx_buffers, queue->rx_buffers_dma); in macb_free_rx_buffers()
2476 queue->rx_buffers = NULL; in macb_free_rx_buffers()
2486 if (bp->rx_ring_tieoff) { in macb_free_consistent()
2487 dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp), in macb_free_consistent()
2488 bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma); in macb_free_consistent()
2489 bp->rx_ring_tieoff = NULL; in macb_free_consistent()
2492 bp->macbgem_ops.mog_free_rx_buffers(bp); in macb_free_consistent()
2494 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_free_consistent()
2495 kfree(queue->tx_skb); in macb_free_consistent()
2496 queue->tx_skb = NULL; in macb_free_consistent()
2497 if (queue->tx_ring) { in macb_free_consistent()
2498 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; in macb_free_consistent()
2499 dma_free_coherent(&bp->pdev->dev, size, in macb_free_consistent()
2500 queue->tx_ring, queue->tx_ring_dma); in macb_free_consistent()
2501 queue->tx_ring = NULL; in macb_free_consistent()
2503 if (queue->rx_ring) { in macb_free_consistent()
2504 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; in macb_free_consistent()
2505 dma_free_coherent(&bp->pdev->dev, size, in macb_free_consistent()
2506 queue->rx_ring, queue->rx_ring_dma); in macb_free_consistent()
2507 queue->rx_ring = NULL; in macb_free_consistent()
2518 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in gem_alloc_rx_buffers()
2519 size = bp->rx_ring_size * sizeof(struct sk_buff *); in gem_alloc_rx_buffers()
2520 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); in gem_alloc_rx_buffers()
2521 if (!queue->rx_skbuff) in gem_alloc_rx_buffers()
2522 return -ENOMEM; in gem_alloc_rx_buffers()
2524 netdev_dbg(bp->dev, in gem_alloc_rx_buffers()
2526 bp->rx_ring_size, queue->rx_skbuff); in gem_alloc_rx_buffers()
2533 struct macb_queue *queue = &bp->queues[0]; in macb_alloc_rx_buffers()
2536 size = bp->rx_ring_size * bp->rx_buffer_size; in macb_alloc_rx_buffers()
2537 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, in macb_alloc_rx_buffers()
2538 &queue->rx_buffers_dma, GFP_KERNEL); in macb_alloc_rx_buffers()
2539 if (!queue->rx_buffers) in macb_alloc_rx_buffers()
2540 return -ENOMEM; in macb_alloc_rx_buffers()
2542 netdev_dbg(bp->dev, in macb_alloc_rx_buffers()
2544 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); in macb_alloc_rx_buffers()
2554 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_alloc_consistent()
2555 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; in macb_alloc_consistent()
2556 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, in macb_alloc_consistent()
2557 &queue->tx_ring_dma, in macb_alloc_consistent()
2559 if (!queue->tx_ring) in macb_alloc_consistent()
2561 netdev_dbg(bp->dev, in macb_alloc_consistent()
2563 q, size, (unsigned long)queue->tx_ring_dma, in macb_alloc_consistent()
2564 queue->tx_ring); in macb_alloc_consistent()
2566 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); in macb_alloc_consistent()
2567 queue->tx_skb = kmalloc(size, GFP_KERNEL); in macb_alloc_consistent()
2568 if (!queue->tx_skb) in macb_alloc_consistent()
2571 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; in macb_alloc_consistent()
2572 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, in macb_alloc_consistent()
2573 &queue->rx_ring_dma, GFP_KERNEL); in macb_alloc_consistent()
2574 if (!queue->rx_ring) in macb_alloc_consistent()
2576 netdev_dbg(bp->dev, in macb_alloc_consistent()
2578 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); in macb_alloc_consistent()
2580 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) in macb_alloc_consistent()
2584 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) { in macb_alloc_consistent()
2585 bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev, in macb_alloc_consistent()
2587 &bp->rx_ring_tieoff_dma, in macb_alloc_consistent()
2589 if (!bp->rx_ring_tieoff) in macb_alloc_consistent()
2597 return -ENOMEM; in macb_alloc_consistent()
2602 struct macb_dma_desc *desc = bp->rx_ring_tieoff; in macb_init_tieoff()
2604 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) in macb_init_tieoff()
2610 desc->ctrl = 0; in macb_init_tieoff()
2620 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in gem_init_rings()
2621 for (i = 0; i < bp->tx_ring_size; i++) { in gem_init_rings()
2624 desc->ctrl = MACB_BIT(TX_USED); in gem_init_rings()
2626 desc->ctrl |= MACB_BIT(TX_WRAP); in gem_init_rings()
2627 queue->tx_head = 0; in gem_init_rings()
2628 queue->tx_tail = 0; in gem_init_rings()
2630 queue->rx_tail = 0; in gem_init_rings()
2631 queue->rx_prepared_head = 0; in gem_init_rings()
2644 macb_init_rx_ring(&bp->queues[0]); in macb_init_rings()
2646 for (i = 0; i < bp->tx_ring_size; i++) { in macb_init_rings()
2647 desc = macb_tx_desc(&bp->queues[0], i); in macb_init_rings()
2649 desc->ctrl = MACB_BIT(TX_USED); in macb_init_rings()
2651 bp->queues[0].tx_head = 0; in macb_init_rings()
2652 bp->queues[0].tx_tail = 0; in macb_init_rings()
2653 desc->ctrl |= MACB_BIT(TX_WRAP); in macb_init_rings()
2675 macb_writel(bp, TSR, -1); in macb_reset_hw()
2676 macb_writel(bp, RSR, -1); in macb_reset_hw()
2682 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_reset_hw()
2683 queue_writel(queue, IDR, -1); in macb_reset_hw()
2685 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_reset_hw()
2686 queue_writel(queue, ISR, -1); in macb_reset_hw()
2693 unsigned long pclk_hz = clk_get_rate(bp->pclk); in gem_mdc_clk_div()
2723 pclk_hz = clk_get_rate(bp->pclk); in macb_mdc_clk_div()
2757 * - use the correct receive buffer size
2758 * - set best burst length for DMA operations
2760 * - set both rx/tx packet buffers to full memory size
2770 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; in macb_configure_dma()
2772 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); in macb_configure_dma()
2773 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_configure_dma()
2779 if (bp->dma_burst_length) in macb_configure_dma()
2780 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); in macb_configure_dma()
2781 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); in macb_configure_dma()
2784 if (bp->native_io) in macb_configure_dma()
2789 if (bp->dev->features & NETIF_F_HW_CSUM) in macb_configure_dma()
2796 if (bp->hw_dma_cap & HW_DMA_CAP_64B) in macb_configure_dma()
2800 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) in macb_configure_dma()
2803 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", in macb_configure_dma()
2819 if (bp->caps & MACB_CAPS_JUMBO) in macb_init_hw()
2823 if (bp->dev->flags & IFF_PROMISC) in macb_init_hw()
2825 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) in macb_init_hw()
2827 if (!(bp->dev->flags & IFF_BROADCAST)) in macb_init_hw()
2831 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) in macb_init_hw()
2832 gem_writel(bp, JML, bp->jumbo_max_len); in macb_init_hw()
2833 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; in macb_init_hw()
2834 if (bp->caps & MACB_CAPS_JUMBO) in macb_init_hw()
2835 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; in macb_init_hw()
2840 if (bp->rx_watermark) in macb_init_hw()
2841 gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU))); in macb_init_hw()
2900 /* Add multicast addresses to the internal multicast-hash table. */
2912 bitnr = hash_get_index(ha->addr); in macb_sethashtable()
2928 if (dev->flags & IFF_PROMISC) { in macb_set_rx_mode()
2940 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) in macb_set_rx_mode()
2944 if (dev->flags & IFF_ALLMULTI) { in macb_set_rx_mode()
2946 macb_or_gem_writel(bp, HRB, -1); in macb_set_rx_mode()
2947 macb_or_gem_writel(bp, HRT, -1); in macb_set_rx_mode()
2953 } else if (dev->flags & (~IFF_ALLMULTI)) { in macb_set_rx_mode()
2965 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; in macb_open()
2971 netdev_dbg(bp->dev, "open\n"); in macb_open()
2973 err = pm_runtime_resume_and_get(&bp->pdev->dev); in macb_open()
2987 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_open()
2988 napi_enable(&queue->napi_rx); in macb_open()
2989 napi_enable(&queue->napi_tx); in macb_open()
2994 err = phy_power_on(bp->sgmii_phy); in macb_open()
3004 if (bp->ptp_info) in macb_open()
3005 bp->ptp_info->ptp_init(dev); in macb_open()
3010 phy_power_off(bp->sgmii_phy); in macb_open()
3014 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_open()
3015 napi_disable(&queue->napi_rx); in macb_open()
3016 napi_disable(&queue->napi_tx); in macb_open()
3020 pm_runtime_put_sync(&bp->pdev->dev); in macb_open()
3033 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in macb_close()
3034 napi_disable(&queue->napi_rx); in macb_close()
3035 napi_disable(&queue->napi_tx); in macb_close()
3039 phylink_stop(bp->phylink); in macb_close()
3040 phylink_disconnect_phy(bp->phylink); in macb_close()
3042 phy_power_off(bp->sgmii_phy); in macb_close()
3044 spin_lock_irqsave(&bp->lock, flags); in macb_close()
3047 spin_unlock_irqrestore(&bp->lock, flags); in macb_close()
3051 if (bp->ptp_info) in macb_close()
3052 bp->ptp_info->ptp_remove(dev); in macb_close()
3054 pm_runtime_put(&bp->pdev->dev); in macb_close()
3062 return -EBUSY; in macb_change_mtu()
3064 WRITE_ONCE(dev->mtu, new_mtu); in macb_change_mtu()
3087 u64 *p = &bp->hw_stats.gem.tx_octets; in gem_update_stats()
3091 u64 val = bp->macb_reg_readl(bp, offset); in gem_update_stats()
3093 bp->ethtool_stats[i] += val; in gem_update_stats()
3098 val = bp->macb_reg_readl(bp, offset + 4); in gem_update_stats()
3099 bp->ethtool_stats[i] += ((u64)val) << 32; in gem_update_stats()
3105 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) in gem_update_stats()
3106 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) in gem_update_stats()
3107 bp->ethtool_stats[idx++] = *stat; in gem_update_stats()
3112 struct gem_stats *hwstat = &bp->hw_stats.gem; in gem_get_stats()
3114 spin_lock_irq(&bp->stats_lock); in gem_get_stats()
3115 if (netif_running(bp->dev)) in gem_get_stats()
3118 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + in gem_get_stats()
3119 hwstat->rx_alignment_errors + in gem_get_stats()
3120 hwstat->rx_resource_errors + in gem_get_stats()
3121 hwstat->rx_overruns + in gem_get_stats()
3122 hwstat->rx_oversize_frames + in gem_get_stats()
3123 hwstat->rx_jabbers + in gem_get_stats()
3124 hwstat->rx_undersized_frames + in gem_get_stats()
3125 hwstat->rx_length_field_frame_errors); in gem_get_stats()
3126 nstat->tx_errors = (hwstat->tx_late_collisions + in gem_get_stats()
3127 hwstat->tx_excessive_collisions + in gem_get_stats()
3128 hwstat->tx_underrun + in gem_get_stats()
3129 hwstat->tx_carrier_sense_errors); in gem_get_stats()
3130 nstat->multicast = hwstat->rx_multicast_frames; in gem_get_stats()
3131 nstat->collisions = (hwstat->tx_single_collision_frames + in gem_get_stats()
3132 hwstat->tx_multiple_collision_frames + in gem_get_stats()
3133 hwstat->tx_excessive_collisions); in gem_get_stats()
3134 nstat->rx_length_errors = (hwstat->rx_oversize_frames + in gem_get_stats()
3135 hwstat->rx_jabbers + in gem_get_stats()
3136 hwstat->rx_undersized_frames + in gem_get_stats()
3137 hwstat->rx_length_field_frame_errors); in gem_get_stats()
3138 nstat->rx_over_errors = hwstat->rx_resource_errors; in gem_get_stats()
3139 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; in gem_get_stats()
3140 nstat->rx_frame_errors = hwstat->rx_alignment_errors; in gem_get_stats()
3141 nstat->rx_fifo_errors = hwstat->rx_overruns; in gem_get_stats()
3142 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; in gem_get_stats()
3143 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; in gem_get_stats()
3144 nstat->tx_fifo_errors = hwstat->tx_underrun; in gem_get_stats()
3145 spin_unlock_irq(&bp->stats_lock); in gem_get_stats()
3153 spin_lock_irq(&bp->stats_lock); in gem_get_ethtool_stats()
3155 memcpy(data, &bp->ethtool_stats, sizeof(u64) in gem_get_ethtool_stats()
3157 spin_unlock_irq(&bp->stats_lock); in gem_get_ethtool_stats()
3166 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; in gem_get_sset_count()
3168 return -EOPNOTSUPP; in gem_get_sset_count()
3186 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { in gem_get_ethtool_strings()
3201 struct macb_stats *hwstat = &bp->hw_stats.macb; in macb_get_stats()
3203 netdev_stats_to_stats64(nstat, &bp->dev->stats); in macb_get_stats()
3210 spin_lock_irq(&bp->stats_lock); in macb_get_stats()
3214 nstat->rx_errors = (hwstat->rx_fcs_errors + in macb_get_stats()
3215 hwstat->rx_align_errors + in macb_get_stats()
3216 hwstat->rx_resource_errors + in macb_get_stats()
3217 hwstat->rx_overruns + in macb_get_stats()
3218 hwstat->rx_oversize_pkts + in macb_get_stats()
3219 hwstat->rx_jabbers + in macb_get_stats()
3220 hwstat->rx_undersize_pkts + in macb_get_stats()
3221 hwstat->rx_length_mismatch); in macb_get_stats()
3222 nstat->tx_errors = (hwstat->tx_late_cols + in macb_get_stats()
3223 hwstat->tx_excessive_cols + in macb_get_stats()
3224 hwstat->tx_underruns + in macb_get_stats()
3225 hwstat->tx_carrier_errors + in macb_get_stats()
3226 hwstat->sqe_test_errors); in macb_get_stats()
3227 nstat->collisions = (hwstat->tx_single_cols + in macb_get_stats()
3228 hwstat->tx_multiple_cols + in macb_get_stats()
3229 hwstat->tx_excessive_cols); in macb_get_stats()
3230 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + in macb_get_stats()
3231 hwstat->rx_jabbers + in macb_get_stats()
3232 hwstat->rx_undersize_pkts + in macb_get_stats()
3233 hwstat->rx_length_mismatch); in macb_get_stats()
3234 nstat->rx_over_errors = hwstat->rx_resource_errors + in macb_get_stats()
3235 hwstat->rx_overruns; in macb_get_stats()
3236 nstat->rx_crc_errors = hwstat->rx_fcs_errors; in macb_get_stats()
3237 nstat->rx_frame_errors = hwstat->rx_align_errors; in macb_get_stats()
3238 nstat->rx_fifo_errors = hwstat->rx_overruns; in macb_get_stats()
3240 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; in macb_get_stats()
3241 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; in macb_get_stats()
3242 nstat->tx_fifo_errors = hwstat->tx_underruns; in macb_get_stats()
3244 spin_unlock_irq(&bp->stats_lock); in macb_get_stats()
3251 struct macb_stats *hwstat = &bp->hw_stats.macb; in macb_get_pause_stats()
3253 spin_lock_irq(&bp->stats_lock); in macb_get_pause_stats()
3255 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; in macb_get_pause_stats()
3256 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; in macb_get_pause_stats()
3257 spin_unlock_irq(&bp->stats_lock); in macb_get_pause_stats()
3264 struct gem_stats *hwstat = &bp->hw_stats.gem; in gem_get_pause_stats()
3266 spin_lock_irq(&bp->stats_lock); in gem_get_pause_stats()
3268 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; in gem_get_pause_stats()
3269 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; in gem_get_pause_stats()
3270 spin_unlock_irq(&bp->stats_lock); in gem_get_pause_stats()
3277 struct macb_stats *hwstat = &bp->hw_stats.macb; in macb_get_eth_mac_stats()
3279 spin_lock_irq(&bp->stats_lock); in macb_get_eth_mac_stats()
3281 mac_stats->FramesTransmittedOK = hwstat->tx_ok; in macb_get_eth_mac_stats()
3282 mac_stats->SingleCollisionFrames = hwstat->tx_single_cols; in macb_get_eth_mac_stats()
3283 mac_stats->MultipleCollisionFrames = hwstat->tx_multiple_cols; in macb_get_eth_mac_stats()
3284 mac_stats->FramesReceivedOK = hwstat->rx_ok; in macb_get_eth_mac_stats()
3285 mac_stats->FrameCheckSequenceErrors = hwstat->rx_fcs_errors; in macb_get_eth_mac_stats()
3286 mac_stats->AlignmentErrors = hwstat->rx_align_errors; in macb_get_eth_mac_stats()
3287 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred; in macb_get_eth_mac_stats()
3288 mac_stats->LateCollisions = hwstat->tx_late_cols; in macb_get_eth_mac_stats()
3289 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_cols; in macb_get_eth_mac_stats()
3290 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underruns; in macb_get_eth_mac_stats()
3291 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_errors; in macb_get_eth_mac_stats()
3292 mac_stats->FramesLostDueToIntMACRcvError = hwstat->rx_overruns; in macb_get_eth_mac_stats()
3293 mac_stats->InRangeLengthErrors = hwstat->rx_length_mismatch; in macb_get_eth_mac_stats()
3294 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_pkts; in macb_get_eth_mac_stats()
3295 spin_unlock_irq(&bp->stats_lock); in macb_get_eth_mac_stats()
3302 struct gem_stats *hwstat = &bp->hw_stats.gem; in gem_get_eth_mac_stats()
3304 spin_lock_irq(&bp->stats_lock); in gem_get_eth_mac_stats()
3306 mac_stats->FramesTransmittedOK = hwstat->tx_frames; in gem_get_eth_mac_stats()
3307 mac_stats->SingleCollisionFrames = hwstat->tx_single_collision_frames; in gem_get_eth_mac_stats()
3308 mac_stats->MultipleCollisionFrames = in gem_get_eth_mac_stats()
3309 hwstat->tx_multiple_collision_frames; in gem_get_eth_mac_stats()
3310 mac_stats->FramesReceivedOK = hwstat->rx_frames; in gem_get_eth_mac_stats()
3311 mac_stats->FrameCheckSequenceErrors = in gem_get_eth_mac_stats()
3312 hwstat->rx_frame_check_sequence_errors; in gem_get_eth_mac_stats()
3313 mac_stats->AlignmentErrors = hwstat->rx_alignment_errors; in gem_get_eth_mac_stats()
3314 mac_stats->OctetsTransmittedOK = hwstat->tx_octets; in gem_get_eth_mac_stats()
3315 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred_frames; in gem_get_eth_mac_stats()
3316 mac_stats->LateCollisions = hwstat->tx_late_collisions; in gem_get_eth_mac_stats()
3317 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_collisions; in gem_get_eth_mac_stats()
3318 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underrun; in gem_get_eth_mac_stats()
3319 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_sense_errors; in gem_get_eth_mac_stats()
3320 mac_stats->OctetsReceivedOK = hwstat->rx_octets; in gem_get_eth_mac_stats()
3321 mac_stats->MulticastFramesXmittedOK = hwstat->tx_multicast_frames; in gem_get_eth_mac_stats()
3322 mac_stats->BroadcastFramesXmittedOK = hwstat->tx_broadcast_frames; in gem_get_eth_mac_stats()
3323 mac_stats->MulticastFramesReceivedOK = hwstat->rx_multicast_frames; in gem_get_eth_mac_stats()
3324 mac_stats->BroadcastFramesReceivedOK = hwstat->rx_broadcast_frames; in gem_get_eth_mac_stats()
3325 mac_stats->InRangeLengthErrors = hwstat->rx_length_field_frame_errors; in gem_get_eth_mac_stats()
3326 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_frames; in gem_get_eth_mac_stats()
3327 spin_unlock_irq(&bp->stats_lock); in gem_get_eth_mac_stats()
3335 struct macb_stats *hwstat = &bp->hw_stats.macb; in macb_get_eth_phy_stats()
3337 spin_lock_irq(&bp->stats_lock); in macb_get_eth_phy_stats()
3339 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; in macb_get_eth_phy_stats()
3340 spin_unlock_irq(&bp->stats_lock); in macb_get_eth_phy_stats()
3347 struct gem_stats *hwstat = &bp->hw_stats.gem; in gem_get_eth_phy_stats()
3349 spin_lock_irq(&bp->stats_lock); in gem_get_eth_phy_stats()
3351 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; in gem_get_eth_phy_stats()
3352 spin_unlock_irq(&bp->stats_lock); in gem_get_eth_phy_stats()
3360 struct macb_stats *hwstat = &bp->hw_stats.macb; in macb_get_rmon_stats()
3362 spin_lock_irq(&bp->stats_lock); in macb_get_rmon_stats()
3364 rmon_stats->undersize_pkts = hwstat->rx_undersize_pkts; in macb_get_rmon_stats()
3365 rmon_stats->oversize_pkts = hwstat->rx_oversize_pkts; in macb_get_rmon_stats()
3366 rmon_stats->jabbers = hwstat->rx_jabbers; in macb_get_rmon_stats()
3367 spin_unlock_irq(&bp->stats_lock); in macb_get_rmon_stats()
3386 struct gem_stats *hwstat = &bp->hw_stats.gem; in gem_get_rmon_stats()
3388 spin_lock_irq(&bp->stats_lock); in gem_get_rmon_stats()
3390 rmon_stats->undersize_pkts = hwstat->rx_undersized_frames; in gem_get_rmon_stats()
3391 rmon_stats->oversize_pkts = hwstat->rx_oversize_frames; in gem_get_rmon_stats()
3392 rmon_stats->jabbers = hwstat->rx_jabbers; in gem_get_rmon_stats()
3393 rmon_stats->hist[0] = hwstat->rx_64_byte_frames; in gem_get_rmon_stats()
3394 rmon_stats->hist[1] = hwstat->rx_65_127_byte_frames; in gem_get_rmon_stats()
3395 rmon_stats->hist[2] = hwstat->rx_128_255_byte_frames; in gem_get_rmon_stats()
3396 rmon_stats->hist[3] = hwstat->rx_256_511_byte_frames; in gem_get_rmon_stats()
3397 rmon_stats->hist[4] = hwstat->rx_512_1023_byte_frames; in gem_get_rmon_stats()
3398 rmon_stats->hist[5] = hwstat->rx_1024_1518_byte_frames; in gem_get_rmon_stats()
3399 rmon_stats->hist[6] = hwstat->rx_greater_than_1518_byte_frames; in gem_get_rmon_stats()
3400 rmon_stats->hist_tx[0] = hwstat->tx_64_byte_frames; in gem_get_rmon_stats()
3401 rmon_stats->hist_tx[1] = hwstat->tx_65_127_byte_frames; in gem_get_rmon_stats()
3402 rmon_stats->hist_tx[2] = hwstat->tx_128_255_byte_frames; in gem_get_rmon_stats()
3403 rmon_stats->hist_tx[3] = hwstat->tx_256_511_byte_frames; in gem_get_rmon_stats()
3404 rmon_stats->hist_tx[4] = hwstat->tx_512_1023_byte_frames; in gem_get_rmon_stats()
3405 rmon_stats->hist_tx[5] = hwstat->tx_1024_1518_byte_frames; in gem_get_rmon_stats()
3406 rmon_stats->hist_tx[6] = hwstat->tx_greater_than_1518_byte_frames; in gem_get_rmon_stats()
3407 spin_unlock_irq(&bp->stats_lock); in gem_get_rmon_stats()
3423 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) in macb_get_regs()
3426 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); in macb_get_regs()
3427 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); in macb_get_regs()
3440 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); in macb_get_regs()
3441 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); in macb_get_regs()
3443 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) in macb_get_regs()
3453 phylink_ethtool_get_wol(bp->phylink, wol); in macb_get_wol()
3454 wol->supported |= (WAKE_MAGIC | WAKE_ARP); in macb_get_wol()
3457 wol->wolopts |= bp->wolopts; in macb_get_wol()
3466 ret = phylink_ethtool_set_wol(bp->phylink, wol); in macb_set_wol()
3468 if (ret && ret != -EOPNOTSUPP) in macb_set_wol()
3471 bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0; in macb_set_wol()
3472 bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0; in macb_set_wol()
3473 bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0; in macb_set_wol()
3475 device_set_wakeup_enable(&bp->pdev->dev, bp->wol); in macb_set_wol()
3485 return phylink_ethtool_ksettings_get(bp->phylink, kset); in macb_get_link_ksettings()
3493 return phylink_ethtool_ksettings_set(bp->phylink, kset); in macb_set_link_ksettings()
3503 ring->rx_max_pending = MAX_RX_RING_SIZE; in macb_get_ringparam()
3504 ring->tx_max_pending = MAX_TX_RING_SIZE; in macb_get_ringparam()
3506 ring->rx_pending = bp->rx_ring_size; in macb_get_ringparam()
3507 ring->tx_pending = bp->tx_ring_size; in macb_get_ringparam()
3519 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) in macb_set_ringparam()
3520 return -EINVAL; in macb_set_ringparam()
3522 new_rx_size = clamp_t(u32, ring->rx_pending, in macb_set_ringparam()
3526 new_tx_size = clamp_t(u32, ring->tx_pending, in macb_set_ringparam()
3530 if ((new_tx_size == bp->tx_ring_size) && in macb_set_ringparam()
3531 (new_rx_size == bp->rx_ring_size)) { in macb_set_ringparam()
3536 if (netif_running(bp->dev)) { in macb_set_ringparam()
3538 macb_close(bp->dev); in macb_set_ringparam()
3541 bp->rx_ring_size = new_rx_size; in macb_set_ringparam()
3542 bp->tx_ring_size = new_tx_size; in macb_set_ringparam()
3545 macb_open(bp->dev); in macb_set_ringparam()
3556 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); in gem_get_tsu_rate()
3560 else if (!IS_ERR(bp->pclk)) { in gem_get_tsu_rate()
3561 tsu_clk = bp->pclk; in gem_get_tsu_rate()
3564 return -ENOTSUPP; in gem_get_tsu_rate()
3578 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { in gem_get_ts_info()
3583 info->so_timestamping = in gem_get_ts_info()
3588 info->tx_types = in gem_get_ts_info()
3592 info->rx_filters = in gem_get_ts_info()
3596 if (bp->ptp_clock) in gem_get_ts_info()
3597 info->phc_index = ptp_clock_index(bp->ptp_clock); in gem_get_ts_info()
3618 if (bp->ptp_info) in macb_get_ts_info()
3619 return bp->ptp_info->get_ts_info(netdev, info); in macb_get_ts_info()
3626 struct net_device *netdev = bp->dev; in gem_enable_flow_filters()
3631 if (!(netdev->features & NETIF_F_NTUPLE)) in gem_enable_flow_filters()
3636 list_for_each_entry(item, &bp->rx_fs_list.list, list) { in gem_enable_flow_filters()
3637 struct ethtool_rx_flow_spec *fs = &item->fs; in gem_enable_flow_filters()
3640 if (fs->location >= num_t2_scr) in gem_enable_flow_filters()
3643 t2_scr = gem_readl_n(bp, SCRT2, fs->location); in gem_enable_flow_filters()
3649 tp4sp_m = &(fs->m_u.tcp_ip4_spec); in gem_enable_flow_filters()
3651 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) in gem_enable_flow_filters()
3656 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) in gem_enable_flow_filters()
3661 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) in gem_enable_flow_filters()
3666 gem_writel_n(bp, SCRT2, fs->location, t2_scr); in gem_enable_flow_filters()
3673 uint16_t index = fs->location; in gem_prog_cmp_regs()
3682 tp4sp_v = &(fs->h_u.tcp_ip4_spec); in gem_prog_cmp_regs()
3683 tp4sp_m = &(fs->m_u.tcp_ip4_spec); in gem_prog_cmp_regs()
3686 if (tp4sp_m->ip4src == 0xFFFFFFFF) { in gem_prog_cmp_regs()
3687 /* 1st compare reg - IP source address */ in gem_prog_cmp_regs()
3690 w0 = tp4sp_v->ip4src; in gem_prog_cmp_regs()
3691 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ in gem_prog_cmp_regs()
3700 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { in gem_prog_cmp_regs()
3701 /* 2nd compare reg - IP destination address */ in gem_prog_cmp_regs()
3704 w0 = tp4sp_v->ip4dst; in gem_prog_cmp_regs()
3705 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ in gem_prog_cmp_regs()
3714 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { in gem_prog_cmp_regs()
3715 /* 3rd compare reg - source port, destination port */ in gem_prog_cmp_regs()
3719 if (tp4sp_m->psrc == tp4sp_m->pdst) { in gem_prog_cmp_regs()
3720 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); in gem_prog_cmp_regs()
3721 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); in gem_prog_cmp_regs()
3722 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ in gem_prog_cmp_regs()
3726 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ in gem_prog_cmp_regs()
3728 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ in gem_prog_cmp_regs()
3729 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); in gem_prog_cmp_regs()
3732 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); in gem_prog_cmp_regs()
3742 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); in gem_prog_cmp_regs()
3757 struct ethtool_rx_flow_spec *fs = &cmd->fs; in gem_add_flow_filter()
3760 int ret = -EINVAL; in gem_add_flow_filter()
3765 return -ENOMEM; in gem_add_flow_filter()
3766 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); in gem_add_flow_filter()
3770 fs->flow_type, (int)fs->ring_cookie, fs->location, in gem_add_flow_filter()
3771 htonl(fs->h_u.tcp_ip4_spec.ip4src), in gem_add_flow_filter()
3772 htonl(fs->h_u.tcp_ip4_spec.ip4dst), in gem_add_flow_filter()
3773 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), in gem_add_flow_filter()
3774 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); in gem_add_flow_filter()
3776 spin_lock_irqsave(&bp->rx_fs_lock, flags); in gem_add_flow_filter()
3779 list_for_each_entry(item, &bp->rx_fs_list.list, list) { in gem_add_flow_filter()
3780 if (item->fs.location > newfs->fs.location) { in gem_add_flow_filter()
3781 list_add_tail(&newfs->list, &item->list); in gem_add_flow_filter()
3784 } else if (item->fs.location == fs->location) { in gem_add_flow_filter()
3786 fs->location); in gem_add_flow_filter()
3787 ret = -EBUSY; in gem_add_flow_filter()
3792 list_add_tail(&newfs->list, &bp->rx_fs_list.list); in gem_add_flow_filter()
3795 bp->rx_fs_list.count++; in gem_add_flow_filter()
3799 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); in gem_add_flow_filter()
3803 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); in gem_add_flow_filter()
3816 spin_lock_irqsave(&bp->rx_fs_lock, flags); in gem_del_flow_filter()
3818 list_for_each_entry(item, &bp->rx_fs_list.list, list) { in gem_del_flow_filter()
3819 if (item->fs.location == cmd->fs.location) { in gem_del_flow_filter()
3821 fs = &(item->fs); in gem_del_flow_filter()
3824 fs->flow_type, (int)fs->ring_cookie, fs->location, in gem_del_flow_filter()
3825 htonl(fs->h_u.tcp_ip4_spec.ip4src), in gem_del_flow_filter()
3826 htonl(fs->h_u.tcp_ip4_spec.ip4dst), in gem_del_flow_filter()
3827 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), in gem_del_flow_filter()
3828 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); in gem_del_flow_filter()
3830 gem_writel_n(bp, SCRT2, fs->location, 0); in gem_del_flow_filter()
3832 list_del(&item->list); in gem_del_flow_filter()
3833 bp->rx_fs_list.count--; in gem_del_flow_filter()
3834 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); in gem_del_flow_filter()
3840 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); in gem_del_flow_filter()
3841 return -EINVAL; in gem_del_flow_filter()
3850 list_for_each_entry(item, &bp->rx_fs_list.list, list) { in gem_get_flow_entry()
3851 if (item->fs.location == cmd->fs.location) { in gem_get_flow_entry()
3852 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); in gem_get_flow_entry()
3856 return -EINVAL; in gem_get_flow_entry()
3866 list_for_each_entry(item, &bp->rx_fs_list.list, list) { in gem_get_all_flow_entries()
3867 if (cnt == cmd->rule_cnt) in gem_get_all_flow_entries()
3868 return -EMSGSIZE; in gem_get_all_flow_entries()
3869 rule_locs[cnt] = item->fs.location; in gem_get_all_flow_entries()
3872 cmd->data = bp->max_tuples; in gem_get_all_flow_entries()
3873 cmd->rule_cnt = cnt; in gem_get_all_flow_entries()
3884 switch (cmd->cmd) { in gem_get_rxnfc()
3886 cmd->data = bp->num_queues; in gem_get_rxnfc()
3889 cmd->rule_cnt = bp->rx_fs_list.count; in gem_get_rxnfc()
3899 "Command parameter %d is not supported\n", cmd->cmd); in gem_get_rxnfc()
3900 ret = -EOPNOTSUPP; in gem_get_rxnfc()
3911 switch (cmd->cmd) { in gem_set_rxnfc()
3913 if ((cmd->fs.location >= bp->max_tuples) in gem_set_rxnfc()
3914 || (cmd->fs.ring_cookie >= bp->num_queues)) { in gem_set_rxnfc()
3915 ret = -EINVAL; in gem_set_rxnfc()
3925 "Command parameter %d is not supported\n", cmd->cmd); in gem_set_rxnfc()
3926 ret = -EOPNOTSUPP; in gem_set_rxnfc()
3976 return -EINVAL; in macb_ioctl()
3978 return phylink_mii_ioctl(bp->phylink, rq, cmd); in macb_ioctl()
3987 return -EINVAL; in macb_hwtstamp_get()
3989 if (!bp->ptp_info) in macb_hwtstamp_get()
3990 return -EOPNOTSUPP; in macb_hwtstamp_get()
3992 return bp->ptp_info->get_hwtst(dev, cfg); in macb_hwtstamp_get()
4002 return -EINVAL; in macb_hwtstamp_set()
4004 if (!bp->ptp_info) in macb_hwtstamp_set()
4005 return -EOPNOTSUPP; in macb_hwtstamp_set()
4007 return bp->ptp_info->set_hwtst(dev, cfg, extack); in macb_hwtstamp_set()
4030 struct net_device *netdev = bp->dev; in macb_set_rxcsum_feature()
4037 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) in macb_set_rxcsum_feature()
4058 netdev_features_t changed = features ^ netdev->features; in macb_set_features()
4077 struct net_device *netdev = bp->dev; in macb_restore_features()
4078 netdev_features_t features = netdev->features; in macb_restore_features()
4088 list_for_each_entry(item, &bp->rx_fs_list.list, list) in macb_restore_features()
4089 gem_prog_cmp_regs(bp, &item->fs); in macb_restore_features()
4122 bp->caps = dt_conf->caps; in macb_configure_caps()
4124 if (hw_is_gem(bp->regs, bp->native_io)) { in macb_configure_caps()
4125 bp->caps |= MACB_CAPS_MACB_IS_GEM; in macb_configure_caps()
4129 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; in macb_configure_caps()
4131 bp->caps |= MACB_CAPS_PCS; in macb_configure_caps()
4134 bp->caps |= MACB_CAPS_HIGH_SPEED; in macb_configure_caps()
4137 bp->caps |= MACB_CAPS_FIFO_MODE; in macb_configure_caps()
4140 dev_err(&bp->pdev->dev, in macb_configure_caps()
4144 bp->hw_dma_cap |= HW_DMA_CAP_PTP; in macb_configure_caps()
4145 bp->ptp_info = &gem_ptp_info; in macb_configure_caps()
4151 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); in macb_configure_caps()
4197 pdata = dev_get_platdata(&pdev->dev); in macb_clk_init()
4199 *pclk = pdata->pclk; in macb_clk_init()
4200 *hclk = pdata->hclk; in macb_clk_init()
4202 *pclk = devm_clk_get(&pdev->dev, "pclk"); in macb_clk_init()
4203 *hclk = devm_clk_get(&pdev->dev, "hclk"); in macb_clk_init()
4207 return dev_err_probe(&pdev->dev, in macb_clk_init()
4208 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, in macb_clk_init()
4212 return dev_err_probe(&pdev->dev, in macb_clk_init()
4213 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, in macb_clk_init()
4216 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); in macb_clk_init()
4220 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); in macb_clk_init()
4224 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); in macb_clk_init()
4230 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); in macb_clk_init()
4236 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); in macb_clk_init()
4242 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); in macb_clk_init()
4248 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); in macb_clk_init()
4254 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); in macb_clk_init()
4284 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; in macb_init()
4285 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; in macb_init()
4292 if (!(bp->queue_mask & (1 << hw_q))) in macb_init()
4295 queue = &bp->queues[q]; in macb_init()
4296 queue->bp = bp; in macb_init()
4297 spin_lock_init(&queue->tx_ptr_lock); in macb_init()
4298 netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); in macb_init()
4299 netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); in macb_init()
4301 queue->ISR = GEM_ISR(hw_q - 1); in macb_init()
4302 queue->IER = GEM_IER(hw_q - 1); in macb_init()
4303 queue->IDR = GEM_IDR(hw_q - 1); in macb_init()
4304 queue->IMR = GEM_IMR(hw_q - 1); in macb_init()
4305 queue->TBQP = GEM_TBQP(hw_q - 1); in macb_init()
4306 queue->RBQP = GEM_RBQP(hw_q - 1); in macb_init()
4307 queue->RBQS = GEM_RBQS(hw_q - 1); in macb_init()
4309 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { in macb_init()
4310 queue->TBQPH = GEM_TBQPH(hw_q - 1); in macb_init()
4311 queue->RBQPH = GEM_RBQPH(hw_q - 1); in macb_init()
4316 queue->ISR = MACB_ISR; in macb_init()
4317 queue->IER = MACB_IER; in macb_init()
4318 queue->IDR = MACB_IDR; in macb_init()
4319 queue->IMR = MACB_IMR; in macb_init()
4320 queue->TBQP = MACB_TBQP; in macb_init()
4321 queue->RBQP = MACB_RBQP; in macb_init()
4323 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { in macb_init()
4324 queue->TBQPH = MACB_TBQPH; in macb_init()
4325 queue->RBQPH = MACB_RBQPH; in macb_init()
4335 queue->irq = platform_get_irq(pdev, q); in macb_init()
4336 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, in macb_init()
4337 IRQF_SHARED, dev->name, queue); in macb_init()
4339 dev_err(&pdev->dev, in macb_init()
4341 queue->irq, err); in macb_init()
4345 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); in macb_init()
4349 dev->netdev_ops = &macb_netdev_ops; in macb_init()
4353 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; in macb_init()
4354 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; in macb_init()
4355 bp->macbgem_ops.mog_init_rings = gem_init_rings; in macb_init()
4356 bp->macbgem_ops.mog_rx = gem_rx; in macb_init()
4357 dev->ethtool_ops = &gem_ethtool_ops; in macb_init()
4359 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; in macb_init()
4360 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; in macb_init()
4361 bp->macbgem_ops.mog_init_rings = macb_init_rings; in macb_init()
4362 bp->macbgem_ops.mog_rx = macb_rx; in macb_init()
4363 dev->ethtool_ops = &macb_ethtool_ops; in macb_init()
4368 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; in macb_init()
4371 dev->hw_features = NETIF_F_SG; in macb_init()
4375 dev->hw_features |= MACB_NETIF_LSO; in macb_init()
4378 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) in macb_init()
4379 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; in macb_init()
4380 if (bp->caps & MACB_CAPS_SG_DISABLED) in macb_init()
4381 dev->hw_features &= ~NETIF_F_SG; in macb_init()
4382 dev->features = dev->hw_features; in macb_init()
4386 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs in macb_init()
4389 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), in macb_init()
4391 INIT_LIST_HEAD(&bp->rx_fs_list.list); in macb_init()
4392 if (bp->max_tuples > 0) { in macb_init()
4400 dev->hw_features |= NETIF_F_NTUPLE; in macb_init()
4402 bp->rx_fs_list.count = 0; in macb_init()
4403 spin_lock_init(&bp->rx_fs_lock); in macb_init()
4405 bp->max_tuples = 0; in macb_init()
4408 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { in macb_init()
4410 if (phy_interface_mode_is_rgmii(bp->phy_interface)) in macb_init()
4411 val = bp->usrio->rgmii; in macb_init()
4412 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && in macb_init()
4413 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) in macb_init()
4414 val = bp->usrio->rmii; in macb_init()
4415 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) in macb_init()
4416 val = bp->usrio->mii; in macb_init()
4418 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) in macb_init()
4419 val |= bp->usrio->refclk; in macb_init()
4427 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) in macb_init()
4449 static int at91ether_alloc_coherent(struct macb *lp) in at91ether_alloc_coherent() argument
4451 struct macb_queue *q = &lp->queues[0]; in at91ether_alloc_coherent()
4453 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, in at91ether_alloc_coherent()
4455 macb_dma_desc_get_size(lp)), in at91ether_alloc_coherent()
4456 &q->rx_ring_dma, GFP_KERNEL); in at91ether_alloc_coherent()
4457 if (!q->rx_ring) in at91ether_alloc_coherent()
4458 return -ENOMEM; in at91ether_alloc_coherent()
4460 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, in at91ether_alloc_coherent()
4463 &q->rx_buffers_dma, GFP_KERNEL); in at91ether_alloc_coherent()
4464 if (!q->rx_buffers) { in at91ether_alloc_coherent()
4465 dma_free_coherent(&lp->pdev->dev, in at91ether_alloc_coherent()
4467 macb_dma_desc_get_size(lp), in at91ether_alloc_coherent()
4468 q->rx_ring, q->rx_ring_dma); in at91ether_alloc_coherent()
4469 q->rx_ring = NULL; in at91ether_alloc_coherent()
4470 return -ENOMEM; in at91ether_alloc_coherent()
4476 static void at91ether_free_coherent(struct macb *lp) in at91ether_free_coherent() argument
4478 struct macb_queue *q = &lp->queues[0]; in at91ether_free_coherent()
4480 if (q->rx_ring) { in at91ether_free_coherent()
4481 dma_free_coherent(&lp->pdev->dev, in at91ether_free_coherent()
4483 macb_dma_desc_get_size(lp), in at91ether_free_coherent()
4484 q->rx_ring, q->rx_ring_dma); in at91ether_free_coherent()
4485 q->rx_ring = NULL; in at91ether_free_coherent()
4488 if (q->rx_buffers) { in at91ether_free_coherent()
4489 dma_free_coherent(&lp->pdev->dev, in at91ether_free_coherent()
4492 q->rx_buffers, q->rx_buffers_dma); in at91ether_free_coherent()
4493 q->rx_buffers = NULL; in at91ether_free_coherent()
4498 static int at91ether_start(struct macb *lp) in at91ether_start() argument
4500 struct macb_queue *q = &lp->queues[0]; in at91ether_start()
4506 ret = at91ether_alloc_coherent(lp); in at91ether_start()
4510 addr = q->rx_buffers_dma; in at91ether_start()
4513 macb_set_addr(lp, desc, addr); in at91ether_start()
4514 desc->ctrl = 0; in at91ether_start()
4519 desc->addr |= MACB_BIT(RX_WRAP); in at91ether_start()
4522 q->rx_tail = 0; in at91ether_start()
4525 macb_writel(lp, RBQP, q->rx_ring_dma); in at91ether_start()
4528 ctl = macb_readl(lp, NCR); in at91ether_start()
4529 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); in at91ether_start()
4532 macb_writel(lp, IER, MACB_BIT(RCOMP) | in at91ether_start()
4543 static void at91ether_stop(struct macb *lp) in at91ether_stop() argument
4548 macb_writel(lp, IDR, MACB_BIT(RCOMP) | in at91ether_stop()
4557 ctl = macb_readl(lp, NCR); in at91ether_stop()
4558 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); in at91ether_stop()
4561 at91ether_free_coherent(lp); in at91ether_stop()
4567 struct macb *lp = netdev_priv(dev); in at91ether_open() local
4571 ret = pm_runtime_resume_and_get(&lp->pdev->dev); in at91ether_open()
4576 ctl = macb_readl(lp, NCR); in at91ether_open()
4577 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); in at91ether_open()
4579 macb_set_hwaddr(lp); in at91ether_open()
4581 ret = at91ether_start(lp); in at91ether_open()
4585 ret = macb_phylink_connect(lp); in at91ether_open()
4594 at91ether_stop(lp); in at91ether_open()
4596 pm_runtime_put_sync(&lp->pdev->dev); in at91ether_open()
4603 struct macb *lp = netdev_priv(dev); in at91ether_close() local
4607 phylink_stop(lp->phylink); in at91ether_close()
4608 phylink_disconnect_phy(lp->phylink); in at91ether_close()
4610 at91ether_stop(lp); in at91ether_close()
4612 return pm_runtime_put(&lp->pdev->dev); in at91ether_close()
4619 struct macb *lp = netdev_priv(dev); in at91ether_start_xmit() local
4621 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { in at91ether_start_xmit()
4627 lp->rm9200_txq[desc].skb = skb; in at91ether_start_xmit()
4628 lp->rm9200_txq[desc].size = skb->len; in at91ether_start_xmit()
4629 lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, in at91ether_start_xmit()
4630 skb->len, DMA_TO_DEVICE); in at91ether_start_xmit()
4631 if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { in at91ether_start_xmit()
4633 dev->stats.tx_dropped++; in at91ether_start_xmit()
4639 macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping); in at91ether_start_xmit()
4641 macb_writel(lp, TCR, skb->len); in at91ether_start_xmit()
4656 struct macb *lp = netdev_priv(dev); in at91ether_rx() local
4657 struct macb_queue *q = &lp->queues[0]; in at91ether_rx()
4663 desc = macb_rx_desc(q, q->rx_tail); in at91ether_rx()
4664 while (desc->addr & MACB_BIT(RX_USED)) { in at91ether_rx()
4665 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; in at91ether_rx()
4666 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); in at91ether_rx()
4672 skb->protocol = eth_type_trans(skb, dev); in at91ether_rx()
4673 dev->stats.rx_packets++; in at91ether_rx()
4674 dev->stats.rx_bytes += pktlen; in at91ether_rx()
4677 dev->stats.rx_dropped++; in at91ether_rx()
4680 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) in at91ether_rx()
4681 dev->stats.multicast++; in at91ether_rx()
4684 desc->addr &= ~MACB_BIT(RX_USED); in at91ether_rx()
4687 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) in at91ether_rx()
4688 q->rx_tail = 0; in at91ether_rx()
4690 q->rx_tail++; in at91ether_rx()
4692 desc = macb_rx_desc(q, q->rx_tail); in at91ether_rx()
4700 struct macb *lp = netdev_priv(dev); in at91ether_interrupt() local
4707 intstatus = macb_readl(lp, ISR); in at91ether_interrupt()
4717 dev->stats.tx_errors++; in at91ether_interrupt()
4720 if (lp->rm9200_txq[desc].skb) { in at91ether_interrupt()
4721 dev_consume_skb_irq(lp->rm9200_txq[desc].skb); in at91ether_interrupt()
4722 lp->rm9200_txq[desc].skb = NULL; in at91ether_interrupt()
4723 dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, in at91ether_interrupt()
4724 lp->rm9200_txq[desc].size, DMA_TO_DEVICE); in at91ether_interrupt()
4725 dev->stats.tx_packets++; in at91ether_interrupt()
4726 dev->stats.tx_bytes += lp->rm9200_txq[desc].size; in at91ether_interrupt()
4731 /* Work-around for EMAC Errata section 41.3.1 */ in at91ether_interrupt()
4733 ctl = macb_readl(lp, NCR); in at91ether_interrupt()
4734 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); in at91ether_interrupt()
4736 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); in at91ether_interrupt()
4751 at91ether_interrupt(dev->irq, dev); in at91ether_poll_controller()
4783 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); in at91ether_clk_init()
4789 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); in at91ether_clk_init()
4802 bp->queues[0].bp = bp; in at91ether_init()
4804 dev->netdev_ops = &at91ether_netdev_ops; in at91ether_init()
4805 dev->ethtool_ops = &macb_ethtool_ops; in at91ether_init()
4807 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, in at91ether_init()
4808 0, dev->name, dev); in at91ether_init()
4822 return mgmt->rate; in fu540_macb_tx_recalc_rate()
4825 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, in fu540_macb_tx_round_rate() argument
4828 if (WARN_ON(rate < 2500000)) in fu540_macb_tx_round_rate()
4830 else if (rate == 2500000) in fu540_macb_tx_round_rate()
4832 else if (WARN_ON(rate < 13750000)) in fu540_macb_tx_round_rate()
4834 else if (WARN_ON(rate < 25000000)) in fu540_macb_tx_round_rate()
4836 else if (rate == 25000000) in fu540_macb_tx_round_rate()
4838 else if (WARN_ON(rate < 75000000)) in fu540_macb_tx_round_rate()
4840 else if (WARN_ON(rate < 125000000)) in fu540_macb_tx_round_rate()
4842 else if (rate == 125000000) in fu540_macb_tx_round_rate()
4845 WARN_ON(rate > 125000000); in fu540_macb_tx_round_rate()
4850 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, in fu540_macb_tx_set_rate() argument
4853 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); in fu540_macb_tx_set_rate()
4854 if (rate != 125000000) in fu540_macb_tx_set_rate()
4855 iowrite32(1, mgmt->reg); in fu540_macb_tx_set_rate()
4857 iowrite32(0, mgmt->reg); in fu540_macb_tx_set_rate()
4858 mgmt->rate = rate; in fu540_macb_tx_set_rate()
4880 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); in fu540_c000_clk_init()
4882 err = -ENOMEM; in fu540_c000_clk_init()
4886 init.name = "sifive-gemgxl-mgmt"; in fu540_c000_clk_init()
4891 mgmt->rate = 0; in fu540_c000_clk_init()
4892 mgmt->hw.init = &init; in fu540_c000_clk_init()
4894 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); in fu540_c000_clk_init()
4902 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); in fu540_c000_clk_init()
4906 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); in fu540_c000_clk_init()
4919 mgmt->reg = devm_platform_ioremap_resource(pdev, 1); in fu540_c000_init()
4920 if (IS_ERR(mgmt->reg)) in fu540_c000_init()
4921 return PTR_ERR(mgmt->reg); in fu540_c000_init()
4932 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { in init_reset_optional()
4934 bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL); in init_reset_optional()
4936 if (IS_ERR(bp->sgmii_phy)) in init_reset_optional()
4937 return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy), in init_reset_optional()
4940 ret = phy_init(bp->sgmii_phy); in init_reset_optional()
4942 return dev_err_probe(&pdev->dev, ret, in init_reset_optional()
4949 ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", in init_reset_optional()
4952 dev_err(&pdev->dev, "Failed to read power management information\n"); in init_reset_optional()
4967 ret = device_reset_optional(&pdev->dev); in init_reset_optional()
4969 phy_exit(bp->sgmii_phy); in init_reset_optional()
4970 return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); in init_reset_optional()
4977 phy_exit(bp->sgmii_phy); in init_reset_optional()
5135 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
5137 { .compatible = "cdns,np4-macb", .data = &np4_config },
5138 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
5140 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
5141 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
5142 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },
5143 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
5144 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
5145 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
5146 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
5148 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */
5149 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */
5150 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
5151 { .compatible = "microchip,mpfs-macb", .data = &mpfs_config },
5152 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
5153 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
5154 { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config},
5155 { .compatible = "xlnx,zynq-gem", .data = &zynq_config },
5156 { .compatible = "xlnx,versal-gem", .data = &versal_config},
5178 struct clk **) = macb_config->clk_init; in macb_probe()
5179 int (*init)(struct platform_device *) = macb_config->init; in macb_probe()
5180 struct device_node *np = pdev->dev.of_node; in macb_probe()
5201 if (match && match->data) { in macb_probe()
5202 macb_config = match->data; in macb_probe()
5203 clk_init = macb_config->clk_init; in macb_probe()
5204 init = macb_config->init; in macb_probe()
5212 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); in macb_probe()
5213 pm_runtime_use_autosuspend(&pdev->dev); in macb_probe()
5214 pm_runtime_get_noresume(&pdev->dev); in macb_probe()
5215 pm_runtime_set_active(&pdev->dev); in macb_probe()
5216 pm_runtime_enable(&pdev->dev); in macb_probe()
5222 err = -ENOMEM; in macb_probe()
5226 dev->base_addr = regs->start; in macb_probe()
5228 SET_NETDEV_DEV(dev, &pdev->dev); in macb_probe()
5231 bp->pdev = pdev; in macb_probe()
5232 bp->dev = dev; in macb_probe()
5233 bp->regs = mem; in macb_probe()
5234 bp->native_io = native_io; in macb_probe()
5236 bp->macb_reg_readl = hw_readl_native; in macb_probe()
5237 bp->macb_reg_writel = hw_writel_native; in macb_probe()
5239 bp->macb_reg_readl = hw_readl; in macb_probe()
5240 bp->macb_reg_writel = hw_writel; in macb_probe()
5242 bp->num_queues = num_queues; in macb_probe()
5243 bp->queue_mask = queue_mask; in macb_probe()
5245 bp->dma_burst_length = macb_config->dma_burst_length; in macb_probe()
5246 bp->pclk = pclk; in macb_probe()
5247 bp->hclk = hclk; in macb_probe()
5248 bp->tx_clk = tx_clk; in macb_probe()
5249 bp->rx_clk = rx_clk; in macb_probe()
5250 bp->tsu_clk = tsu_clk; in macb_probe()
5252 bp->jumbo_max_len = macb_config->jumbo_max_len; in macb_probe()
5254 if (!hw_is_gem(bp->regs, bp->native_io)) in macb_probe()
5255 bp->max_tx_length = MACB_MAX_TX_LEN; in macb_probe()
5256 else if (macb_config->max_tx_length) in macb_probe()
5257 bp->max_tx_length = macb_config->max_tx_length; in macb_probe()
5259 bp->max_tx_length = GEM_MAX_TX_LEN; in macb_probe()
5261 bp->wol = 0; in macb_probe()
5262 device_set_wakeup_capable(&pdev->dev, 1); in macb_probe()
5264 bp->usrio = macb_config->usrio; in macb_probe()
5270 err = of_property_read_u32(bp->pdev->dev.of_node, in macb_probe()
5271 "cdns,rx-watermark", in macb_probe()
5272 &bp->rx_watermark); in macb_probe()
5278 wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1; in macb_probe()
5279 if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) { in macb_probe()
5280 dev_info(&bp->pdev->dev, "Invalid watermark value\n"); in macb_probe()
5281 bp->rx_watermark = 0; in macb_probe()
5285 spin_lock_init(&bp->lock); in macb_probe()
5286 spin_lock_init(&bp->stats_lock); in macb_probe()
5293 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); in macb_probe()
5294 bp->hw_dma_cap |= HW_DMA_CAP_64B; in macb_probe()
5299 dev->irq = platform_get_irq(pdev, 0); in macb_probe()
5300 if (dev->irq < 0) { in macb_probe()
5301 err = dev->irq; in macb_probe()
5305 /* MTU range: 68 - 1518 or 10240 */ in macb_probe()
5306 dev->min_mtu = GEM_MTU_MIN_SIZE; in macb_probe()
5307 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) in macb_probe()
5308 dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN; in macb_probe()
5310 dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; in macb_probe()
5312 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { in macb_probe()
5315 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * in macb_probe()
5320 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * in macb_probe()
5324 bp->rx_intr_mask = MACB_RX_INT_FLAGS; in macb_probe()
5325 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) in macb_probe()
5326 bp->rx_intr_mask |= MACB_BIT(RXUBR); in macb_probe()
5328 err = of_get_ethdev_address(np, bp->dev); in macb_probe()
5329 if (err == -EPROBE_DEFER) in macb_probe()
5337 bp->phy_interface = PHY_INTERFACE_MODE_MII; in macb_probe()
5339 bp->phy_interface = interface; in macb_probe()
5354 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); in macb_probe()
5358 INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task); in macb_probe()
5362 dev->base_addr, dev->irq, dev->dev_addr); in macb_probe()
5364 pm_runtime_mark_last_busy(&bp->pdev->dev); in macb_probe()
5365 pm_runtime_put_autosuspend(&bp->pdev->dev); in macb_probe()
5370 mdiobus_unregister(bp->mii_bus); in macb_probe()
5371 mdiobus_free(bp->mii_bus); in macb_probe()
5374 phy_exit(bp->sgmii_phy); in macb_probe()
5381 pm_runtime_disable(&pdev->dev); in macb_probe()
5382 pm_runtime_set_suspended(&pdev->dev); in macb_probe()
5383 pm_runtime_dont_use_autosuspend(&pdev->dev); in macb_probe()
5397 phy_exit(bp->sgmii_phy); in macb_remove()
5398 mdiobus_unregister(bp->mii_bus); in macb_remove()
5399 mdiobus_free(bp->mii_bus); in macb_remove()
5402 cancel_work_sync(&bp->hresp_err_bh_work); in macb_remove()
5403 pm_runtime_disable(&pdev->dev); in macb_remove()
5404 pm_runtime_dont_use_autosuspend(&pdev->dev); in macb_remove()
5405 if (!pm_runtime_suspended(&pdev->dev)) { in macb_remove()
5406 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, in macb_remove()
5407 bp->rx_clk, bp->tsu_clk); in macb_remove()
5408 pm_runtime_set_suspended(&pdev->dev); in macb_remove()
5410 phylink_destroy(bp->phylink); in macb_remove()
5427 if (!device_may_wakeup(&bp->dev->dev)) in macb_suspend()
5428 phy_exit(bp->sgmii_phy); in macb_suspend()
5433 if (bp->wol & MACB_WOL_ENABLED) { in macb_suspend()
5435 idev = __in_dev_get_rcu(bp->dev); in macb_suspend()
5437 ifa = rcu_dereference(idev->ifa_list); in macb_suspend()
5438 if ((bp->wolopts & WAKE_ARP) && !ifa) { in macb_suspend()
5440 return -EOPNOTSUPP; in macb_suspend()
5442 spin_lock_irqsave(&bp->lock, flags); in macb_suspend()
5449 for (q = 0, queue = bp->queues; q < bp->num_queues; in macb_suspend()
5452 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) { in macb_suspend()
5457 lower_32_bits(bp->rx_ring_tieoff_dma)); in macb_suspend()
5460 upper_32_bits(bp->rx_ring_tieoff_dma)); in macb_suspend()
5464 queue_writel(queue, IDR, -1); in macb_suspend()
5466 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_suspend()
5467 queue_writel(queue, ISR, -1); in macb_suspend()
5472 macb_writel(bp, TSR, -1); in macb_suspend()
5473 macb_writel(bp, RSR, -1); in macb_suspend()
5475 tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0; in macb_suspend()
5476 if (bp->wolopts & WAKE_ARP) { in macb_suspend()
5479 tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local)); in macb_suspend()
5485 devm_free_irq(dev, bp->queues[0].irq, bp->queues); in macb_suspend()
5487 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, in macb_suspend()
5488 IRQF_SHARED, netdev->name, bp->queues); in macb_suspend()
5492 bp->queues[0].irq, err); in macb_suspend()
5493 spin_unlock_irqrestore(&bp->lock, flags); in macb_suspend()
5496 queue_writel(bp->queues, IER, GEM_BIT(WOL)); in macb_suspend()
5499 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, in macb_suspend()
5500 IRQF_SHARED, netdev->name, bp->queues); in macb_suspend()
5504 bp->queues[0].irq, err); in macb_suspend()
5505 spin_unlock_irqrestore(&bp->lock, flags); in macb_suspend()
5508 queue_writel(bp->queues, IER, MACB_BIT(WOL)); in macb_suspend()
5511 spin_unlock_irqrestore(&bp->lock, flags); in macb_suspend()
5513 enable_irq_wake(bp->queues[0].irq); in macb_suspend()
5517 for (q = 0, queue = bp->queues; q < bp->num_queues; in macb_suspend()
5519 napi_disable(&queue->napi_rx); in macb_suspend()
5520 napi_disable(&queue->napi_tx); in macb_suspend()
5523 if (!(bp->wol & MACB_WOL_ENABLED)) { in macb_suspend()
5525 phylink_stop(bp->phylink); in macb_suspend()
5527 spin_lock_irqsave(&bp->lock, flags); in macb_suspend()
5529 spin_unlock_irqrestore(&bp->lock, flags); in macb_suspend()
5532 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) in macb_suspend()
5533 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); in macb_suspend()
5535 if (netdev->hw_features & NETIF_F_NTUPLE) in macb_suspend()
5536 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); in macb_suspend()
5538 if (bp->ptp_info) in macb_suspend()
5539 bp->ptp_info->ptp_remove(netdev); in macb_suspend()
5555 if (!device_may_wakeup(&bp->dev->dev)) in macb_resume()
5556 phy_init(bp->sgmii_phy); in macb_resume()
5564 if (bp->wol & MACB_WOL_ENABLED) { in macb_resume()
5565 spin_lock_irqsave(&bp->lock, flags); in macb_resume()
5568 queue_writel(bp->queues, IDR, GEM_BIT(WOL)); in macb_resume()
5571 queue_writel(bp->queues, IDR, MACB_BIT(WOL)); in macb_resume()
5575 queue_readl(bp->queues, ISR); in macb_resume()
5576 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) in macb_resume()
5577 queue_writel(bp->queues, ISR, -1); in macb_resume()
5579 devm_free_irq(dev, bp->queues[0].irq, bp->queues); in macb_resume()
5580 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, in macb_resume()
5581 IRQF_SHARED, netdev->name, bp->queues); in macb_resume()
5585 bp->queues[0].irq, err); in macb_resume()
5586 spin_unlock_irqrestore(&bp->lock, flags); in macb_resume()
5589 spin_unlock_irqrestore(&bp->lock, flags); in macb_resume()
5591 disable_irq_wake(bp->queues[0].irq); in macb_resume()
5597 phylink_stop(bp->phylink); in macb_resume()
5601 for (q = 0, queue = bp->queues; q < bp->num_queues; in macb_resume()
5603 napi_enable(&queue->napi_rx); in macb_resume()
5604 napi_enable(&queue->napi_tx); in macb_resume()
5607 if (netdev->hw_features & NETIF_F_NTUPLE) in macb_resume()
5608 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); in macb_resume()
5610 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) in macb_resume()
5611 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); in macb_resume()
5619 phylink_start(bp->phylink); in macb_resume()
5623 if (bp->ptp_info) in macb_resume()
5624 bp->ptp_info->ptp_init(netdev); in macb_resume()
5635 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); in macb_runtime_suspend()
5636 else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) in macb_runtime_suspend()
5637 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); in macb_runtime_suspend()
5648 clk_prepare_enable(bp->pclk); in macb_runtime_resume()
5649 clk_prepare_enable(bp->hclk); in macb_runtime_resume()
5650 clk_prepare_enable(bp->tx_clk); in macb_runtime_resume()
5651 clk_prepare_enable(bp->rx_clk); in macb_runtime_resume()
5652 clk_prepare_enable(bp->tsu_clk); in macb_runtime_resume()
5653 } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { in macb_runtime_resume()
5654 clk_prepare_enable(bp->tsu_clk); in macb_runtime_resume()