1 // SPDX-License-Identifier: GPL-2.0 2 /* Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver 3 * 4 * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/ 5 * 6 */ 7 8 #include <linux/bpf_trace.h> 9 #include <linux/clk.h> 10 #include <linux/etherdevice.h> 11 #include <linux/if_vlan.h> 12 #include <linux/interrupt.h> 13 #include <linux/irqdomain.h> 14 #include <linux/kernel.h> 15 #include <linux/kmemleak.h> 16 #include <linux/module.h> 17 #include <linux/netdevice.h> 18 #include <linux/net_tstamp.h> 19 #include <linux/of.h> 20 #include <linux/of_mdio.h> 21 #include <linux/of_net.h> 22 #include <linux/of_device.h> 23 #include <linux/of_platform.h> 24 #include <linux/phylink.h> 25 #include <linux/phy/phy.h> 26 #include <linux/platform_device.h> 27 #include <linux/pm_runtime.h> 28 #include <linux/regmap.h> 29 #include <linux/rtnetlink.h> 30 #include <linux/mfd/syscon.h> 31 #include <linux/sys_soc.h> 32 #include <linux/dma/ti-cppi5.h> 33 #include <linux/dma/k3-udma-glue.h> 34 #include <net/page_pool/helpers.h> 35 #include <net/dsa.h> 36 #include <net/switchdev.h> 37 38 #include "cpsw_ale.h" 39 #include "cpsw_sl.h" 40 #include "am65-cpsw-nuss.h" 41 #include "am65-cpsw-switchdev.h" 42 #include "k3-cppi-desc-pool.h" 43 #include "am65-cpts.h" 44 45 #define AM65_CPSW_SS_BASE 0x0 46 #define AM65_CPSW_SGMII_BASE 0x100 47 #define AM65_CPSW_XGMII_BASE 0x2100 48 #define AM65_CPSW_CPSW_NU_BASE 0x20000 49 #define AM65_CPSW_NU_PORTS_BASE 0x1000 50 #define AM65_CPSW_NU_FRAM_BASE 0x12000 51 #define AM65_CPSW_NU_STATS_BASE 0x1a000 52 #define AM65_CPSW_NU_ALE_BASE 0x1e000 53 #define AM65_CPSW_NU_CPTS_BASE 0x1d000 54 55 #define AM65_CPSW_NU_PORTS_OFFSET 0x1000 56 #define AM65_CPSW_NU_STATS_PORT_OFFSET 0x200 57 #define AM65_CPSW_NU_FRAM_PORT_OFFSET 0x200 58 59 #define AM65_CPSW_MAX_PORTS 8 60 61 #define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN 62 #define AM65_CPSW_MAX_PACKET_SIZE 2024 63 64 #define AM65_CPSW_REG_CTL 0x004 65 #define AM65_CPSW_REG_STAT_PORT_EN 0x014 66 #define AM65_CPSW_REG_PTYPE 0x018 67 68 #define AM65_CPSW_P0_REG_CTL 0x004 69 #define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET 0x008 70 71 #define AM65_CPSW_PORT_REG_PRI_CTL 0x01c 72 #define AM65_CPSW_PORT_REG_RX_PRI_MAP 0x020 73 #define AM65_CPSW_PORT_REG_RX_MAXLEN 0x024 74 75 #define AM65_CPSW_PORTN_REG_CTL 0x004 76 #define AM65_CPSW_PORTN_REG_DSCP_MAP 0x120 77 #define AM65_CPSW_PORTN_REG_SA_L 0x308 78 #define AM65_CPSW_PORTN_REG_SA_H 0x30c 79 #define AM65_CPSW_PORTN_REG_TS_CTL 0x310 80 #define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG 0x314 81 #define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG 0x318 82 #define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 0x31C 83 84 #define AM65_CPSW_SGMII_CONTROL_REG 0x010 85 #define AM65_CPSW_SGMII_MR_ADV_ABILITY_REG 0x018 86 #define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE BIT(0) 87 88 #define AM65_CPSW_CTL_VLAN_AWARE BIT(1) 89 #define AM65_CPSW_CTL_P0_ENABLE BIT(2) 90 #define AM65_CPSW_CTL_P0_TX_CRC_REMOVE BIT(13) 91 #define AM65_CPSW_CTL_P0_RX_PAD BIT(14) 92 93 /* AM65_CPSW_P0_REG_CTL */ 94 #define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN BIT(0) 95 #define AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN BIT(16) 96 97 /* AM65_CPSW_PORT_REG_PRI_CTL */ 98 #define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN BIT(8) 99 100 /* AM65_CPSW_PN_REG_CTL */ 101 #define AM65_CPSW_PN_REG_CTL_DSCP_IPV4_EN BIT(1) 102 #define AM65_CPSW_PN_REG_CTL_DSCP_IPV6_EN BIT(2) 103 104 /* AM65_CPSW_PN_TS_CTL register fields */ 105 #define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN BIT(4) 106 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN BIT(5) 107 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN BIT(6) 108 #define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN BIT(7) 109 #define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN BIT(10) 110 #define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN BIT(11) 111 #define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT 16 112 113 #define AM65_CPSW_PN_TS_CTL_RX_ANX_F_EN BIT(0) 114 #define AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN BIT(1) 115 #define AM65_CPSW_PN_TS_CTL_RX_VLAN_LT2_EN BIT(2) 116 #define AM65_CPSW_PN_TS_CTL_RX_ANX_D_EN BIT(3) 117 #define AM65_CPSW_PN_TS_CTL_RX_ANX_E_EN BIT(9) 118 119 /* AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG register fields */ 120 #define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT 16 121 122 /* AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 */ 123 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 BIT(16) 124 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 BIT(17) 125 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 BIT(18) 126 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 BIT(19) 127 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 BIT(20) 128 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 BIT(21) 129 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 BIT(22) 130 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23) 131 132 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */ 133 #define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3)) 134 135 #define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e) 136 137 #define AM65_CPSW_TS_TX_ANX_ALL_EN \ 138 (AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN | \ 139 AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN | \ 140 AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN) 141 142 #define AM65_CPSW_TS_RX_ANX_ALL_EN \ 143 (AM65_CPSW_PN_TS_CTL_RX_ANX_D_EN | \ 144 AM65_CPSW_PN_TS_CTL_RX_ANX_E_EN | \ 145 AM65_CPSW_PN_TS_CTL_RX_ANX_F_EN) 146 147 #define AM65_CPSW_ALE_AGEOUT_DEFAULT 30 148 /* Number of TX/RX descriptors per channel/flow */ 149 #define AM65_CPSW_MAX_TX_DESC 500 150 #define AM65_CPSW_MAX_RX_DESC 500 151 152 #define AM65_CPSW_NAV_PS_DATA_SIZE 16 153 #define AM65_CPSW_NAV_SW_DATA_SIZE 16 154 155 #define AM65_CPSW_DEBUG (NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | \ 156 NETIF_MSG_IFUP | NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \ 157 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 158 159 #define AM65_CPSW_DEFAULT_TX_CHNS 8 160 #define AM65_CPSW_DEFAULT_RX_CHN_FLOWS 1 161 162 /* CPPI streaming packet interface */ 163 #define AM65_CPSW_CPPI_TX_FLOW_ID 0x3FFF 164 #define AM65_CPSW_CPPI_TX_PKT_TYPE 0x7 165 166 /* XDP */ 167 #define AM65_CPSW_XDP_TX BIT(2) 168 #define AM65_CPSW_XDP_CONSUMED BIT(1) 169 #define AM65_CPSW_XDP_REDIRECT BIT(0) 170 #define AM65_CPSW_XDP_PASS 0 171 172 /* Include headroom compatible with both skb and xdpf */ 173 #define AM65_CPSW_HEADROOM_NA (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN) 174 #define AM65_CPSW_HEADROOM ALIGN(AM65_CPSW_HEADROOM_NA, sizeof(long)) 175 176 static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave, 177 const u8 *dev_addr) 178 { 179 u32 mac_hi = (dev_addr[0] << 0) | (dev_addr[1] << 8) | 180 (dev_addr[2] << 16) | (dev_addr[3] << 24); 181 u32 mac_lo = (dev_addr[4] << 0) | (dev_addr[5] << 8); 182 183 writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H); 184 writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L); 185 } 186 187 #define AM65_CPSW_DSCP_MAX GENMASK(5, 0) 188 #define AM65_CPSW_PRI_MAX GENMASK(2, 0) 189 #define AM65_CPSW_DSCP_PRI_PER_REG 8 190 #define AM65_CPSW_DSCP_PRI_SIZE 4 /* in bits */ 191 static int am65_cpsw_port_set_dscp_map(struct am65_cpsw_port *slave, u8 dscp, u8 pri) 192 { 193 int reg_ofs; 194 int bit_ofs; 195 u32 val; 196 197 if (dscp > AM65_CPSW_DSCP_MAX) 198 return -EINVAL; 199 200 if (pri > AM65_CPSW_PRI_MAX) 201 return -EINVAL; 202 203 /* 32-bit register offset to this dscp */ 204 reg_ofs = (dscp / AM65_CPSW_DSCP_PRI_PER_REG) * 4; 205 /* bit field offset to this dscp */ 206 bit_ofs = AM65_CPSW_DSCP_PRI_SIZE * (dscp % AM65_CPSW_DSCP_PRI_PER_REG); 207 208 val = readl(slave->port_base + AM65_CPSW_PORTN_REG_DSCP_MAP + reg_ofs); 209 val &= ~(AM65_CPSW_PRI_MAX << bit_ofs); /* clear */ 210 val |= pri << bit_ofs; /* set */ 211 writel(val, slave->port_base + AM65_CPSW_PORTN_REG_DSCP_MAP + reg_ofs); 212 213 return 0; 214 } 215 216 static void am65_cpsw_port_enable_dscp_map(struct am65_cpsw_port *slave) 217 { 218 int dscp, pri; 219 u32 val; 220 221 /* Default DSCP to User Priority mapping as per: 222 * https://datatracker.ietf.org/doc/html/rfc8325#section-4.3 223 * and 224 * https://datatracker.ietf.org/doc/html/rfc8622#section-11 225 */ 226 for (dscp = 0; dscp <= AM65_CPSW_DSCP_MAX; dscp++) { 227 switch (dscp) { 228 case 56: /* CS7 */ 229 case 48: /* CS6 */ 230 pri = 7; 231 break; 232 case 46: /* EF */ 233 case 44: /* VA */ 234 pri = 6; 235 break; 236 case 40: /* CS5 */ 237 pri = 5; 238 break; 239 case 34: /* AF41 */ 240 case 36: /* AF42 */ 241 case 38: /* AF43 */ 242 case 32: /* CS4 */ 243 case 26: /* AF31 */ 244 case 28: /* AF32 */ 245 case 30: /* AF33 */ 246 case 24: /* CS3 */ 247 pri = 4; 248 break; 249 case 18: /* AF21 */ 250 case 20: /* AF22 */ 251 case 22: /* AF23 */ 252 pri = 3; 253 break; 254 case 16: /* CS2 */ 255 case 10: /* AF11 */ 256 case 12: /* AF12 */ 257 case 14: /* AF13 */ 258 case 0: /* DF */ 259 pri = 0; 260 break; 261 case 8: /* CS1 */ 262 case 1: /* LE */ 263 pri = 1; 264 break; 265 default: 266 pri = 0; 267 break; 268 } 269 270 am65_cpsw_port_set_dscp_map(slave, dscp, pri); 271 } 272 273 /* enable port IPV4 and IPV6 DSCP for this port */ 274 val = readl(slave->port_base + AM65_CPSW_PORTN_REG_CTL); 275 val |= AM65_CPSW_PN_REG_CTL_DSCP_IPV4_EN | 276 AM65_CPSW_PN_REG_CTL_DSCP_IPV6_EN; 277 writel(val, slave->port_base + AM65_CPSW_PORTN_REG_CTL); 278 } 279 280 static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port) 281 { 282 cpsw_sl_reset(port->slave.mac_sl, 100); 283 /* Max length register has to be restored after MAC SL reset */ 284 writel(AM65_CPSW_MAX_PACKET_SIZE, 285 port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN); 286 } 287 288 static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common) 289 { 290 common->nuss_ver = readl(common->ss_base); 291 common->cpsw_ver = readl(common->cpsw_base); 292 dev_info(common->dev, 293 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n", 294 common->nuss_ver, 295 common->cpsw_ver, 296 common->port_num + 1, 297 common->pdata.quirks); 298 } 299 300 static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev, 301 __be16 proto, u16 vid) 302 { 303 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 304 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 305 u32 port_mask; 306 int ret; 307 308 if (!common->is_emac_mode) 309 return 0; 310 311 if (!netif_running(ndev) || !vid) 312 return 0; 313 314 ret = pm_runtime_resume_and_get(common->dev); 315 if (ret < 0) 316 return ret; 317 318 port_mask = BIT(port->port_id) | ALE_PORT_HOST; 319 dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid); 320 ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask, 321 0, port_mask, 0); 322 323 pm_runtime_put(common->dev); 324 return ret; 325 } 326 327 static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev, 328 __be16 proto, u16 vid) 329 { 330 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 331 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 332 int ret; 333 334 if (!common->is_emac_mode) 335 return 0; 336 337 if (!netif_running(ndev) || !vid) 338 return 0; 339 340 ret = pm_runtime_resume_and_get(common->dev); 341 if (ret < 0) 342 return ret; 343 344 dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid); 345 ret = cpsw_ale_del_vlan(common->ale, vid, 346 BIT(port->port_id) | ALE_PORT_HOST); 347 348 pm_runtime_put(common->dev); 349 return ret; 350 } 351 352 static void am65_cpsw_slave_set_promisc(struct am65_cpsw_port *port, 353 bool promisc) 354 { 355 struct am65_cpsw_common *common = port->common; 356 357 if (promisc && !common->is_emac_mode) { 358 dev_dbg(common->dev, "promisc mode requested in switch mode"); 359 return; 360 } 361 362 if (promisc) { 363 /* Enable promiscuous mode */ 364 cpsw_ale_control_set(common->ale, port->port_id, 365 ALE_PORT_MACONLY_CAF, 1); 366 dev_dbg(common->dev, "promisc enabled\n"); 367 } else { 368 /* Disable promiscuous mode */ 369 cpsw_ale_control_set(common->ale, port->port_id, 370 ALE_PORT_MACONLY_CAF, 0); 371 dev_dbg(common->dev, "promisc disabled\n"); 372 } 373 } 374 375 static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev) 376 { 377 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 378 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 379 u32 port_mask; 380 bool promisc; 381 382 promisc = !!(ndev->flags & IFF_PROMISC); 383 am65_cpsw_slave_set_promisc(port, promisc); 384 385 if (promisc) 386 return; 387 388 /* Restore allmulti on vlans if necessary */ 389 cpsw_ale_set_allmulti(common->ale, 390 ndev->flags & IFF_ALLMULTI, port->port_id); 391 392 port_mask = BIT(port->port_id) | ALE_PORT_HOST; 393 /* Clear all mcast from ALE */ 394 cpsw_ale_flush_multicast(common->ale, port_mask, -1); 395 396 if (!netdev_mc_empty(ndev)) { 397 struct netdev_hw_addr *ha; 398 399 /* program multicast address list into ALE register */ 400 netdev_for_each_mc_addr(ha, ndev) { 401 cpsw_ale_add_mcast(common->ale, ha->addr, 402 port_mask, 0, 0, 0); 403 } 404 } 405 } 406 407 static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev, 408 unsigned int txqueue) 409 { 410 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 411 struct am65_cpsw_tx_chn *tx_chn; 412 struct netdev_queue *netif_txq; 413 unsigned long trans_start; 414 415 netif_txq = netdev_get_tx_queue(ndev, txqueue); 416 tx_chn = &common->tx_chns[txqueue]; 417 trans_start = READ_ONCE(netif_txq->trans_start); 418 419 netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n", 420 txqueue, 421 netif_tx_queue_stopped(netif_txq), 422 jiffies_to_msecs(jiffies - trans_start), 423 netdev_queue_dql_avail(netif_txq), 424 k3_cppi_desc_pool_avail(tx_chn->desc_pool)); 425 426 if (netif_tx_queue_stopped(netif_txq)) { 427 /* try recover if stopped by us */ 428 txq_trans_update(ndev, netif_txq); 429 netif_tx_wake_queue(netif_txq); 430 } 431 } 432 433 static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common, 434 struct page *page, u32 flow_idx) 435 { 436 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; 437 struct cppi5_host_desc_t *desc_rx; 438 struct device *dev = common->dev; 439 struct am65_cpsw_swdata *swdata; 440 dma_addr_t desc_dma; 441 dma_addr_t buf_dma; 442 443 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool); 444 if (!desc_rx) { 445 dev_err(dev, "Failed to allocate RXFDQ descriptor\n"); 446 return -ENOMEM; 447 } 448 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx); 449 450 buf_dma = dma_map_single(rx_chn->dma_dev, 451 page_address(page) + AM65_CPSW_HEADROOM, 452 AM65_CPSW_MAX_PACKET_SIZE, DMA_FROM_DEVICE); 453 if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) { 454 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 455 dev_err(dev, "Failed to map rx buffer\n"); 456 return -EINVAL; 457 } 458 459 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT, 460 AM65_CPSW_NAV_PS_DATA_SIZE); 461 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma); 462 cppi5_hdesc_attach_buf(desc_rx, buf_dma, AM65_CPSW_MAX_PACKET_SIZE, 463 buf_dma, AM65_CPSW_MAX_PACKET_SIZE); 464 swdata = cppi5_hdesc_get_swdata(desc_rx); 465 swdata->page = page; 466 swdata->flow_id = flow_idx; 467 468 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, flow_idx, 469 desc_rx, desc_dma); 470 } 471 472 void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common) 473 { 474 struct am65_cpsw_host *host_p = am65_common_get_host(common); 475 u32 val, pri_map; 476 477 /* P0 set Receive Priority Type */ 478 val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL); 479 480 if (common->pf_p0_rx_ptype_rrobin) { 481 val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN; 482 /* Enet Ports fifos works in fixed priority mode only, so 483 * reset P0_Rx_Pri_Map so all packet will go in Enet fifo 0 484 */ 485 pri_map = 0x0; 486 } else { 487 val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN; 488 /* restore P0_Rx_Pri_Map */ 489 pri_map = 0x76543210; 490 } 491 492 writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP); 493 writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL); 494 } 495 496 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common); 497 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common); 498 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port); 499 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port); 500 static inline void am65_cpsw_put_page(struct am65_cpsw_rx_flow *flow, 501 struct page *page, 502 bool allow_direct); 503 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma); 504 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma); 505 506 static void am65_cpsw_destroy_rxq(struct am65_cpsw_common *common, int id) 507 { 508 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; 509 struct am65_cpsw_rx_flow *flow; 510 struct xdp_rxq_info *rxq; 511 int port; 512 513 flow = &rx_chn->flows[id]; 514 napi_disable(&flow->napi_rx); 515 hrtimer_cancel(&flow->rx_hrtimer); 516 k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, id, rx_chn, 517 am65_cpsw_nuss_rx_cleanup); 518 519 for (port = 0; port < common->port_num; port++) { 520 if (!common->ports[port].ndev) 521 continue; 522 523 rxq = &common->ports[port].xdp_rxq[id]; 524 525 if (xdp_rxq_info_is_reg(rxq)) 526 xdp_rxq_info_unreg(rxq); 527 } 528 529 if (flow->page_pool) { 530 page_pool_destroy(flow->page_pool); 531 flow->page_pool = NULL; 532 } 533 } 534 535 static void am65_cpsw_destroy_rxqs(struct am65_cpsw_common *common) 536 { 537 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; 538 int id; 539 540 reinit_completion(&common->tdown_complete); 541 k3_udma_glue_tdown_rx_chn(rx_chn->rx_chn, true); 542 543 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) { 544 id = wait_for_completion_timeout(&common->tdown_complete, msecs_to_jiffies(1000)); 545 if (!id) 546 dev_err(common->dev, "rx teardown timeout\n"); 547 } 548 549 for (id = common->rx_ch_num_flows - 1; id >= 0; id--) 550 am65_cpsw_destroy_rxq(common, id); 551 552 k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn); 553 } 554 555 static int am65_cpsw_create_rxq(struct am65_cpsw_common *common, int id) 556 { 557 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; 558 struct page_pool_params pp_params = { 559 .flags = PP_FLAG_DMA_MAP, 560 .order = 0, 561 .pool_size = AM65_CPSW_MAX_RX_DESC, 562 .nid = dev_to_node(common->dev), 563 .dev = common->dev, 564 .dma_dir = DMA_BIDIRECTIONAL, 565 /* .napi set dynamically */ 566 }; 567 struct am65_cpsw_rx_flow *flow; 568 struct xdp_rxq_info *rxq; 569 struct page_pool *pool; 570 struct page *page; 571 int port, ret, i; 572 573 flow = &rx_chn->flows[id]; 574 pp_params.napi = &flow->napi_rx; 575 pool = page_pool_create(&pp_params); 576 if (IS_ERR(pool)) { 577 ret = PTR_ERR(pool); 578 return ret; 579 } 580 581 flow->page_pool = pool; 582 583 /* using same page pool is allowed as no running rx handlers 584 * simultaneously for both ndevs 585 */ 586 for (port = 0; port < common->port_num; port++) { 587 if (!common->ports[port].ndev) 588 /* FIXME should we BUG here? */ 589 continue; 590 591 rxq = &common->ports[port].xdp_rxq[id]; 592 ret = xdp_rxq_info_reg(rxq, common->ports[port].ndev, 593 id, flow->napi_rx.napi_id); 594 if (ret) 595 goto err; 596 597 ret = xdp_rxq_info_reg_mem_model(rxq, 598 MEM_TYPE_PAGE_POOL, 599 pool); 600 if (ret) 601 goto err; 602 } 603 604 for (i = 0; i < AM65_CPSW_MAX_RX_DESC; i++) { 605 page = page_pool_dev_alloc_pages(flow->page_pool); 606 if (!page) { 607 dev_err(common->dev, "cannot allocate page in flow %d\n", 608 id); 609 ret = -ENOMEM; 610 goto err; 611 } 612 613 ret = am65_cpsw_nuss_rx_push(common, page, id); 614 if (ret < 0) { 615 dev_err(common->dev, 616 "cannot submit page to rx channel flow %d, error %d\n", 617 id, ret); 618 am65_cpsw_put_page(flow, page, false); 619 goto err; 620 } 621 } 622 623 napi_enable(&flow->napi_rx); 624 return 0; 625 626 err: 627 am65_cpsw_destroy_rxq(common, id); 628 return ret; 629 } 630 631 static int am65_cpsw_create_rxqs(struct am65_cpsw_common *common) 632 { 633 int id, ret; 634 635 for (id = 0; id < common->rx_ch_num_flows; id++) { 636 ret = am65_cpsw_create_rxq(common, id); 637 if (ret) { 638 dev_err(common->dev, "couldn't create rxq %d: %d\n", 639 id, ret); 640 goto err; 641 } 642 } 643 644 ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn); 645 if (ret) { 646 dev_err(common->dev, "couldn't enable rx chn: %d\n", ret); 647 goto err; 648 } 649 650 return 0; 651 652 err: 653 for (--id; id >= 0; id--) 654 am65_cpsw_destroy_rxq(common, id); 655 656 return ret; 657 } 658 659 static void am65_cpsw_destroy_txq(struct am65_cpsw_common *common, int id) 660 { 661 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id]; 662 663 napi_disable(&tx_chn->napi_tx); 664 hrtimer_cancel(&tx_chn->tx_hrtimer); 665 k3_udma_glue_reset_tx_chn(tx_chn->tx_chn, tx_chn, 666 am65_cpsw_nuss_tx_cleanup); 667 k3_udma_glue_disable_tx_chn(tx_chn->tx_chn); 668 } 669 670 static void am65_cpsw_destroy_txqs(struct am65_cpsw_common *common) 671 { 672 struct am65_cpsw_tx_chn *tx_chn = common->tx_chns; 673 int id; 674 675 /* shutdown tx channels */ 676 atomic_set(&common->tdown_cnt, common->tx_ch_num); 677 /* ensure new tdown_cnt value is visible */ 678 smp_mb__after_atomic(); 679 reinit_completion(&common->tdown_complete); 680 681 for (id = 0; id < common->tx_ch_num; id++) 682 k3_udma_glue_tdown_tx_chn(tx_chn[id].tx_chn, false); 683 684 id = wait_for_completion_timeout(&common->tdown_complete, 685 msecs_to_jiffies(1000)); 686 if (!id) 687 dev_err(common->dev, "tx teardown timeout\n"); 688 689 for (id = common->tx_ch_num - 1; id >= 0; id--) 690 am65_cpsw_destroy_txq(common, id); 691 } 692 693 static int am65_cpsw_create_txq(struct am65_cpsw_common *common, int id) 694 { 695 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id]; 696 int ret; 697 698 ret = k3_udma_glue_enable_tx_chn(tx_chn->tx_chn); 699 if (ret) 700 return ret; 701 702 napi_enable(&tx_chn->napi_tx); 703 704 return 0; 705 } 706 707 static int am65_cpsw_create_txqs(struct am65_cpsw_common *common) 708 { 709 int id, ret; 710 711 for (id = 0; id < common->tx_ch_num; id++) { 712 ret = am65_cpsw_create_txq(common, id); 713 if (ret) { 714 dev_err(common->dev, "couldn't create txq %d: %d\n", 715 id, ret); 716 goto err; 717 } 718 } 719 720 return 0; 721 722 err: 723 for (--id; id >= 0; id--) 724 am65_cpsw_destroy_txq(common, id); 725 726 return ret; 727 } 728 729 static int am65_cpsw_nuss_desc_idx(struct k3_cppi_desc_pool *desc_pool, 730 void *desc, 731 unsigned char dsize_log2) 732 { 733 void *pool_addr = k3_cppi_desc_pool_cpuaddr(desc_pool); 734 735 return (desc - pool_addr) >> dsize_log2; 736 } 737 738 static void am65_cpsw_nuss_set_buf_type(struct am65_cpsw_tx_chn *tx_chn, 739 struct cppi5_host_desc_t *desc, 740 enum am65_cpsw_tx_buf_type buf_type) 741 { 742 int desc_idx; 743 744 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc, 745 tx_chn->dsize_log2); 746 k3_cppi_desc_pool_desc_info_set(tx_chn->desc_pool, desc_idx, 747 (void *)buf_type); 748 } 749 750 static enum am65_cpsw_tx_buf_type am65_cpsw_nuss_buf_type(struct am65_cpsw_tx_chn *tx_chn, 751 dma_addr_t desc_dma) 752 { 753 struct cppi5_host_desc_t *desc_tx; 754 int desc_idx; 755 756 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); 757 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc_tx, 758 tx_chn->dsize_log2); 759 760 return (enum am65_cpsw_tx_buf_type)k3_cppi_desc_pool_desc_info(tx_chn->desc_pool, 761 desc_idx); 762 } 763 764 static inline void am65_cpsw_put_page(struct am65_cpsw_rx_flow *flow, 765 struct page *page, 766 bool allow_direct) 767 { 768 page_pool_put_full_page(flow->page_pool, page, allow_direct); 769 } 770 771 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma) 772 { 773 struct am65_cpsw_rx_chn *rx_chn = data; 774 struct cppi5_host_desc_t *desc_rx; 775 struct am65_cpsw_swdata *swdata; 776 dma_addr_t buf_dma; 777 struct page *page; 778 u32 buf_dma_len; 779 u32 flow_id; 780 781 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); 782 swdata = cppi5_hdesc_get_swdata(desc_rx); 783 page = swdata->page; 784 flow_id = swdata->flow_id; 785 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); 786 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); 787 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); 788 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 789 am65_cpsw_put_page(&rx_chn->flows[flow_id], page, false); 790 } 791 792 static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn, 793 struct cppi5_host_desc_t *desc) 794 { 795 struct cppi5_host_desc_t *first_desc, *next_desc; 796 dma_addr_t buf_dma, next_desc_dma; 797 u32 buf_dma_len; 798 799 first_desc = desc; 800 next_desc = first_desc; 801 802 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len); 803 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); 804 805 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE); 806 807 next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc); 808 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); 809 while (next_desc_dma) { 810 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, 811 next_desc_dma); 812 cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len); 813 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); 814 815 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len, 816 DMA_TO_DEVICE); 817 818 next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc); 819 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); 820 821 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); 822 } 823 824 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc); 825 } 826 827 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma) 828 { 829 struct am65_cpsw_tx_chn *tx_chn = data; 830 enum am65_cpsw_tx_buf_type buf_type; 831 struct am65_cpsw_tx_swdata *swdata; 832 struct cppi5_host_desc_t *desc_tx; 833 struct xdp_frame *xdpf; 834 struct sk_buff *skb; 835 836 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); 837 swdata = cppi5_hdesc_get_swdata(desc_tx); 838 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma); 839 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) { 840 skb = swdata->skb; 841 dev_kfree_skb_any(skb); 842 } else { 843 xdpf = swdata->xdpf; 844 xdp_return_frame(xdpf); 845 } 846 847 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx); 848 } 849 850 static struct sk_buff *am65_cpsw_build_skb(void *page_addr, 851 struct net_device *ndev, 852 unsigned int len, 853 unsigned int headroom) 854 { 855 struct sk_buff *skb; 856 857 skb = build_skb(page_addr, len); 858 if (unlikely(!skb)) 859 return NULL; 860 861 skb_reserve(skb, headroom); 862 skb->dev = ndev; 863 864 return skb; 865 } 866 867 static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) 868 { 869 struct am65_cpsw_host *host_p = am65_common_get_host(common); 870 u32 val, port_mask; 871 int port_idx, ret; 872 873 if (common->usage_count) 874 return 0; 875 876 /* Control register */ 877 writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE | 878 AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD, 879 common->cpsw_base + AM65_CPSW_REG_CTL); 880 /* Max length register */ 881 writel(AM65_CPSW_MAX_PACKET_SIZE, 882 host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN); 883 /* set base flow_id */ 884 writel(common->rx_flow_id_base, 885 host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET); 886 writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN | AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN, 887 host_p->port_base + AM65_CPSW_P0_REG_CTL); 888 889 am65_cpsw_nuss_set_p0_ptype(common); 890 891 /* enable statistic */ 892 val = BIT(HOST_PORT_NUM); 893 for (port_idx = 0; port_idx < common->port_num; port_idx++) { 894 struct am65_cpsw_port *port = &common->ports[port_idx]; 895 896 if (!port->disabled) 897 val |= BIT(port->port_id); 898 } 899 writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN); 900 901 /* disable priority elevation */ 902 writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE); 903 904 cpsw_ale_start(common->ale); 905 906 /* limit to one RX flow only */ 907 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, 908 ALE_DEFAULT_THREAD_ID, 0); 909 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, 910 ALE_DEFAULT_THREAD_ENABLE, 1); 911 /* switch to vlan aware mode */ 912 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1); 913 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, 914 ALE_PORT_STATE, ALE_PORT_STATE_FORWARD); 915 916 /* default vlan cfg: create mask based on enabled ports */ 917 port_mask = GENMASK(common->port_num, 0) & 918 ~common->disabled_ports_mask; 919 920 cpsw_ale_add_vlan(common->ale, 0, port_mask, 921 port_mask, port_mask, 922 port_mask & ~ALE_PORT_HOST); 923 924 if (common->is_emac_mode) 925 am65_cpsw_init_host_port_emac(common); 926 else 927 am65_cpsw_init_host_port_switch(common); 928 929 am65_cpsw_qos_tx_p0_rate_init(common); 930 931 ret = am65_cpsw_create_rxqs(common); 932 if (ret) 933 return ret; 934 935 ret = am65_cpsw_create_txqs(common); 936 if (ret) 937 goto cleanup_rx; 938 939 dev_dbg(common->dev, "cpsw_nuss started\n"); 940 return 0; 941 942 cleanup_rx: 943 am65_cpsw_destroy_rxqs(common); 944 945 return ret; 946 } 947 948 static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common) 949 { 950 if (common->usage_count != 1) 951 return 0; 952 953 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, 954 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE); 955 956 am65_cpsw_destroy_txqs(common); 957 am65_cpsw_destroy_rxqs(common); 958 cpsw_ale_stop(common->ale); 959 960 writel(0, common->cpsw_base + AM65_CPSW_REG_CTL); 961 writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN); 962 963 dev_dbg(common->dev, "cpsw_nuss stopped\n"); 964 return 0; 965 } 966 967 static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev) 968 { 969 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 970 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 971 int ret; 972 973 phylink_stop(port->slave.phylink); 974 975 netif_tx_stop_all_queues(ndev); 976 977 phylink_disconnect_phy(port->slave.phylink); 978 979 ret = am65_cpsw_nuss_common_stop(common); 980 if (ret) 981 return ret; 982 983 common->usage_count--; 984 pm_runtime_put(common->dev); 985 return 0; 986 } 987 988 static int cpsw_restore_vlans(struct net_device *vdev, int vid, void *arg) 989 { 990 struct am65_cpsw_port *port = arg; 991 992 if (!vdev) 993 return 0; 994 995 return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid); 996 } 997 998 static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev) 999 { 1000 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 1001 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1002 int ret, i; 1003 u32 reg; 1004 1005 ret = pm_runtime_resume_and_get(common->dev); 1006 if (ret < 0) 1007 return ret; 1008 1009 /* Idle MAC port */ 1010 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE); 1011 cpsw_sl_wait_for_idle(port->slave.mac_sl, 100); 1012 cpsw_sl_ctl_reset(port->slave.mac_sl); 1013 1014 /* soft reset MAC */ 1015 cpsw_sl_reg_write(port->slave.mac_sl, CPSW_SL_SOFT_RESET, 1); 1016 mdelay(1); 1017 reg = cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_SOFT_RESET); 1018 if (reg) { 1019 dev_err(common->dev, "soft RESET didn't complete\n"); 1020 ret = -ETIMEDOUT; 1021 goto runtime_put; 1022 } 1023 1024 /* Notify the stack of the actual queue counts. */ 1025 ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num); 1026 if (ret) { 1027 dev_err(common->dev, "cannot set real number of tx queues\n"); 1028 goto runtime_put; 1029 } 1030 1031 ret = netif_set_real_num_rx_queues(ndev, common->rx_ch_num_flows); 1032 if (ret) { 1033 dev_err(common->dev, "cannot set real number of rx queues\n"); 1034 goto runtime_put; 1035 } 1036 1037 for (i = 0; i < common->tx_ch_num; i++) { 1038 struct netdev_queue *txq = netdev_get_tx_queue(ndev, i); 1039 1040 netdev_tx_reset_queue(txq); 1041 txq->tx_maxrate = common->tx_chns[i].rate_mbps; 1042 } 1043 1044 ret = am65_cpsw_nuss_common_open(common); 1045 if (ret) 1046 goto runtime_put; 1047 1048 common->usage_count++; 1049 1050 /* VLAN aware CPSW mode is incompatible with some DSA tagging schemes. 1051 * Therefore disable VLAN_AWARE mode if any of the ports is a DSA Port. 1052 */ 1053 if (netdev_uses_dsa(ndev)) { 1054 reg = readl(common->cpsw_base + AM65_CPSW_REG_CTL); 1055 reg &= ~AM65_CPSW_CTL_VLAN_AWARE; 1056 writel(reg, common->cpsw_base + AM65_CPSW_REG_CTL); 1057 } 1058 1059 am65_cpsw_port_set_sl_mac(port, ndev->dev_addr); 1060 am65_cpsw_port_enable_dscp_map(port); 1061 1062 if (common->is_emac_mode) 1063 am65_cpsw_init_port_emac_ale(port); 1064 else 1065 am65_cpsw_init_port_switch_ale(port); 1066 1067 /* mac_sl should be configured via phy-link interface */ 1068 am65_cpsw_sl_ctl_reset(port); 1069 1070 ret = phylink_of_phy_connect(port->slave.phylink, port->slave.port_np, 0); 1071 if (ret) 1072 goto error_cleanup; 1073 1074 /* restore vlan configurations */ 1075 vlan_for_each(ndev, cpsw_restore_vlans, port); 1076 1077 phylink_start(port->slave.phylink); 1078 1079 return 0; 1080 1081 error_cleanup: 1082 am65_cpsw_nuss_ndo_slave_stop(ndev); 1083 return ret; 1084 1085 runtime_put: 1086 pm_runtime_put(common->dev); 1087 return ret; 1088 } 1089 1090 static int am65_cpsw_xdp_tx_frame(struct net_device *ndev, 1091 struct am65_cpsw_tx_chn *tx_chn, 1092 struct xdp_frame *xdpf, 1093 enum am65_cpsw_tx_buf_type buf_type) 1094 { 1095 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 1096 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1097 struct cppi5_host_desc_t *host_desc; 1098 struct am65_cpsw_tx_swdata *swdata; 1099 struct netdev_queue *netif_txq; 1100 dma_addr_t dma_desc, dma_buf; 1101 u32 pkt_len = xdpf->len; 1102 int ret; 1103 1104 host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); 1105 if (unlikely(!host_desc)) { 1106 ndev->stats.tx_dropped++; 1107 return AM65_CPSW_XDP_CONSUMED; /* drop */ 1108 } 1109 1110 am65_cpsw_nuss_set_buf_type(tx_chn, host_desc, buf_type); 1111 1112 dma_buf = dma_map_single(tx_chn->dma_dev, xdpf->data, 1113 pkt_len, DMA_TO_DEVICE); 1114 if (unlikely(dma_mapping_error(tx_chn->dma_dev, dma_buf))) { 1115 ndev->stats.tx_dropped++; 1116 ret = AM65_CPSW_XDP_CONSUMED; /* drop */ 1117 goto pool_free; 1118 } 1119 1120 cppi5_hdesc_init(host_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT, 1121 AM65_CPSW_NAV_PS_DATA_SIZE); 1122 cppi5_hdesc_set_pkttype(host_desc, AM65_CPSW_CPPI_TX_PKT_TYPE); 1123 cppi5_hdesc_set_pktlen(host_desc, pkt_len); 1124 cppi5_desc_set_pktids(&host_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID); 1125 cppi5_desc_set_tags_ids(&host_desc->hdr, 0, port->port_id); 1126 1127 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf); 1128 cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf, pkt_len); 1129 1130 swdata = cppi5_hdesc_get_swdata(host_desc); 1131 swdata->ndev = ndev; 1132 swdata->xdpf = xdpf; 1133 1134 /* Report BQL before sending the packet */ 1135 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); 1136 netdev_tx_sent_queue(netif_txq, pkt_len); 1137 1138 dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, host_desc); 1139 if (AM65_CPSW_IS_CPSW2G(common)) { 1140 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc, 1141 dma_desc); 1142 } else { 1143 spin_lock_bh(&tx_chn->lock); 1144 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc, 1145 dma_desc); 1146 spin_unlock_bh(&tx_chn->lock); 1147 } 1148 if (ret) { 1149 /* Inform BQL */ 1150 netdev_tx_completed_queue(netif_txq, 1, pkt_len); 1151 ndev->stats.tx_errors++; 1152 ret = AM65_CPSW_XDP_CONSUMED; /* drop */ 1153 goto dma_unmap; 1154 } 1155 1156 return 0; 1157 1158 dma_unmap: 1159 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &dma_buf); 1160 dma_unmap_single(tx_chn->dma_dev, dma_buf, pkt_len, DMA_TO_DEVICE); 1161 pool_free: 1162 k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc); 1163 return ret; 1164 } 1165 1166 static int am65_cpsw_run_xdp(struct am65_cpsw_rx_flow *flow, 1167 struct am65_cpsw_port *port, 1168 struct xdp_buff *xdp, int *len) 1169 { 1170 struct am65_cpsw_common *common = flow->common; 1171 struct net_device *ndev = port->ndev; 1172 int ret = AM65_CPSW_XDP_CONSUMED; 1173 struct am65_cpsw_tx_chn *tx_chn; 1174 struct netdev_queue *netif_txq; 1175 int cpu = smp_processor_id(); 1176 struct xdp_frame *xdpf; 1177 struct bpf_prog *prog; 1178 int pkt_len; 1179 u32 act; 1180 int err; 1181 1182 pkt_len = *len; 1183 prog = READ_ONCE(port->xdp_prog); 1184 if (!prog) 1185 return AM65_CPSW_XDP_PASS; 1186 1187 act = bpf_prog_run_xdp(prog, xdp); 1188 /* XDP prog might have changed packet data and boundaries */ 1189 *len = xdp->data_end - xdp->data; 1190 1191 switch (act) { 1192 case XDP_PASS: 1193 return AM65_CPSW_XDP_PASS; 1194 case XDP_TX: 1195 tx_chn = &common->tx_chns[cpu % AM65_CPSW_MAX_QUEUES]; 1196 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); 1197 1198 xdpf = xdp_convert_buff_to_frame(xdp); 1199 if (unlikely(!xdpf)) { 1200 ndev->stats.tx_dropped++; 1201 goto drop; 1202 } 1203 1204 __netif_tx_lock(netif_txq, cpu); 1205 err = am65_cpsw_xdp_tx_frame(ndev, tx_chn, xdpf, 1206 AM65_CPSW_TX_BUF_TYPE_XDP_TX); 1207 __netif_tx_unlock(netif_txq); 1208 if (err) 1209 goto drop; 1210 1211 dev_sw_netstats_rx_add(ndev, pkt_len); 1212 return AM65_CPSW_XDP_TX; 1213 case XDP_REDIRECT: 1214 if (unlikely(xdp_do_redirect(ndev, xdp, prog))) 1215 goto drop; 1216 1217 dev_sw_netstats_rx_add(ndev, pkt_len); 1218 return AM65_CPSW_XDP_REDIRECT; 1219 default: 1220 bpf_warn_invalid_xdp_action(ndev, prog, act); 1221 fallthrough; 1222 case XDP_ABORTED: 1223 drop: 1224 trace_xdp_exception(ndev, prog, act); 1225 fallthrough; 1226 case XDP_DROP: 1227 ndev->stats.rx_dropped++; 1228 } 1229 1230 return ret; 1231 } 1232 1233 /* RX psdata[2] word format - checksum information */ 1234 #define AM65_CPSW_RX_PSD_CSUM_ADD GENMASK(15, 0) 1235 #define AM65_CPSW_RX_PSD_CSUM_ERR BIT(16) 1236 #define AM65_CPSW_RX_PSD_IS_FRAGMENT BIT(17) 1237 #define AM65_CPSW_RX_PSD_IS_TCP BIT(18) 1238 #define AM65_CPSW_RX_PSD_IPV6_VALID BIT(19) 1239 #define AM65_CPSW_RX_PSD_IPV4_VALID BIT(20) 1240 1241 static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info) 1242 { 1243 /* HW can verify IPv4/IPv6 TCP/UDP packets checksum 1244 * csum information provides in psdata[2] word: 1245 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error 1246 * AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID 1247 * bits - indicates IPv4/IPv6 packet 1248 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet 1249 * AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets 1250 * or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR 1251 */ 1252 skb_checksum_none_assert(skb); 1253 1254 if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) 1255 return; 1256 1257 if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID | 1258 AM65_CPSW_RX_PSD_IPV4_VALID)) && 1259 !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) { 1260 /* csum for fragmented packets is unsupported */ 1261 if (!(csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT)) 1262 skb->ip_summed = CHECKSUM_UNNECESSARY; 1263 } 1264 } 1265 1266 static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow, 1267 int *xdp_state) 1268 { 1269 struct am65_cpsw_rx_chn *rx_chn = &flow->common->rx_chns; 1270 u32 buf_dma_len, pkt_len, port_id = 0, csum_info; 1271 struct am65_cpsw_common *common = flow->common; 1272 struct am65_cpsw_ndev_priv *ndev_priv; 1273 struct cppi5_host_desc_t *desc_rx; 1274 struct device *dev = common->dev; 1275 struct am65_cpsw_swdata *swdata; 1276 struct page *page, *new_page; 1277 dma_addr_t desc_dma, buf_dma; 1278 struct am65_cpsw_port *port; 1279 struct net_device *ndev; 1280 u32 flow_idx = flow->id; 1281 struct sk_buff *skb; 1282 struct xdp_buff xdp; 1283 int headroom, ret; 1284 void *page_addr; 1285 u32 *psdata; 1286 1287 *xdp_state = AM65_CPSW_XDP_PASS; 1288 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma); 1289 if (ret) { 1290 if (ret != -ENODATA) 1291 dev_err(dev, "RX: pop chn fail %d\n", ret); 1292 return ret; 1293 } 1294 1295 if (cppi5_desc_is_tdcm(desc_dma)) { 1296 dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx); 1297 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) 1298 complete(&common->tdown_complete); 1299 return 0; 1300 } 1301 1302 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); 1303 dev_dbg(dev, "%s flow_idx: %u desc %pad\n", 1304 __func__, flow_idx, &desc_dma); 1305 1306 swdata = cppi5_hdesc_get_swdata(desc_rx); 1307 page = swdata->page; 1308 page_addr = page_address(page); 1309 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); 1310 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); 1311 pkt_len = cppi5_hdesc_get_pktlen(desc_rx); 1312 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); 1313 dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id); 1314 port = am65_common_get_port(common, port_id); 1315 ndev = port->ndev; 1316 psdata = cppi5_hdesc_get_psdata(desc_rx); 1317 csum_info = psdata[2]; 1318 dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info); 1319 1320 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); 1321 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 1322 1323 if (port->xdp_prog) { 1324 xdp_init_buff(&xdp, PAGE_SIZE, &port->xdp_rxq[flow->id]); 1325 xdp_prepare_buff(&xdp, page_addr, AM65_CPSW_HEADROOM, 1326 pkt_len, false); 1327 *xdp_state = am65_cpsw_run_xdp(flow, port, &xdp, &pkt_len); 1328 if (*xdp_state == AM65_CPSW_XDP_CONSUMED) { 1329 page = virt_to_head_page(xdp.data); 1330 am65_cpsw_put_page(flow, page, true); 1331 goto allocate; 1332 } 1333 1334 if (*xdp_state != AM65_CPSW_XDP_PASS) 1335 goto allocate; 1336 1337 headroom = xdp.data - xdp.data_hard_start; 1338 } else { 1339 headroom = AM65_CPSW_HEADROOM; 1340 } 1341 1342 skb = am65_cpsw_build_skb(page_addr, ndev, 1343 PAGE_SIZE, headroom); 1344 if (unlikely(!skb)) { 1345 new_page = page; 1346 goto requeue; 1347 } 1348 1349 ndev_priv = netdev_priv(ndev); 1350 am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark); 1351 skb_put(skb, pkt_len); 1352 if (port->rx_ts_filter) 1353 am65_cpts_rx_timestamp(common->cpts, port_id, skb); 1354 skb_mark_for_recycle(skb); 1355 skb->protocol = eth_type_trans(skb, ndev); 1356 am65_cpsw_nuss_rx_csum(skb, csum_info); 1357 napi_gro_receive(&flow->napi_rx, skb); 1358 1359 dev_sw_netstats_rx_add(ndev, pkt_len); 1360 1361 allocate: 1362 new_page = page_pool_dev_alloc_pages(flow->page_pool); 1363 if (unlikely(!new_page)) { 1364 dev_err(dev, "page alloc failed\n"); 1365 return -ENOMEM; 1366 } 1367 1368 if (netif_dormant(ndev)) { 1369 am65_cpsw_put_page(flow, new_page, true); 1370 ndev->stats.rx_dropped++; 1371 return 0; 1372 } 1373 1374 requeue: 1375 ret = am65_cpsw_nuss_rx_push(common, new_page, flow_idx); 1376 if (WARN_ON(ret < 0)) { 1377 am65_cpsw_put_page(flow, new_page, true); 1378 ndev->stats.rx_errors++; 1379 ndev->stats.rx_dropped++; 1380 } 1381 1382 return ret; 1383 } 1384 1385 static enum hrtimer_restart am65_cpsw_nuss_rx_timer_callback(struct hrtimer *timer) 1386 { 1387 struct am65_cpsw_rx_flow *flow = container_of(timer, 1388 struct am65_cpsw_rx_flow, 1389 rx_hrtimer); 1390 1391 enable_irq(flow->irq); 1392 return HRTIMER_NORESTART; 1393 } 1394 1395 static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget) 1396 { 1397 struct am65_cpsw_rx_flow *flow = am65_cpsw_napi_to_rx_flow(napi_rx); 1398 struct am65_cpsw_common *common = flow->common; 1399 int xdp_state_or = 0; 1400 int cur_budget, ret; 1401 int xdp_state; 1402 int num_rx = 0; 1403 1404 /* process only this flow */ 1405 cur_budget = budget; 1406 while (cur_budget--) { 1407 ret = am65_cpsw_nuss_rx_packets(flow, &xdp_state); 1408 xdp_state_or |= xdp_state; 1409 if (ret) 1410 break; 1411 num_rx++; 1412 } 1413 1414 if (xdp_state_or & AM65_CPSW_XDP_REDIRECT) 1415 xdp_do_flush(); 1416 1417 dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget); 1418 1419 if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) { 1420 if (flow->irq_disabled) { 1421 flow->irq_disabled = false; 1422 if (unlikely(flow->rx_pace_timeout)) { 1423 hrtimer_start(&flow->rx_hrtimer, 1424 ns_to_ktime(flow->rx_pace_timeout), 1425 HRTIMER_MODE_REL_PINNED); 1426 } else { 1427 enable_irq(flow->irq); 1428 } 1429 } 1430 } 1431 1432 return num_rx; 1433 } 1434 1435 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev, 1436 struct netdev_queue *netif_txq) 1437 { 1438 if (netif_tx_queue_stopped(netif_txq)) { 1439 /* Check whether the queue is stopped due to stalled 1440 * tx dma, if the queue is stopped then wake the queue 1441 * as we have free desc for tx 1442 */ 1443 __netif_tx_lock(netif_txq, smp_processor_id()); 1444 if (netif_running(ndev) && 1445 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS)) 1446 netif_tx_wake_queue(netif_txq); 1447 1448 __netif_tx_unlock(netif_txq); 1449 } 1450 } 1451 1452 static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, 1453 int chn, unsigned int budget, bool *tdown) 1454 { 1455 bool single_port = AM65_CPSW_IS_CPSW2G(common); 1456 enum am65_cpsw_tx_buf_type buf_type; 1457 struct am65_cpsw_tx_swdata *swdata; 1458 struct cppi5_host_desc_t *desc_tx; 1459 struct device *dev = common->dev; 1460 struct am65_cpsw_tx_chn *tx_chn; 1461 struct netdev_queue *netif_txq; 1462 unsigned int total_bytes = 0; 1463 struct net_device *ndev; 1464 struct xdp_frame *xdpf; 1465 unsigned int pkt_len; 1466 struct sk_buff *skb; 1467 dma_addr_t desc_dma; 1468 int res, num_tx = 0; 1469 1470 tx_chn = &common->tx_chns[chn]; 1471 1472 while (true) { 1473 if (!single_port) 1474 spin_lock(&tx_chn->lock); 1475 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma); 1476 if (!single_port) 1477 spin_unlock(&tx_chn->lock); 1478 1479 if (res == -ENODATA) 1480 break; 1481 1482 if (cppi5_desc_is_tdcm(desc_dma)) { 1483 if (atomic_dec_and_test(&common->tdown_cnt)) 1484 complete(&common->tdown_complete); 1485 *tdown = true; 1486 break; 1487 } 1488 1489 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, 1490 desc_dma); 1491 swdata = cppi5_hdesc_get_swdata(desc_tx); 1492 ndev = swdata->ndev; 1493 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma); 1494 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) { 1495 skb = swdata->skb; 1496 am65_cpts_tx_timestamp(tx_chn->common->cpts, skb); 1497 pkt_len = skb->len; 1498 napi_consume_skb(skb, budget); 1499 } else { 1500 xdpf = swdata->xdpf; 1501 pkt_len = xdpf->len; 1502 if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX) 1503 xdp_return_frame_rx_napi(xdpf); 1504 else 1505 xdp_return_frame(xdpf); 1506 } 1507 1508 total_bytes += pkt_len; 1509 num_tx++; 1510 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx); 1511 dev_sw_netstats_tx_add(ndev, 1, pkt_len); 1512 if (!single_port) { 1513 /* as packets from multi ports can be interleaved 1514 * on the same channel, we have to figure out the 1515 * port/queue at every packet and report it/wake queue. 1516 */ 1517 netif_txq = netdev_get_tx_queue(ndev, chn); 1518 netdev_tx_completed_queue(netif_txq, 1, pkt_len); 1519 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq); 1520 } 1521 } 1522 1523 if (single_port && num_tx) { 1524 netif_txq = netdev_get_tx_queue(ndev, chn); 1525 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes); 1526 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq); 1527 } 1528 1529 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx); 1530 1531 return num_tx; 1532 } 1533 1534 static enum hrtimer_restart am65_cpsw_nuss_tx_timer_callback(struct hrtimer *timer) 1535 { 1536 struct am65_cpsw_tx_chn *tx_chns = 1537 container_of(timer, struct am65_cpsw_tx_chn, tx_hrtimer); 1538 1539 enable_irq(tx_chns->irq); 1540 return HRTIMER_NORESTART; 1541 } 1542 1543 static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget) 1544 { 1545 struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx); 1546 bool tdown = false; 1547 int num_tx; 1548 1549 num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, 1550 tx_chn->id, budget, &tdown); 1551 if (num_tx >= budget) 1552 return budget; 1553 1554 if (napi_complete_done(napi_tx, num_tx)) { 1555 if (unlikely(tx_chn->tx_pace_timeout && !tdown)) { 1556 hrtimer_start(&tx_chn->tx_hrtimer, 1557 ns_to_ktime(tx_chn->tx_pace_timeout), 1558 HRTIMER_MODE_REL_PINNED); 1559 } else { 1560 enable_irq(tx_chn->irq); 1561 } 1562 } 1563 1564 return 0; 1565 } 1566 1567 static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id) 1568 { 1569 struct am65_cpsw_rx_flow *flow = dev_id; 1570 1571 flow->irq_disabled = true; 1572 disable_irq_nosync(irq); 1573 napi_schedule(&flow->napi_rx); 1574 1575 return IRQ_HANDLED; 1576 } 1577 1578 static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id) 1579 { 1580 struct am65_cpsw_tx_chn *tx_chn = dev_id; 1581 1582 disable_irq_nosync(irq); 1583 napi_schedule(&tx_chn->napi_tx); 1584 1585 return IRQ_HANDLED; 1586 } 1587 1588 static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb, 1589 struct net_device *ndev) 1590 { 1591 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 1592 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc; 1593 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1594 struct am65_cpsw_tx_swdata *swdata; 1595 struct device *dev = common->dev; 1596 struct am65_cpsw_tx_chn *tx_chn; 1597 struct netdev_queue *netif_txq; 1598 dma_addr_t desc_dma, buf_dma; 1599 int ret, q_idx, i; 1600 u32 *psdata; 1601 u32 pkt_len; 1602 1603 /* padding enabled in hw */ 1604 pkt_len = skb_headlen(skb); 1605 1606 /* SKB TX timestamp */ 1607 if (port->tx_ts_enabled) 1608 am65_cpts_prep_tx_timestamp(common->cpts, port->port_id, skb); 1609 1610 q_idx = skb_get_queue_mapping(skb); 1611 dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx); 1612 1613 tx_chn = &common->tx_chns[q_idx]; 1614 netif_txq = netdev_get_tx_queue(ndev, q_idx); 1615 1616 /* Map the linear buffer */ 1617 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len, 1618 DMA_TO_DEVICE); 1619 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) { 1620 dev_err(dev, "Failed to map tx skb buffer\n"); 1621 ndev->stats.tx_errors++; 1622 goto err_free_skb; 1623 } 1624 1625 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); 1626 if (!first_desc) { 1627 dev_dbg(dev, "Failed to allocate descriptor\n"); 1628 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len, 1629 DMA_TO_DEVICE); 1630 goto busy_stop_q; 1631 } 1632 1633 am65_cpsw_nuss_set_buf_type(tx_chn, first_desc, 1634 AM65_CPSW_TX_BUF_TYPE_SKB); 1635 1636 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT, 1637 AM65_CPSW_NAV_PS_DATA_SIZE); 1638 cppi5_desc_set_pktids(&first_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID); 1639 cppi5_hdesc_set_pkttype(first_desc, AM65_CPSW_CPPI_TX_PKT_TYPE); 1640 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id); 1641 1642 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); 1643 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len); 1644 swdata = cppi5_hdesc_get_swdata(first_desc); 1645 swdata->ndev = ndev; 1646 swdata->skb = skb; 1647 psdata = cppi5_hdesc_get_psdata(first_desc); 1648 1649 /* HW csum offload if enabled */ 1650 psdata[2] = 0; 1651 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 1652 unsigned int cs_start, cs_offset; 1653 1654 cs_start = skb_transport_offset(skb); 1655 cs_offset = cs_start + skb->csum_offset; 1656 /* HW numerates bytes starting from 1 */ 1657 psdata[2] = ((cs_offset + 1) << 24) | 1658 ((cs_start + 1) << 16) | (skb->len - cs_start); 1659 dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]); 1660 } 1661 1662 if (!skb_is_nonlinear(skb)) 1663 goto done_tx; 1664 1665 dev_dbg(dev, "fragmented SKB\n"); 1666 1667 /* Handle the case where skb is fragmented in pages */ 1668 cur_desc = first_desc; 1669 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1670 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1671 u32 frag_size = skb_frag_size(frag); 1672 1673 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); 1674 if (!next_desc) { 1675 dev_err(dev, "Failed to allocate descriptor\n"); 1676 goto busy_free_descs; 1677 } 1678 1679 am65_cpsw_nuss_set_buf_type(tx_chn, next_desc, 1680 AM65_CPSW_TX_BUF_TYPE_SKB); 1681 1682 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size, 1683 DMA_TO_DEVICE); 1684 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) { 1685 dev_err(dev, "Failed to map tx skb page\n"); 1686 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); 1687 ndev->stats.tx_errors++; 1688 goto err_free_descs; 1689 } 1690 1691 cppi5_hdesc_reset_hbdesc(next_desc); 1692 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); 1693 cppi5_hdesc_attach_buf(next_desc, 1694 buf_dma, frag_size, buf_dma, frag_size); 1695 1696 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, 1697 next_desc); 1698 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma); 1699 cppi5_hdesc_link_hbdesc(cur_desc, desc_dma); 1700 1701 pkt_len += frag_size; 1702 cur_desc = next_desc; 1703 } 1704 WARN_ON(pkt_len != skb->len); 1705 1706 done_tx: 1707 skb_tx_timestamp(skb); 1708 1709 /* report bql before sending packet */ 1710 netdev_tx_sent_queue(netif_txq, pkt_len); 1711 1712 cppi5_hdesc_set_pktlen(first_desc, pkt_len); 1713 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc); 1714 if (AM65_CPSW_IS_CPSW2G(common)) { 1715 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma); 1716 } else { 1717 spin_lock_bh(&tx_chn->lock); 1718 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma); 1719 spin_unlock_bh(&tx_chn->lock); 1720 } 1721 if (ret) { 1722 dev_err(dev, "can't push desc %d\n", ret); 1723 /* inform bql */ 1724 netdev_tx_completed_queue(netif_txq, 1, pkt_len); 1725 ndev->stats.tx_errors++; 1726 goto err_free_descs; 1727 } 1728 1729 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) { 1730 netif_tx_stop_queue(netif_txq); 1731 /* Barrier, so that stop_queue visible to other cpus */ 1732 smp_mb__after_atomic(); 1733 dev_dbg(dev, "netif_tx_stop_queue %d\n", q_idx); 1734 1735 /* re-check for smp */ 1736 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= 1737 MAX_SKB_FRAGS) { 1738 netif_tx_wake_queue(netif_txq); 1739 dev_dbg(dev, "netif_tx_wake_queue %d\n", q_idx); 1740 } 1741 } 1742 1743 return NETDEV_TX_OK; 1744 1745 err_free_descs: 1746 am65_cpsw_nuss_xmit_free(tx_chn, first_desc); 1747 err_free_skb: 1748 ndev->stats.tx_dropped++; 1749 dev_kfree_skb_any(skb); 1750 return NETDEV_TX_OK; 1751 1752 busy_free_descs: 1753 am65_cpsw_nuss_xmit_free(tx_chn, first_desc); 1754 busy_stop_q: 1755 netif_tx_stop_queue(netif_txq); 1756 return NETDEV_TX_BUSY; 1757 } 1758 1759 static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev, 1760 void *addr) 1761 { 1762 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 1763 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1764 struct sockaddr *sockaddr = (struct sockaddr *)addr; 1765 int ret; 1766 1767 ret = eth_prepare_mac_addr_change(ndev, addr); 1768 if (ret < 0) 1769 return ret; 1770 1771 ret = pm_runtime_resume_and_get(common->dev); 1772 if (ret < 0) 1773 return ret; 1774 1775 cpsw_ale_del_ucast(common->ale, ndev->dev_addr, 1776 HOST_PORT_NUM, 0, 0); 1777 cpsw_ale_add_ucast(common->ale, sockaddr->sa_data, 1778 HOST_PORT_NUM, ALE_SECURE, 0); 1779 1780 am65_cpsw_port_set_sl_mac(port, addr); 1781 eth_commit_mac_addr_change(ndev, sockaddr); 1782 1783 pm_runtime_put(common->dev); 1784 1785 return 0; 1786 } 1787 1788 static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev, 1789 struct kernel_hwtstamp_config *cfg, 1790 struct netlink_ext_ack *extack) 1791 { 1792 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1793 u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype; 1794 1795 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS)) { 1796 NL_SET_ERR_MSG(extack, "Time stamping is not supported"); 1797 return -EOPNOTSUPP; 1798 } 1799 1800 /* TX HW timestamp */ 1801 switch (cfg->tx_type) { 1802 case HWTSTAMP_TX_OFF: 1803 case HWTSTAMP_TX_ON: 1804 break; 1805 default: 1806 NL_SET_ERR_MSG(extack, "TX mode is not supported"); 1807 return -ERANGE; 1808 } 1809 1810 switch (cfg->rx_filter) { 1811 case HWTSTAMP_FILTER_NONE: 1812 port->rx_ts_filter = HWTSTAMP_FILTER_NONE; 1813 break; 1814 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 1815 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 1816 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 1817 port->rx_ts_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 1818 cfg->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 1819 break; 1820 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 1821 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 1822 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 1823 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 1824 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 1825 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 1826 case HWTSTAMP_FILTER_PTP_V2_EVENT: 1827 case HWTSTAMP_FILTER_PTP_V2_SYNC: 1828 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 1829 port->rx_ts_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 1830 cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 1831 break; 1832 case HWTSTAMP_FILTER_ALL: 1833 case HWTSTAMP_FILTER_SOME: 1834 case HWTSTAMP_FILTER_NTP_ALL: 1835 NL_SET_ERR_MSG(extack, "RX filter is not supported"); 1836 return -EOPNOTSUPP; 1837 default: 1838 NL_SET_ERR_MSG(extack, "RX filter is not supported"); 1839 return -ERANGE; 1840 } 1841 1842 port->tx_ts_enabled = (cfg->tx_type == HWTSTAMP_TX_ON); 1843 1844 /* cfg TX timestamp */ 1845 seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET << 1846 AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588; 1847 1848 ts_vlan_ltype = ETH_P_8021Q; 1849 1850 ts_ctrl_ltype2 = ETH_P_1588 | 1851 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 | 1852 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 | 1853 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 | 1854 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 | 1855 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 | 1856 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 | 1857 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 | 1858 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO; 1859 1860 ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS << 1861 AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT; 1862 1863 if (port->tx_ts_enabled) 1864 ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN | 1865 AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN; 1866 1867 if (port->rx_ts_filter) 1868 ts_ctrl |= AM65_CPSW_TS_RX_ANX_ALL_EN | 1869 AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN; 1870 1871 writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG); 1872 writel(ts_vlan_ltype, port->port_base + 1873 AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG); 1874 writel(ts_ctrl_ltype2, port->port_base + 1875 AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2); 1876 writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL); 1877 1878 return 0; 1879 } 1880 1881 static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev, 1882 struct kernel_hwtstamp_config *cfg) 1883 { 1884 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1885 1886 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS)) 1887 return -EOPNOTSUPP; 1888 1889 cfg->flags = 0; 1890 cfg->tx_type = port->tx_ts_enabled ? 1891 HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; 1892 cfg->rx_filter = port->rx_ts_filter; 1893 1894 return 0; 1895 } 1896 1897 static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev, 1898 struct ifreq *req, int cmd) 1899 { 1900 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1901 1902 if (!netif_running(ndev)) 1903 return -EINVAL; 1904 1905 return phylink_mii_ioctl(port->slave.phylink, req, cmd); 1906 } 1907 1908 static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev, 1909 struct rtnl_link_stats64 *stats) 1910 { 1911 dev_fetch_sw_netstats(stats, dev->tstats); 1912 1913 stats->rx_errors = dev->stats.rx_errors; 1914 stats->rx_dropped = dev->stats.rx_dropped; 1915 stats->tx_dropped = dev->stats.tx_dropped; 1916 } 1917 1918 static int am65_cpsw_xdp_prog_setup(struct net_device *ndev, 1919 struct bpf_prog *prog) 1920 { 1921 struct am65_cpsw_port *port = am65_ndev_to_port(ndev); 1922 bool running = netif_running(ndev); 1923 struct bpf_prog *old_prog; 1924 1925 if (running) 1926 am65_cpsw_nuss_ndo_slave_stop(ndev); 1927 1928 old_prog = xchg(&port->xdp_prog, prog); 1929 if (old_prog) 1930 bpf_prog_put(old_prog); 1931 1932 if (running) 1933 return am65_cpsw_nuss_ndo_slave_open(ndev); 1934 1935 return 0; 1936 } 1937 1938 static int am65_cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 1939 { 1940 switch (bpf->command) { 1941 case XDP_SETUP_PROG: 1942 return am65_cpsw_xdp_prog_setup(ndev, bpf->prog); 1943 default: 1944 return -EINVAL; 1945 } 1946 } 1947 1948 static int am65_cpsw_ndo_xdp_xmit(struct net_device *ndev, int n, 1949 struct xdp_frame **frames, u32 flags) 1950 { 1951 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 1952 struct am65_cpsw_tx_chn *tx_chn; 1953 struct netdev_queue *netif_txq; 1954 int cpu = smp_processor_id(); 1955 int i, nxmit = 0; 1956 1957 tx_chn = &common->tx_chns[cpu % common->tx_ch_num]; 1958 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id); 1959 1960 __netif_tx_lock(netif_txq, cpu); 1961 for (i = 0; i < n; i++) { 1962 if (am65_cpsw_xdp_tx_frame(ndev, tx_chn, frames[i], 1963 AM65_CPSW_TX_BUF_TYPE_XDP_NDO)) 1964 break; 1965 nxmit++; 1966 } 1967 __netif_tx_unlock(netif_txq); 1968 1969 return nxmit; 1970 } 1971 1972 static const struct net_device_ops am65_cpsw_nuss_netdev_ops = { 1973 .ndo_open = am65_cpsw_nuss_ndo_slave_open, 1974 .ndo_stop = am65_cpsw_nuss_ndo_slave_stop, 1975 .ndo_start_xmit = am65_cpsw_nuss_ndo_slave_xmit, 1976 .ndo_set_rx_mode = am65_cpsw_nuss_ndo_slave_set_rx_mode, 1977 .ndo_get_stats64 = am65_cpsw_nuss_ndo_get_stats, 1978 .ndo_validate_addr = eth_validate_addr, 1979 .ndo_set_mac_address = am65_cpsw_nuss_ndo_slave_set_mac_address, 1980 .ndo_tx_timeout = am65_cpsw_nuss_ndo_host_tx_timeout, 1981 .ndo_vlan_rx_add_vid = am65_cpsw_nuss_ndo_slave_add_vid, 1982 .ndo_vlan_rx_kill_vid = am65_cpsw_nuss_ndo_slave_kill_vid, 1983 .ndo_eth_ioctl = am65_cpsw_nuss_ndo_slave_ioctl, 1984 .ndo_setup_tc = am65_cpsw_qos_ndo_setup_tc, 1985 .ndo_set_tx_maxrate = am65_cpsw_qos_ndo_tx_p0_set_maxrate, 1986 .ndo_bpf = am65_cpsw_ndo_bpf, 1987 .ndo_xdp_xmit = am65_cpsw_ndo_xdp_xmit, 1988 .ndo_hwtstamp_get = am65_cpsw_nuss_hwtstamp_get, 1989 .ndo_hwtstamp_set = am65_cpsw_nuss_hwtstamp_set, 1990 }; 1991 1992 static void am65_cpsw_disable_phy(struct phy *phy) 1993 { 1994 phy_power_off(phy); 1995 phy_exit(phy); 1996 } 1997 1998 static int am65_cpsw_enable_phy(struct phy *phy) 1999 { 2000 int ret; 2001 2002 ret = phy_init(phy); 2003 if (ret < 0) 2004 return ret; 2005 2006 ret = phy_power_on(phy); 2007 if (ret < 0) { 2008 phy_exit(phy); 2009 return ret; 2010 } 2011 2012 return 0; 2013 } 2014 2015 static void am65_cpsw_disable_serdes_phy(struct am65_cpsw_common *common) 2016 { 2017 struct am65_cpsw_port *port; 2018 struct phy *phy; 2019 int i; 2020 2021 for (i = 0; i < common->port_num; i++) { 2022 port = &common->ports[i]; 2023 phy = port->slave.serdes_phy; 2024 if (phy) 2025 am65_cpsw_disable_phy(phy); 2026 } 2027 } 2028 2029 static int am65_cpsw_init_serdes_phy(struct device *dev, struct device_node *port_np, 2030 struct am65_cpsw_port *port) 2031 { 2032 const char *name = "serdes"; 2033 struct phy *phy; 2034 int ret; 2035 2036 phy = devm_of_phy_optional_get(dev, port_np, name); 2037 if (IS_ERR_OR_NULL(phy)) 2038 return PTR_ERR_OR_ZERO(phy); 2039 2040 /* Serdes PHY exists. Store it. */ 2041 port->slave.serdes_phy = phy; 2042 2043 ret = am65_cpsw_enable_phy(phy); 2044 if (ret < 0) 2045 goto err_phy; 2046 2047 return 0; 2048 2049 err_phy: 2050 devm_phy_put(dev, phy); 2051 return ret; 2052 } 2053 2054 static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned int mode, 2055 const struct phylink_link_state *state) 2056 { 2057 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data, 2058 phylink_config); 2059 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave); 2060 struct am65_cpsw_common *common = port->common; 2061 2062 if (common->pdata.extra_modes & BIT(state->interface)) { 2063 if (state->interface == PHY_INTERFACE_MODE_SGMII) { 2064 writel(ADVERTISE_SGMII, 2065 port->sgmii_base + AM65_CPSW_SGMII_MR_ADV_ABILITY_REG); 2066 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN); 2067 } else { 2068 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN); 2069 } 2070 2071 if (state->interface == PHY_INTERFACE_MODE_USXGMII) { 2072 cpsw_sl_ctl_set(port->slave.mac_sl, 2073 CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN); 2074 } else { 2075 cpsw_sl_ctl_clr(port->slave.mac_sl, 2076 CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN); 2077 } 2078 2079 writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE, 2080 port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG); 2081 } 2082 } 2083 2084 static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode, 2085 phy_interface_t interface) 2086 { 2087 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data, 2088 phylink_config); 2089 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave); 2090 struct am65_cpsw_common *common = port->common; 2091 struct net_device *ndev = port->ndev; 2092 u32 mac_control; 2093 int tmo; 2094 2095 /* disable forwarding */ 2096 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_DISABLE); 2097 2098 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE); 2099 2100 tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100); 2101 dev_dbg(common->dev, "down msc_sl %08x tmo %d\n", 2102 cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo); 2103 2104 /* All the bits that am65_cpsw_nuss_mac_link_up() can possibly set */ 2105 mac_control = CPSW_SL_CTL_GMII_EN | CPSW_SL_CTL_GIG | CPSW_SL_CTL_IFCTL_A | 2106 CPSW_SL_CTL_FULLDUPLEX | CPSW_SL_CTL_RX_FLOW_EN | CPSW_SL_CTL_TX_FLOW_EN; 2107 /* If interface mode is RGMII, CPSW_SL_CTL_EXT_EN might have been set for 10 Mbps */ 2108 if (phy_interface_mode_is_rgmii(interface)) 2109 mac_control |= CPSW_SL_CTL_EXT_EN; 2110 /* Only clear those bits that can be set by am65_cpsw_nuss_mac_link_up() */ 2111 cpsw_sl_ctl_clr(port->slave.mac_sl, mac_control); 2112 2113 am65_cpsw_qos_link_down(ndev); 2114 netif_tx_stop_all_queues(ndev); 2115 } 2116 2117 static void am65_cpsw_nuss_mac_link_up(struct phylink_config *config, struct phy_device *phy, 2118 unsigned int mode, phy_interface_t interface, int speed, 2119 int duplex, bool tx_pause, bool rx_pause) 2120 { 2121 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data, 2122 phylink_config); 2123 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave); 2124 struct am65_cpsw_common *common = port->common; 2125 u32 mac_control = CPSW_SL_CTL_GMII_EN; 2126 struct net_device *ndev = port->ndev; 2127 2128 /* Bring the port out of idle state */ 2129 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE); 2130 2131 if (speed == SPEED_1000) 2132 mac_control |= CPSW_SL_CTL_GIG; 2133 /* TODO: Verify whether in-band is necessary for 10 Mbps RGMII */ 2134 if (speed == SPEED_10 && phy_interface_mode_is_rgmii(interface)) 2135 /* Can be used with in band mode only */ 2136 mac_control |= CPSW_SL_CTL_EXT_EN; 2137 if (speed == SPEED_100 && interface == PHY_INTERFACE_MODE_RMII) 2138 mac_control |= CPSW_SL_CTL_IFCTL_A; 2139 if (duplex) 2140 mac_control |= CPSW_SL_CTL_FULLDUPLEX; 2141 2142 /* rx_pause/tx_pause */ 2143 if (rx_pause) 2144 mac_control |= CPSW_SL_CTL_TX_FLOW_EN; 2145 2146 if (tx_pause) 2147 mac_control |= CPSW_SL_CTL_RX_FLOW_EN; 2148 2149 cpsw_sl_ctl_set(port->slave.mac_sl, mac_control); 2150 2151 /* enable forwarding */ 2152 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_FORWARD); 2153 2154 am65_cpsw_qos_link_up(ndev, speed); 2155 netif_tx_wake_all_queues(ndev); 2156 } 2157 2158 static const struct phylink_mac_ops am65_cpsw_phylink_mac_ops = { 2159 .mac_config = am65_cpsw_nuss_mac_config, 2160 .mac_link_down = am65_cpsw_nuss_mac_link_down, 2161 .mac_link_up = am65_cpsw_nuss_mac_link_up, 2162 }; 2163 2164 static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port) 2165 { 2166 struct am65_cpsw_common *common = port->common; 2167 2168 if (!port->disabled) 2169 return; 2170 2171 cpsw_ale_control_set(common->ale, port->port_id, 2172 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE); 2173 2174 cpsw_sl_reset(port->slave.mac_sl, 100); 2175 cpsw_sl_ctl_reset(port->slave.mac_sl); 2176 } 2177 2178 static void am65_cpsw_nuss_free_tx_chns(void *data) 2179 { 2180 struct am65_cpsw_common *common = data; 2181 int i; 2182 2183 for (i = 0; i < common->tx_ch_num; i++) { 2184 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; 2185 2186 if (!IS_ERR_OR_NULL(tx_chn->desc_pool)) 2187 k3_cppi_desc_pool_destroy(tx_chn->desc_pool); 2188 2189 if (!IS_ERR_OR_NULL(tx_chn->tx_chn)) 2190 k3_udma_glue_release_tx_chn(tx_chn->tx_chn); 2191 2192 memset(tx_chn, 0, sizeof(*tx_chn)); 2193 } 2194 } 2195 2196 static void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common) 2197 { 2198 struct device *dev = common->dev; 2199 int i; 2200 2201 common->tx_ch_rate_msk = 0; 2202 for (i = 0; i < common->tx_ch_num; i++) { 2203 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; 2204 2205 if (tx_chn->irq > 0) 2206 devm_free_irq(dev, tx_chn->irq, tx_chn); 2207 2208 netif_napi_del(&tx_chn->napi_tx); 2209 } 2210 2211 am65_cpsw_nuss_free_tx_chns(common); 2212 } 2213 2214 static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common) 2215 { 2216 struct device *dev = common->dev; 2217 struct am65_cpsw_tx_chn *tx_chn; 2218 int i, ret = 0; 2219 2220 for (i = 0; i < common->tx_ch_num; i++) { 2221 tx_chn = &common->tx_chns[i]; 2222 2223 hrtimer_setup(&tx_chn->tx_hrtimer, &am65_cpsw_nuss_tx_timer_callback, 2224 CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 2225 2226 netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx, 2227 am65_cpsw_nuss_tx_poll); 2228 2229 ret = devm_request_irq(dev, tx_chn->irq, 2230 am65_cpsw_nuss_tx_irq, 2231 IRQF_TRIGGER_HIGH, 2232 tx_chn->tx_chn_name, tx_chn); 2233 if (ret) { 2234 dev_err(dev, "failure requesting tx%u irq %u, %d\n", 2235 tx_chn->id, tx_chn->irq, ret); 2236 goto err; 2237 } 2238 } 2239 2240 return 0; 2241 2242 err: 2243 netif_napi_del(&tx_chn->napi_tx); 2244 for (--i; i >= 0; i--) { 2245 tx_chn = &common->tx_chns[i]; 2246 devm_free_irq(dev, tx_chn->irq, tx_chn); 2247 netif_napi_del(&tx_chn->napi_tx); 2248 } 2249 2250 return ret; 2251 } 2252 2253 static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common) 2254 { 2255 u32 max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS); 2256 struct k3_udma_glue_tx_channel_cfg tx_cfg = { 0 }; 2257 struct device *dev = common->dev; 2258 struct k3_ring_cfg ring_cfg = { 2259 .elm_size = K3_RINGACC_RING_ELSIZE_8, 2260 .mode = K3_RINGACC_RING_MODE_RING, 2261 .flags = 0 2262 }; 2263 u32 hdesc_size, hdesc_size_out; 2264 int i, ret = 0; 2265 2266 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE, 2267 AM65_CPSW_NAV_SW_DATA_SIZE); 2268 2269 tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE; 2270 tx_cfg.tx_cfg = ring_cfg; 2271 tx_cfg.txcq_cfg = ring_cfg; 2272 tx_cfg.tx_cfg.size = max_desc_num; 2273 tx_cfg.txcq_cfg.size = max_desc_num; 2274 2275 for (i = 0; i < common->tx_ch_num; i++) { 2276 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; 2277 2278 snprintf(tx_chn->tx_chn_name, 2279 sizeof(tx_chn->tx_chn_name), "tx%d", i); 2280 2281 spin_lock_init(&tx_chn->lock); 2282 tx_chn->common = common; 2283 tx_chn->id = i; 2284 tx_chn->descs_num = max_desc_num; 2285 2286 tx_chn->tx_chn = 2287 k3_udma_glue_request_tx_chn(dev, 2288 tx_chn->tx_chn_name, 2289 &tx_cfg); 2290 if (IS_ERR(tx_chn->tx_chn)) { 2291 ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn), 2292 "Failed to request tx dma channel\n"); 2293 goto err; 2294 } 2295 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn); 2296 2297 tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev, 2298 tx_chn->descs_num, 2299 hdesc_size, 2300 tx_chn->tx_chn_name); 2301 if (IS_ERR(tx_chn->desc_pool)) { 2302 ret = PTR_ERR(tx_chn->desc_pool); 2303 dev_err(dev, "Failed to create poll %d\n", ret); 2304 goto err; 2305 } 2306 2307 hdesc_size_out = k3_cppi_desc_pool_desc_size(tx_chn->desc_pool); 2308 tx_chn->dsize_log2 = __fls(hdesc_size_out); 2309 WARN_ON(hdesc_size_out != (1 << tx_chn->dsize_log2)); 2310 2311 tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn); 2312 if (tx_chn->irq < 0) { 2313 dev_err(dev, "Failed to get tx dma irq %d\n", 2314 tx_chn->irq); 2315 ret = tx_chn->irq; 2316 goto err; 2317 } 2318 2319 snprintf(tx_chn->tx_chn_name, 2320 sizeof(tx_chn->tx_chn_name), "%s-tx%d", 2321 dev_name(dev), tx_chn->id); 2322 } 2323 2324 ret = am65_cpsw_nuss_ndev_add_tx_napi(common); 2325 if (ret) { 2326 dev_err(dev, "Failed to add tx NAPI %d\n", ret); 2327 goto err; 2328 } 2329 2330 return 0; 2331 2332 err: 2333 am65_cpsw_nuss_free_tx_chns(common); 2334 2335 return ret; 2336 } 2337 2338 static void am65_cpsw_nuss_free_rx_chns(void *data) 2339 { 2340 struct am65_cpsw_common *common = data; 2341 struct am65_cpsw_rx_chn *rx_chn; 2342 2343 rx_chn = &common->rx_chns; 2344 2345 if (!IS_ERR_OR_NULL(rx_chn->desc_pool)) 2346 k3_cppi_desc_pool_destroy(rx_chn->desc_pool); 2347 2348 if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) 2349 k3_udma_glue_release_rx_chn(rx_chn->rx_chn); 2350 } 2351 2352 static void am65_cpsw_nuss_remove_rx_chns(struct am65_cpsw_common *common) 2353 { 2354 struct device *dev = common->dev; 2355 struct am65_cpsw_rx_chn *rx_chn; 2356 struct am65_cpsw_rx_flow *flows; 2357 int i; 2358 2359 rx_chn = &common->rx_chns; 2360 flows = rx_chn->flows; 2361 2362 for (i = 0; i < common->rx_ch_num_flows; i++) { 2363 if (!(flows[i].irq < 0)) 2364 devm_free_irq(dev, flows[i].irq, &flows[i]); 2365 netif_napi_del(&flows[i].napi_rx); 2366 } 2367 2368 am65_cpsw_nuss_free_rx_chns(common); 2369 2370 common->rx_flow_id_base = -1; 2371 } 2372 2373 static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) 2374 { 2375 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; 2376 struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 }; 2377 u32 max_desc_num = AM65_CPSW_MAX_RX_DESC; 2378 struct device *dev = common->dev; 2379 struct am65_cpsw_rx_flow *flow; 2380 u32 hdesc_size, hdesc_size_out; 2381 u32 fdqring_id; 2382 int i, ret = 0; 2383 2384 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE, 2385 AM65_CPSW_NAV_SW_DATA_SIZE); 2386 2387 rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE; 2388 rx_cfg.flow_id_num = common->rx_ch_num_flows; 2389 rx_cfg.flow_id_base = common->rx_flow_id_base; 2390 2391 /* init all flows */ 2392 rx_chn->dev = dev; 2393 rx_chn->descs_num = max_desc_num * rx_cfg.flow_id_num; 2394 2395 for (i = 0; i < common->rx_ch_num_flows; i++) { 2396 flow = &rx_chn->flows[i]; 2397 flow->page_pool = NULL; 2398 } 2399 2400 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg); 2401 if (IS_ERR(rx_chn->rx_chn)) { 2402 ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn), 2403 "Failed to request rx dma channel\n"); 2404 goto err; 2405 } 2406 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn); 2407 2408 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev, 2409 rx_chn->descs_num, 2410 hdesc_size, "rx"); 2411 if (IS_ERR(rx_chn->desc_pool)) { 2412 ret = PTR_ERR(rx_chn->desc_pool); 2413 dev_err(dev, "Failed to create rx poll %d\n", ret); 2414 goto err; 2415 } 2416 2417 hdesc_size_out = k3_cppi_desc_pool_desc_size(rx_chn->desc_pool); 2418 rx_chn->dsize_log2 = __fls(hdesc_size_out); 2419 WARN_ON(hdesc_size_out != (1 << rx_chn->dsize_log2)); 2420 2421 common->rx_flow_id_base = 2422 k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn); 2423 dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base); 2424 2425 fdqring_id = K3_RINGACC_RING_ID_ANY; 2426 for (i = 0; i < rx_cfg.flow_id_num; i++) { 2427 struct k3_ring_cfg rxring_cfg = { 2428 .elm_size = K3_RINGACC_RING_ELSIZE_8, 2429 .mode = K3_RINGACC_RING_MODE_RING, 2430 .flags = 0, 2431 }; 2432 struct k3_ring_cfg fdqring_cfg = { 2433 .elm_size = K3_RINGACC_RING_ELSIZE_8, 2434 .flags = K3_RINGACC_RING_SHARED, 2435 }; 2436 struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = { 2437 .rx_cfg = rxring_cfg, 2438 .rxfdq_cfg = fdqring_cfg, 2439 .ring_rxq_id = K3_RINGACC_RING_ID_ANY, 2440 .src_tag_lo_sel = 2441 K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG, 2442 }; 2443 2444 flow = &rx_chn->flows[i]; 2445 flow->id = i; 2446 flow->common = common; 2447 flow->irq = -EINVAL; 2448 2449 rx_flow_cfg.ring_rxfdq0_id = fdqring_id; 2450 rx_flow_cfg.rx_cfg.size = max_desc_num; 2451 /* share same FDQ for all flows */ 2452 rx_flow_cfg.rxfdq_cfg.size = max_desc_num * rx_cfg.flow_id_num; 2453 rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode; 2454 2455 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn, 2456 i, &rx_flow_cfg); 2457 if (ret) { 2458 dev_err(dev, "Failed to init rx flow%d %d\n", i, ret); 2459 goto err_flow; 2460 } 2461 if (!i) 2462 fdqring_id = 2463 k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn, 2464 i); 2465 2466 flow->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i); 2467 if (flow->irq <= 0) { 2468 dev_err(dev, "Failed to get rx dma irq %d\n", 2469 flow->irq); 2470 ret = flow->irq; 2471 goto err_flow; 2472 } 2473 2474 snprintf(flow->name, 2475 sizeof(flow->name), "%s-rx%d", 2476 dev_name(dev), i); 2477 hrtimer_setup(&flow->rx_hrtimer, &am65_cpsw_nuss_rx_timer_callback, CLOCK_MONOTONIC, 2478 HRTIMER_MODE_REL_PINNED); 2479 2480 netif_napi_add(common->dma_ndev, &flow->napi_rx, 2481 am65_cpsw_nuss_rx_poll); 2482 2483 ret = devm_request_irq(dev, flow->irq, 2484 am65_cpsw_nuss_rx_irq, 2485 IRQF_TRIGGER_HIGH, 2486 flow->name, flow); 2487 if (ret) { 2488 dev_err(dev, "failure requesting rx %d irq %u, %d\n", 2489 i, flow->irq, ret); 2490 flow->irq = -EINVAL; 2491 goto err_request_irq; 2492 } 2493 } 2494 2495 /* setup classifier to route priorities to flows */ 2496 cpsw_ale_classifier_setup_default(common->ale, common->rx_ch_num_flows); 2497 2498 return 0; 2499 2500 err_request_irq: 2501 netif_napi_del(&flow->napi_rx); 2502 2503 err_flow: 2504 for (--i; i >= 0; i--) { 2505 flow = &rx_chn->flows[i]; 2506 devm_free_irq(dev, flow->irq, flow); 2507 netif_napi_del(&flow->napi_rx); 2508 } 2509 2510 err: 2511 am65_cpsw_nuss_free_rx_chns(common); 2512 2513 return ret; 2514 } 2515 2516 static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common) 2517 { 2518 struct am65_cpsw_host *host_p = am65_common_get_host(common); 2519 2520 host_p->common = common; 2521 host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE; 2522 host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE; 2523 2524 return 0; 2525 } 2526 2527 static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node, 2528 int slave, u8 *mac_addr) 2529 { 2530 u32 mac_lo, mac_hi, offset; 2531 struct regmap *syscon; 2532 2533 syscon = syscon_regmap_lookup_by_phandle_args(of_node, "ti,syscon-efuse", 2534 1, &offset); 2535 if (IS_ERR(syscon)) { 2536 if (PTR_ERR(syscon) == -ENODEV) 2537 return 0; 2538 return PTR_ERR(syscon); 2539 } 2540 2541 regmap_read(syscon, offset, &mac_lo); 2542 regmap_read(syscon, offset + 4, &mac_hi); 2543 2544 mac_addr[0] = (mac_hi >> 8) & 0xff; 2545 mac_addr[1] = mac_hi & 0xff; 2546 mac_addr[2] = (mac_lo >> 24) & 0xff; 2547 mac_addr[3] = (mac_lo >> 16) & 0xff; 2548 mac_addr[4] = (mac_lo >> 8) & 0xff; 2549 mac_addr[5] = mac_lo & 0xff; 2550 2551 return 0; 2552 } 2553 2554 static int am65_cpsw_init_cpts(struct am65_cpsw_common *common) 2555 { 2556 struct device *dev = common->dev; 2557 struct device_node *node; 2558 struct am65_cpts *cpts; 2559 void __iomem *reg_base; 2560 2561 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS)) 2562 return 0; 2563 2564 node = of_get_child_by_name(dev->of_node, "cpts"); 2565 if (!node) { 2566 dev_err(dev, "%s cpts not found\n", __func__); 2567 return -ENOENT; 2568 } 2569 2570 reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE; 2571 cpts = am65_cpts_create(dev, reg_base, node); 2572 if (IS_ERR(cpts)) { 2573 int ret = PTR_ERR(cpts); 2574 2575 of_node_put(node); 2576 dev_err(dev, "cpts create err %d\n", ret); 2577 return ret; 2578 } 2579 common->cpts = cpts; 2580 /* Forbid PM runtime if CPTS is running. 2581 * K3 CPSWxG modules may completely lose context during ON->OFF 2582 * transitions depending on integration. 2583 * AM65x/J721E MCU CPSW2G: false 2584 * J721E MAIN_CPSW9G: true 2585 */ 2586 pm_runtime_forbid(dev); 2587 2588 return 0; 2589 } 2590 2591 static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common) 2592 { 2593 struct device_node *node, *port_np; 2594 struct device *dev = common->dev; 2595 int ret; 2596 2597 node = of_get_child_by_name(dev->of_node, "ethernet-ports"); 2598 if (!node) 2599 return -ENOENT; 2600 2601 for_each_child_of_node(node, port_np) { 2602 phy_interface_t phy_if; 2603 struct am65_cpsw_port *port; 2604 u32 port_id; 2605 2606 /* it is not a slave port node, continue */ 2607 if (strcmp(port_np->name, "port")) 2608 continue; 2609 2610 ret = of_property_read_u32(port_np, "reg", &port_id); 2611 if (ret < 0) { 2612 dev_err(dev, "%pOF error reading port_id %d\n", 2613 port_np, ret); 2614 goto of_node_put; 2615 } 2616 2617 if (!port_id || port_id > common->port_num) { 2618 dev_err(dev, "%pOF has invalid port_id %u %s\n", 2619 port_np, port_id, port_np->name); 2620 ret = -EINVAL; 2621 goto of_node_put; 2622 } 2623 2624 port = am65_common_get_port(common, port_id); 2625 port->port_id = port_id; 2626 port->common = common; 2627 port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE + 2628 AM65_CPSW_NU_PORTS_OFFSET * (port_id); 2629 if (common->pdata.extra_modes) 2630 port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id); 2631 port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE + 2632 (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id); 2633 port->name = of_get_property(port_np, "label", NULL); 2634 port->fetch_ram_base = 2635 common->cpsw_base + AM65_CPSW_NU_FRAM_BASE + 2636 (AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1)); 2637 2638 port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base); 2639 if (IS_ERR(port->slave.mac_sl)) { 2640 ret = PTR_ERR(port->slave.mac_sl); 2641 goto of_node_put; 2642 } 2643 2644 port->disabled = !of_device_is_available(port_np); 2645 if (port->disabled) { 2646 common->disabled_ports_mask |= BIT(port->port_id); 2647 continue; 2648 } 2649 2650 port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL); 2651 if (IS_ERR(port->slave.ifphy)) { 2652 ret = PTR_ERR(port->slave.ifphy); 2653 dev_err(dev, "%pOF error retrieving port phy: %d\n", 2654 port_np, ret); 2655 goto of_node_put; 2656 } 2657 2658 /* Initialize the Serdes PHY for the port */ 2659 ret = am65_cpsw_init_serdes_phy(dev, port_np, port); 2660 if (ret) 2661 goto of_node_put; 2662 2663 port->slave.mac_only = 2664 of_property_read_bool(port_np, "ti,mac-only"); 2665 2666 /* get phy/link info */ 2667 port->slave.port_np = of_node_get(port_np); 2668 ret = of_get_phy_mode(port_np, &phy_if); 2669 if (ret) { 2670 dev_err(dev, "%pOF read phy-mode err %d\n", 2671 port_np, ret); 2672 goto of_node_put; 2673 } 2674 2675 /* CPSW controllers supported by this driver have a fixed 2676 * internal TX delay in RGMII mode. Fix up PHY mode to account 2677 * for this and warn about Device Trees that claim to have a TX 2678 * delay on the PCB. 2679 */ 2680 switch (phy_if) { 2681 case PHY_INTERFACE_MODE_RGMII_ID: 2682 phy_if = PHY_INTERFACE_MODE_RGMII_RXID; 2683 break; 2684 case PHY_INTERFACE_MODE_RGMII_TXID: 2685 phy_if = PHY_INTERFACE_MODE_RGMII; 2686 break; 2687 case PHY_INTERFACE_MODE_RGMII: 2688 case PHY_INTERFACE_MODE_RGMII_RXID: 2689 dev_warn(dev, 2690 "RGMII mode without internal TX delay unsupported; please fix your Device Tree\n"); 2691 break; 2692 default: 2693 break; 2694 } 2695 2696 port->slave.phy_if = phy_if; 2697 ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, phy_if); 2698 if (ret) 2699 goto of_node_put; 2700 2701 ret = of_get_mac_address(port_np, port->slave.mac_addr); 2702 if (ret == -EPROBE_DEFER) { 2703 goto of_node_put; 2704 } else if (ret) { 2705 am65_cpsw_am654_get_efuse_macid(port_np, 2706 port->port_id, 2707 port->slave.mac_addr); 2708 if (!is_valid_ether_addr(port->slave.mac_addr)) { 2709 eth_random_addr(port->slave.mac_addr); 2710 dev_info(dev, "Use random MAC address\n"); 2711 } 2712 } 2713 2714 /* Reset all Queue priorities to 0 */ 2715 writel(0, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP); 2716 } 2717 of_node_put(node); 2718 2719 /* is there at least one ext.port */ 2720 if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) { 2721 dev_err(dev, "No Ext. port are available\n"); 2722 return -ENODEV; 2723 } 2724 2725 return 0; 2726 2727 of_node_put: 2728 of_node_put(port_np); 2729 of_node_put(node); 2730 return ret; 2731 } 2732 2733 static void am65_cpsw_nuss_phylink_cleanup(struct am65_cpsw_common *common) 2734 { 2735 struct am65_cpsw_port *port; 2736 int i; 2737 2738 for (i = 0; i < common->port_num; i++) { 2739 port = &common->ports[i]; 2740 if (port->slave.phylink) 2741 phylink_destroy(port->slave.phylink); 2742 } 2743 } 2744 2745 static void am65_cpsw_remove_dt(struct am65_cpsw_common *common) 2746 { 2747 struct am65_cpsw_port *port; 2748 int i; 2749 2750 for (i = 0; i < common->port_num; i++) { 2751 port = &common->ports[i]; 2752 of_node_put(port->slave.port_np); 2753 } 2754 } 2755 2756 static int 2757 am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx) 2758 { 2759 struct am65_cpsw_ndev_priv *ndev_priv; 2760 struct device *dev = common->dev; 2761 struct am65_cpsw_port *port; 2762 struct phylink *phylink; 2763 2764 port = &common->ports[port_idx]; 2765 2766 if (port->disabled) 2767 return 0; 2768 2769 /* alloc netdev */ 2770 port->ndev = alloc_etherdev_mqs(sizeof(struct am65_cpsw_ndev_priv), 2771 AM65_CPSW_MAX_QUEUES, 2772 AM65_CPSW_MAX_QUEUES); 2773 if (!port->ndev) { 2774 dev_err(dev, "error allocating slave net_device %u\n", 2775 port->port_id); 2776 return -ENOMEM; 2777 } 2778 2779 ndev_priv = netdev_priv(port->ndev); 2780 ndev_priv->port = port; 2781 ndev_priv->msg_enable = AM65_CPSW_DEBUG; 2782 mutex_init(&ndev_priv->mm_lock); 2783 port->qos.link_speed = SPEED_UNKNOWN; 2784 SET_NETDEV_DEV(port->ndev, dev); 2785 device_set_node(&port->ndev->dev, of_fwnode_handle(port->slave.port_np)); 2786 2787 eth_hw_addr_set(port->ndev, port->slave.mac_addr); 2788 2789 port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE; 2790 port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE - 2791 (VLAN_ETH_HLEN + ETH_FCS_LEN); 2792 port->ndev->hw_features = NETIF_F_SG | 2793 NETIF_F_RXCSUM | 2794 NETIF_F_HW_CSUM | 2795 NETIF_F_HW_TC; 2796 port->ndev->features = port->ndev->hw_features | 2797 NETIF_F_HW_VLAN_CTAG_FILTER; 2798 port->ndev->xdp_features = NETDEV_XDP_ACT_BASIC | 2799 NETDEV_XDP_ACT_REDIRECT | 2800 NETDEV_XDP_ACT_NDO_XMIT; 2801 port->ndev->vlan_features |= NETIF_F_SG; 2802 port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops; 2803 port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave; 2804 2805 /* Configuring Phylink */ 2806 port->slave.phylink_config.dev = &port->ndev->dev; 2807 port->slave.phylink_config.type = PHYLINK_NETDEV; 2808 port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 | 2809 MAC_1000FD | MAC_5000FD; 2810 port->slave.phylink_config.mac_managed_pm = true; /* MAC does PM */ 2811 2812 switch (port->slave.phy_if) { 2813 case PHY_INTERFACE_MODE_RGMII: 2814 case PHY_INTERFACE_MODE_RGMII_ID: 2815 case PHY_INTERFACE_MODE_RGMII_RXID: 2816 case PHY_INTERFACE_MODE_RGMII_TXID: 2817 phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces); 2818 break; 2819 2820 case PHY_INTERFACE_MODE_RMII: 2821 __set_bit(PHY_INTERFACE_MODE_RMII, 2822 port->slave.phylink_config.supported_interfaces); 2823 break; 2824 2825 case PHY_INTERFACE_MODE_QSGMII: 2826 case PHY_INTERFACE_MODE_SGMII: 2827 case PHY_INTERFACE_MODE_USXGMII: 2828 if (common->pdata.extra_modes & BIT(port->slave.phy_if)) { 2829 __set_bit(port->slave.phy_if, 2830 port->slave.phylink_config.supported_interfaces); 2831 } else { 2832 dev_err(dev, "selected phy-mode is not supported\n"); 2833 return -EOPNOTSUPP; 2834 } 2835 break; 2836 2837 default: 2838 dev_err(dev, "selected phy-mode is not supported\n"); 2839 return -EOPNOTSUPP; 2840 } 2841 2842 phylink = phylink_create(&port->slave.phylink_config, 2843 of_fwnode_handle(port->slave.port_np), 2844 port->slave.phy_if, 2845 &am65_cpsw_phylink_mac_ops); 2846 if (IS_ERR(phylink)) 2847 return PTR_ERR(phylink); 2848 2849 port->slave.phylink = phylink; 2850 2851 /* Disable TX checksum offload by default due to HW bug */ 2852 if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM) 2853 port->ndev->features &= ~NETIF_F_HW_CSUM; 2854 2855 port->ndev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 2856 port->xdp_prog = NULL; 2857 2858 if (!common->dma_ndev) 2859 common->dma_ndev = port->ndev; 2860 2861 return 0; 2862 } 2863 2864 static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common) 2865 { 2866 int ret; 2867 int i; 2868 2869 for (i = 0; i < common->port_num; i++) { 2870 ret = am65_cpsw_nuss_init_port_ndev(common, i); 2871 if (ret) 2872 return ret; 2873 } 2874 2875 return ret; 2876 } 2877 2878 static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common) 2879 { 2880 struct am65_cpsw_port *port; 2881 int i; 2882 2883 for (i = 0; i < common->port_num; i++) { 2884 port = &common->ports[i]; 2885 if (!port->ndev) 2886 continue; 2887 if (port->ndev->reg_state == NETREG_REGISTERED) 2888 unregister_netdev(port->ndev); 2889 free_netdev(port->ndev); 2890 port->ndev = NULL; 2891 } 2892 } 2893 2894 static void am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common *common) 2895 { 2896 int set_val = 0; 2897 int i; 2898 2899 if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask)) 2900 set_val = 1; 2901 2902 dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val); 2903 2904 for (i = 1; i <= common->port_num; i++) { 2905 struct am65_cpsw_port *port = am65_common_get_port(common, i); 2906 struct am65_cpsw_ndev_priv *priv; 2907 2908 if (!port->ndev) 2909 continue; 2910 2911 priv = am65_ndev_to_priv(port->ndev); 2912 priv->offload_fwd_mark = set_val; 2913 } 2914 } 2915 2916 bool am65_cpsw_port_dev_check(const struct net_device *ndev) 2917 { 2918 if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) { 2919 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 2920 2921 return !common->is_emac_mode; 2922 } 2923 2924 return false; 2925 } 2926 2927 static int am65_cpsw_netdevice_port_link(struct net_device *ndev, 2928 struct net_device *br_ndev, 2929 struct netlink_ext_ack *extack) 2930 { 2931 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 2932 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev); 2933 int err; 2934 2935 if (!common->br_members) { 2936 common->hw_bridge_dev = br_ndev; 2937 } else { 2938 /* This is adding the port to a second bridge, this is 2939 * unsupported 2940 */ 2941 if (common->hw_bridge_dev != br_ndev) 2942 return -EOPNOTSUPP; 2943 } 2944 2945 err = switchdev_bridge_port_offload(ndev, ndev, NULL, NULL, NULL, 2946 false, extack); 2947 if (err) 2948 return err; 2949 2950 common->br_members |= BIT(priv->port->port_id); 2951 2952 am65_cpsw_port_offload_fwd_mark_update(common); 2953 2954 return NOTIFY_DONE; 2955 } 2956 2957 static void am65_cpsw_netdevice_port_unlink(struct net_device *ndev) 2958 { 2959 struct am65_cpsw_common *common = am65_ndev_to_common(ndev); 2960 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev); 2961 2962 switchdev_bridge_port_unoffload(ndev, NULL, NULL, NULL); 2963 2964 common->br_members &= ~BIT(priv->port->port_id); 2965 2966 am65_cpsw_port_offload_fwd_mark_update(common); 2967 2968 if (!common->br_members) 2969 common->hw_bridge_dev = NULL; 2970 } 2971 2972 /* netdev notifier */ 2973 static int am65_cpsw_netdevice_event(struct notifier_block *unused, 2974 unsigned long event, void *ptr) 2975 { 2976 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 2977 struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 2978 struct netdev_notifier_changeupper_info *info; 2979 int ret = NOTIFY_DONE; 2980 2981 if (!am65_cpsw_port_dev_check(ndev)) 2982 return NOTIFY_DONE; 2983 2984 switch (event) { 2985 case NETDEV_CHANGEUPPER: 2986 info = ptr; 2987 2988 if (netif_is_bridge_master(info->upper_dev)) { 2989 if (info->linking) 2990 ret = am65_cpsw_netdevice_port_link(ndev, 2991 info->upper_dev, 2992 extack); 2993 else 2994 am65_cpsw_netdevice_port_unlink(ndev); 2995 } 2996 break; 2997 default: 2998 return NOTIFY_DONE; 2999 } 3000 3001 return notifier_from_errno(ret); 3002 } 3003 3004 static int am65_cpsw_register_notifiers(struct am65_cpsw_common *cpsw) 3005 { 3006 int ret = 0; 3007 3008 if (AM65_CPSW_IS_CPSW2G(cpsw) || 3009 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) 3010 return 0; 3011 3012 cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event; 3013 ret = register_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb); 3014 if (ret) { 3015 dev_err(cpsw->dev, "can't register netdevice notifier\n"); 3016 return ret; 3017 } 3018 3019 ret = am65_cpsw_switchdev_register_notifiers(cpsw); 3020 if (ret) 3021 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb); 3022 3023 return ret; 3024 } 3025 3026 static void am65_cpsw_unregister_notifiers(struct am65_cpsw_common *cpsw) 3027 { 3028 if (AM65_CPSW_IS_CPSW2G(cpsw) || 3029 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) 3030 return; 3031 3032 am65_cpsw_switchdev_unregister_notifiers(cpsw); 3033 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb); 3034 } 3035 3036 static const struct devlink_ops am65_cpsw_devlink_ops = {}; 3037 3038 static void am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common *cpsw) 3039 { 3040 cpsw_ale_add_mcast(cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, 0, 3041 ALE_MCAST_BLOCK_LEARN_FWD); 3042 } 3043 3044 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common) 3045 { 3046 struct am65_cpsw_host *host = am65_common_get_host(common); 3047 3048 writel(common->default_vlan, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3049 3050 am65_cpsw_init_stp_ale_entry(common); 3051 3052 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 1); 3053 dev_dbg(common->dev, "Set P0_UNI_FLOOD\n"); 3054 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 0); 3055 } 3056 3057 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common) 3058 { 3059 struct am65_cpsw_host *host = am65_common_get_host(common); 3060 3061 writel(0, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3062 3063 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 0); 3064 dev_dbg(common->dev, "unset P0_UNI_FLOOD\n"); 3065 3066 /* learning make no sense in multi-mac mode */ 3067 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 1); 3068 } 3069 3070 static int am65_cpsw_dl_switch_mode_get(struct devlink *dl, u32 id, 3071 struct devlink_param_gset_ctx *ctx, 3072 struct netlink_ext_ack *extack) 3073 { 3074 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl); 3075 struct am65_cpsw_common *common = dl_priv->common; 3076 3077 dev_dbg(common->dev, "%s id:%u\n", __func__, id); 3078 3079 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE) 3080 return -EOPNOTSUPP; 3081 3082 ctx->val.vbool = !common->is_emac_mode; 3083 3084 return 0; 3085 } 3086 3087 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port) 3088 { 3089 struct am65_cpsw_slave_data *slave = &port->slave; 3090 struct am65_cpsw_common *common = port->common; 3091 u32 port_mask; 3092 3093 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3094 3095 if (slave->mac_only) 3096 /* enable mac-only mode on port */ 3097 cpsw_ale_control_set(common->ale, port->port_id, 3098 ALE_PORT_MACONLY, 1); 3099 3100 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_NOLEARN, 1); 3101 3102 port_mask = BIT(port->port_id) | ALE_PORT_HOST; 3103 3104 cpsw_ale_add_ucast(common->ale, port->ndev->dev_addr, 3105 HOST_PORT_NUM, ALE_SECURE, slave->port_vlan); 3106 cpsw_ale_add_mcast(common->ale, port->ndev->broadcast, 3107 port_mask, ALE_VLAN, slave->port_vlan, ALE_MCAST_FWD_2); 3108 } 3109 3110 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port) 3111 { 3112 struct am65_cpsw_slave_data *slave = &port->slave; 3113 struct am65_cpsw_common *cpsw = port->common; 3114 u32 port_mask; 3115 3116 cpsw_ale_control_set(cpsw->ale, port->port_id, 3117 ALE_PORT_NOLEARN, 0); 3118 3119 cpsw_ale_add_ucast(cpsw->ale, port->ndev->dev_addr, 3120 HOST_PORT_NUM, ALE_SECURE | ALE_BLOCKED | ALE_VLAN, 3121 slave->port_vlan); 3122 3123 port_mask = BIT(port->port_id) | ALE_PORT_HOST; 3124 3125 cpsw_ale_add_mcast(cpsw->ale, port->ndev->broadcast, 3126 port_mask, ALE_VLAN, slave->port_vlan, 3127 ALE_MCAST_FWD_2); 3128 3129 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3130 3131 cpsw_ale_control_set(cpsw->ale, port->port_id, 3132 ALE_PORT_MACONLY, 0); 3133 } 3134 3135 static int am65_cpsw_dl_switch_mode_set(struct devlink *dl, u32 id, 3136 struct devlink_param_gset_ctx *ctx, 3137 struct netlink_ext_ack *extack) 3138 { 3139 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl); 3140 struct am65_cpsw_common *cpsw = dl_priv->common; 3141 bool switch_en = ctx->val.vbool; 3142 bool if_running = false; 3143 int i; 3144 3145 dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id); 3146 3147 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE) 3148 return -EOPNOTSUPP; 3149 3150 if (switch_en == !cpsw->is_emac_mode) 3151 return 0; 3152 3153 if (!switch_en && cpsw->br_members) { 3154 dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n"); 3155 return -EINVAL; 3156 } 3157 3158 rtnl_lock(); 3159 3160 cpsw->is_emac_mode = !switch_en; 3161 3162 for (i = 0; i < cpsw->port_num; i++) { 3163 struct net_device *sl_ndev = cpsw->ports[i].ndev; 3164 3165 if (!sl_ndev || !netif_running(sl_ndev)) 3166 continue; 3167 3168 if_running = true; 3169 } 3170 3171 if (!if_running) { 3172 /* all ndevs are down */ 3173 for (i = 0; i < cpsw->port_num; i++) { 3174 struct net_device *sl_ndev = cpsw->ports[i].ndev; 3175 struct am65_cpsw_slave_data *slave; 3176 3177 if (!sl_ndev) 3178 continue; 3179 3180 slave = am65_ndev_to_slave(sl_ndev); 3181 if (switch_en) 3182 slave->port_vlan = cpsw->default_vlan; 3183 else 3184 slave->port_vlan = 0; 3185 } 3186 3187 goto exit; 3188 } 3189 3190 cpsw_ale_control_set(cpsw->ale, 0, ALE_BYPASS, 1); 3191 /* clean up ALE table */ 3192 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_CLEAR, 1); 3193 cpsw_ale_control_get(cpsw->ale, HOST_PORT_NUM, ALE_AGEOUT); 3194 3195 if (switch_en) { 3196 dev_info(cpsw->dev, "Enable switch mode\n"); 3197 3198 am65_cpsw_init_host_port_switch(cpsw); 3199 3200 for (i = 0; i < cpsw->port_num; i++) { 3201 struct net_device *sl_ndev = cpsw->ports[i].ndev; 3202 struct am65_cpsw_slave_data *slave; 3203 struct am65_cpsw_port *port; 3204 3205 if (!sl_ndev) 3206 continue; 3207 3208 port = am65_ndev_to_port(sl_ndev); 3209 slave = am65_ndev_to_slave(sl_ndev); 3210 slave->port_vlan = cpsw->default_vlan; 3211 3212 if (netif_running(sl_ndev)) 3213 am65_cpsw_init_port_switch_ale(port); 3214 } 3215 3216 } else { 3217 dev_info(cpsw->dev, "Disable switch mode\n"); 3218 3219 am65_cpsw_init_host_port_emac(cpsw); 3220 3221 for (i = 0; i < cpsw->port_num; i++) { 3222 struct net_device *sl_ndev = cpsw->ports[i].ndev; 3223 struct am65_cpsw_port *port; 3224 3225 if (!sl_ndev) 3226 continue; 3227 3228 port = am65_ndev_to_port(sl_ndev); 3229 port->slave.port_vlan = 0; 3230 if (netif_running(sl_ndev)) 3231 am65_cpsw_init_port_emac_ale(port); 3232 } 3233 } 3234 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_BYPASS, 0); 3235 exit: 3236 rtnl_unlock(); 3237 3238 return 0; 3239 } 3240 3241 static const struct devlink_param am65_cpsw_devlink_params[] = { 3242 DEVLINK_PARAM_DRIVER(AM65_CPSW_DL_PARAM_SWITCH_MODE, "switch_mode", 3243 DEVLINK_PARAM_TYPE_BOOL, 3244 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 3245 am65_cpsw_dl_switch_mode_get, 3246 am65_cpsw_dl_switch_mode_set, NULL), 3247 }; 3248 3249 static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common) 3250 { 3251 struct devlink_port_attrs attrs = {}; 3252 struct am65_cpsw_devlink *dl_priv; 3253 struct device *dev = common->dev; 3254 struct devlink_port *dl_port; 3255 struct am65_cpsw_port *port; 3256 int ret = 0; 3257 int i; 3258 3259 common->devlink = 3260 devlink_alloc(&am65_cpsw_devlink_ops, sizeof(*dl_priv), dev); 3261 if (!common->devlink) 3262 return -ENOMEM; 3263 3264 dl_priv = devlink_priv(common->devlink); 3265 dl_priv->common = common; 3266 3267 /* Provide devlink hook to switch mode when multiple external ports 3268 * are present NUSS switchdev driver is enabled. 3269 */ 3270 if (!AM65_CPSW_IS_CPSW2G(common) && 3271 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) { 3272 ret = devlink_params_register(common->devlink, 3273 am65_cpsw_devlink_params, 3274 ARRAY_SIZE(am65_cpsw_devlink_params)); 3275 if (ret) { 3276 dev_err(dev, "devlink params reg fail ret:%d\n", ret); 3277 goto dl_unreg; 3278 } 3279 } 3280 3281 for (i = 1; i <= common->port_num; i++) { 3282 port = am65_common_get_port(common, i); 3283 dl_port = &port->devlink_port; 3284 3285 if (port->ndev) 3286 attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; 3287 else 3288 attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED; 3289 attrs.phys.port_number = port->port_id; 3290 attrs.switch_id.id_len = sizeof(resource_size_t); 3291 memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len); 3292 devlink_port_attrs_set(dl_port, &attrs); 3293 3294 ret = devlink_port_register(common->devlink, dl_port, port->port_id); 3295 if (ret) { 3296 dev_err(dev, "devlink_port reg fail for port %d, ret:%d\n", 3297 port->port_id, ret); 3298 goto dl_port_unreg; 3299 } 3300 } 3301 devlink_register(common->devlink); 3302 return ret; 3303 3304 dl_port_unreg: 3305 for (i = i - 1; i >= 1; i--) { 3306 port = am65_common_get_port(common, i); 3307 dl_port = &port->devlink_port; 3308 3309 devlink_port_unregister(dl_port); 3310 } 3311 dl_unreg: 3312 devlink_free(common->devlink); 3313 return ret; 3314 } 3315 3316 static void am65_cpsw_unregister_devlink(struct am65_cpsw_common *common) 3317 { 3318 struct devlink_port *dl_port; 3319 struct am65_cpsw_port *port; 3320 int i; 3321 3322 devlink_unregister(common->devlink); 3323 3324 for (i = 1; i <= common->port_num; i++) { 3325 port = am65_common_get_port(common, i); 3326 dl_port = &port->devlink_port; 3327 3328 devlink_port_unregister(dl_port); 3329 } 3330 3331 if (!AM65_CPSW_IS_CPSW2G(common) && 3332 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) 3333 devlink_params_unregister(common->devlink, 3334 am65_cpsw_devlink_params, 3335 ARRAY_SIZE(am65_cpsw_devlink_params)); 3336 3337 devlink_free(common->devlink); 3338 } 3339 3340 static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common) 3341 { 3342 struct am65_cpsw_rx_chn *rx_chan = &common->rx_chns; 3343 struct am65_cpsw_tx_chn *tx_chan = common->tx_chns; 3344 struct device *dev = common->dev; 3345 struct am65_cpsw_port *port; 3346 int ret = 0, i; 3347 3348 /* init tx channels */ 3349 ret = am65_cpsw_nuss_init_tx_chns(common); 3350 if (ret) 3351 return ret; 3352 ret = am65_cpsw_nuss_init_rx_chns(common); 3353 if (ret) 3354 goto err_remove_tx; 3355 3356 /* The DMA Channels are not guaranteed to be in a clean state. 3357 * Reset and disable them to ensure that they are back to the 3358 * clean state and ready to be used. 3359 */ 3360 for (i = 0; i < common->tx_ch_num; i++) { 3361 k3_udma_glue_reset_tx_chn(tx_chan[i].tx_chn, &tx_chan[i], 3362 am65_cpsw_nuss_tx_cleanup); 3363 k3_udma_glue_disable_tx_chn(tx_chan[i].tx_chn); 3364 } 3365 3366 for (i = 0; i < common->rx_ch_num_flows; i++) 3367 k3_udma_glue_reset_rx_chn(rx_chan->rx_chn, i, 3368 rx_chan, 3369 am65_cpsw_nuss_rx_cleanup); 3370 3371 k3_udma_glue_disable_rx_chn(rx_chan->rx_chn); 3372 3373 ret = am65_cpsw_nuss_register_devlink(common); 3374 if (ret) 3375 goto err_remove_rx; 3376 3377 for (i = 0; i < common->port_num; i++) { 3378 port = &common->ports[i]; 3379 3380 if (!port->ndev) 3381 continue; 3382 3383 SET_NETDEV_DEVLINK_PORT(port->ndev, &port->devlink_port); 3384 3385 ret = register_netdev(port->ndev); 3386 if (ret) { 3387 dev_err(dev, "error registering slave net device%i %d\n", 3388 i, ret); 3389 goto err_cleanup_ndev; 3390 } 3391 } 3392 3393 ret = am65_cpsw_register_notifiers(common); 3394 if (ret) 3395 goto err_cleanup_ndev; 3396 3397 /* can't auto unregister ndev using devm_add_action() due to 3398 * devres release sequence in DD core for DMA 3399 */ 3400 3401 return 0; 3402 3403 err_cleanup_ndev: 3404 am65_cpsw_nuss_cleanup_ndev(common); 3405 am65_cpsw_unregister_devlink(common); 3406 err_remove_rx: 3407 am65_cpsw_nuss_remove_rx_chns(common); 3408 err_remove_tx: 3409 am65_cpsw_nuss_remove_tx_chns(common); 3410 3411 return ret; 3412 } 3413 3414 int am65_cpsw_nuss_update_tx_rx_chns(struct am65_cpsw_common *common, 3415 int num_tx, int num_rx) 3416 { 3417 int ret; 3418 3419 am65_cpsw_nuss_remove_tx_chns(common); 3420 am65_cpsw_nuss_remove_rx_chns(common); 3421 3422 common->tx_ch_num = num_tx; 3423 common->rx_ch_num_flows = num_rx; 3424 ret = am65_cpsw_nuss_init_tx_chns(common); 3425 if (ret) 3426 return ret; 3427 3428 ret = am65_cpsw_nuss_init_rx_chns(common); 3429 if (ret) 3430 am65_cpsw_nuss_remove_tx_chns(common); 3431 3432 return ret; 3433 } 3434 3435 struct am65_cpsw_soc_pdata { 3436 u32 quirks_dis; 3437 }; 3438 3439 static const struct am65_cpsw_soc_pdata am65x_soc_sr2_0 = { 3440 .quirks_dis = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM, 3441 }; 3442 3443 static const struct soc_device_attribute am65_cpsw_socinfo[] = { 3444 { .family = "AM65X", 3445 .revision = "SR2.0", 3446 .data = &am65x_soc_sr2_0 3447 }, 3448 {/* sentinel */} 3449 }; 3450 3451 static const struct am65_cpsw_pdata am65x_sr1_0 = { 3452 .quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM, 3453 .ale_dev_id = "am65x-cpsw2g", 3454 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE, 3455 }; 3456 3457 static const struct am65_cpsw_pdata j721e_pdata = { 3458 .quirks = 0, 3459 .ale_dev_id = "am65x-cpsw2g", 3460 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE, 3461 }; 3462 3463 static const struct am65_cpsw_pdata am64x_cpswxg_pdata = { 3464 .quirks = AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ, 3465 .ale_dev_id = "am64-cpswxg", 3466 .fdqring_mode = K3_RINGACC_RING_MODE_RING, 3467 }; 3468 3469 static const struct am65_cpsw_pdata j722s_cpswxg_pdata = { 3470 .quirks = AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ, 3471 .ale_dev_id = "am64-cpswxg", 3472 .fdqring_mode = K3_RINGACC_RING_MODE_RING, 3473 .extra_modes = BIT(PHY_INTERFACE_MODE_SGMII), 3474 }; 3475 3476 static const struct am65_cpsw_pdata j7200_cpswxg_pdata = { 3477 .quirks = 0, 3478 .ale_dev_id = "am64-cpswxg", 3479 .fdqring_mode = K3_RINGACC_RING_MODE_RING, 3480 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII) | 3481 BIT(PHY_INTERFACE_MODE_USXGMII), 3482 }; 3483 3484 static const struct am65_cpsw_pdata j721e_cpswxg_pdata = { 3485 .quirks = 0, 3486 .ale_dev_id = "am64-cpswxg", 3487 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE, 3488 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII), 3489 }; 3490 3491 static const struct am65_cpsw_pdata j784s4_cpswxg_pdata = { 3492 .quirks = 0, 3493 .ale_dev_id = "am64-cpswxg", 3494 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE, 3495 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII) | 3496 BIT(PHY_INTERFACE_MODE_USXGMII), 3497 }; 3498 3499 static const struct of_device_id am65_cpsw_nuss_of_mtable[] = { 3500 { .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0}, 3501 { .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata}, 3502 { .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata}, 3503 { .compatible = "ti,j722s-cpsw-nuss", .data = &j722s_cpswxg_pdata}, 3504 { .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata}, 3505 { .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata}, 3506 { .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata}, 3507 { /* sentinel */ }, 3508 }; 3509 MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable); 3510 3511 static void am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common *common) 3512 { 3513 const struct soc_device_attribute *soc; 3514 3515 soc = soc_device_match(am65_cpsw_socinfo); 3516 if (soc && soc->data) { 3517 const struct am65_cpsw_soc_pdata *socdata = soc->data; 3518 3519 /* disable quirks */ 3520 common->pdata.quirks &= ~socdata->quirks_dis; 3521 } 3522 } 3523 3524 static int am65_cpsw_nuss_probe(struct platform_device *pdev) 3525 { 3526 struct cpsw_ale_params ale_params = { 0 }; 3527 const struct of_device_id *of_id; 3528 struct device *dev = &pdev->dev; 3529 struct am65_cpsw_common *common; 3530 struct device_node *node; 3531 struct resource *res; 3532 struct clk *clk; 3533 int ale_entries; 3534 __be64 id_temp; 3535 int ret, i; 3536 3537 BUILD_BUG_ON_MSG(sizeof(struct am65_cpsw_tx_swdata) > AM65_CPSW_NAV_SW_DATA_SIZE, 3538 "TX SW_DATA size exceeds AM65_CPSW_NAV_SW_DATA_SIZE"); 3539 BUILD_BUG_ON_MSG(sizeof(struct am65_cpsw_swdata) > AM65_CPSW_NAV_SW_DATA_SIZE, 3540 "SW_DATA size exceeds AM65_CPSW_NAV_SW_DATA_SIZE"); 3541 common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL); 3542 if (!common) 3543 return -ENOMEM; 3544 common->dev = dev; 3545 3546 of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev); 3547 if (!of_id) 3548 return -EINVAL; 3549 common->pdata = *(const struct am65_cpsw_pdata *)of_id->data; 3550 3551 am65_cpsw_nuss_apply_socinfo(common); 3552 3553 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss"); 3554 common->ss_base = devm_ioremap_resource(&pdev->dev, res); 3555 if (IS_ERR(common->ss_base)) 3556 return PTR_ERR(common->ss_base); 3557 common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE; 3558 /* Use device's physical base address as switch id */ 3559 id_temp = cpu_to_be64(res->start); 3560 memcpy(common->switch_id, &id_temp, sizeof(res->start)); 3561 3562 node = of_get_child_by_name(dev->of_node, "ethernet-ports"); 3563 if (!node) 3564 return -ENOENT; 3565 common->port_num = of_get_child_count(node); 3566 of_node_put(node); 3567 if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS) 3568 return -ENOENT; 3569 3570 common->rx_flow_id_base = -1; 3571 init_completion(&common->tdown_complete); 3572 common->tx_ch_num = AM65_CPSW_DEFAULT_TX_CHNS; 3573 common->rx_ch_num_flows = AM65_CPSW_DEFAULT_RX_CHN_FLOWS; 3574 common->pf_p0_rx_ptype_rrobin = true; 3575 common->default_vlan = 1; 3576 3577 common->ports = devm_kcalloc(dev, common->port_num, 3578 sizeof(*common->ports), 3579 GFP_KERNEL); 3580 if (!common->ports) 3581 return -ENOMEM; 3582 3583 clk = devm_clk_get(dev, "fck"); 3584 if (IS_ERR(clk)) 3585 return dev_err_probe(dev, PTR_ERR(clk), "getting fck clock\n"); 3586 common->bus_freq = clk_get_rate(clk); 3587 3588 pm_runtime_enable(dev); 3589 ret = pm_runtime_resume_and_get(dev); 3590 if (ret < 0) { 3591 pm_runtime_disable(dev); 3592 return ret; 3593 } 3594 3595 am65_cpsw_nuss_get_ver(common); 3596 3597 ret = am65_cpsw_nuss_init_host_p(common); 3598 if (ret) 3599 goto err_pm_clear; 3600 3601 ret = am65_cpsw_nuss_init_slave_ports(common); 3602 if (ret) 3603 goto err_pm_clear; 3604 3605 node = of_get_child_by_name(dev->of_node, "mdio"); 3606 if (!node) { 3607 dev_warn(dev, "MDIO node not found\n"); 3608 } else if (of_device_is_available(node)) { 3609 struct platform_device *mdio_pdev; 3610 3611 mdio_pdev = of_platform_device_create(node, NULL, dev); 3612 if (!mdio_pdev) { 3613 ret = -ENODEV; 3614 goto err_pm_clear; 3615 } 3616 3617 common->mdio_dev = &mdio_pdev->dev; 3618 } 3619 of_node_put(node); 3620 3621 /* init common data */ 3622 ale_params.dev = dev; 3623 ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT; 3624 ale_params.ale_ports = common->port_num + 1; 3625 ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE; 3626 ale_params.dev_id = common->pdata.ale_dev_id; 3627 ale_params.bus_freq = common->bus_freq; 3628 3629 common->ale = cpsw_ale_create(&ale_params); 3630 if (IS_ERR(common->ale)) { 3631 dev_err(dev, "error initializing ale engine\n"); 3632 ret = PTR_ERR(common->ale); 3633 goto err_of_clear; 3634 } 3635 3636 ale_entries = common->ale->params.ale_entries; 3637 common->ale_context = devm_kzalloc(dev, 3638 ale_entries * ALE_ENTRY_WORDS * sizeof(u32), 3639 GFP_KERNEL); 3640 ret = am65_cpsw_init_cpts(common); 3641 if (ret) 3642 goto err_of_clear; 3643 3644 /* init ports */ 3645 for (i = 0; i < common->port_num; i++) 3646 am65_cpsw_nuss_slave_disable_unused(&common->ports[i]); 3647 3648 dev_set_drvdata(dev, common); 3649 3650 common->is_emac_mode = true; 3651 3652 ret = am65_cpsw_nuss_init_ndevs(common); 3653 if (ret) 3654 goto err_ndevs_clear; 3655 3656 ret = am65_cpsw_nuss_register_ndevs(common); 3657 if (ret) 3658 goto err_ndevs_clear; 3659 3660 pm_runtime_put(dev); 3661 return 0; 3662 3663 err_ndevs_clear: 3664 am65_cpsw_nuss_cleanup_ndev(common); 3665 am65_cpsw_nuss_phylink_cleanup(common); 3666 am65_cpts_release(common->cpts); 3667 am65_cpsw_remove_dt(common); 3668 err_of_clear: 3669 if (common->mdio_dev) 3670 of_platform_device_destroy(common->mdio_dev, NULL); 3671 err_pm_clear: 3672 pm_runtime_put_sync(dev); 3673 pm_runtime_disable(dev); 3674 return ret; 3675 } 3676 3677 static void am65_cpsw_nuss_remove(struct platform_device *pdev) 3678 { 3679 struct device *dev = &pdev->dev; 3680 struct am65_cpsw_common *common; 3681 int ret; 3682 3683 common = dev_get_drvdata(dev); 3684 3685 ret = pm_runtime_resume_and_get(&pdev->dev); 3686 if (ret < 0) { 3687 /* Note, if this error path is taken, we're leaking some 3688 * resources. 3689 */ 3690 dev_err(&pdev->dev, "Failed to resume device (%pe)\n", 3691 ERR_PTR(ret)); 3692 return; 3693 } 3694 3695 am65_cpsw_unregister_notifiers(common); 3696 3697 /* must unregister ndevs here because DD release_driver routine calls 3698 * dma_deconfigure(dev) before devres_release_all(dev) 3699 */ 3700 am65_cpsw_nuss_cleanup_ndev(common); 3701 am65_cpsw_unregister_devlink(common); 3702 am65_cpsw_nuss_remove_rx_chns(common); 3703 am65_cpsw_nuss_remove_tx_chns(common); 3704 am65_cpsw_nuss_phylink_cleanup(common); 3705 am65_cpts_release(common->cpts); 3706 am65_cpsw_disable_serdes_phy(common); 3707 am65_cpsw_remove_dt(common); 3708 3709 if (common->mdio_dev) 3710 of_platform_device_destroy(common->mdio_dev, NULL); 3711 3712 pm_runtime_put_sync(&pdev->dev); 3713 pm_runtime_disable(&pdev->dev); 3714 } 3715 3716 static int am65_cpsw_nuss_suspend(struct device *dev) 3717 { 3718 struct am65_cpsw_common *common = dev_get_drvdata(dev); 3719 struct am65_cpsw_host *host_p = am65_common_get_host(common); 3720 struct am65_cpsw_port *port; 3721 struct net_device *ndev; 3722 int i, ret; 3723 3724 cpsw_ale_dump(common->ale, common->ale_context); 3725 host_p->vid_context = readl(host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3726 for (i = 0; i < common->port_num; i++) { 3727 port = &common->ports[i]; 3728 ndev = port->ndev; 3729 3730 if (!ndev) 3731 continue; 3732 3733 port->vid_context = readl(port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3734 netif_device_detach(ndev); 3735 if (netif_running(ndev)) { 3736 rtnl_lock(); 3737 ret = am65_cpsw_nuss_ndo_slave_stop(ndev); 3738 rtnl_unlock(); 3739 if (ret < 0) { 3740 netdev_err(ndev, "failed to stop: %d", ret); 3741 return ret; 3742 } 3743 } 3744 } 3745 3746 am65_cpts_suspend(common->cpts); 3747 3748 am65_cpsw_nuss_remove_rx_chns(common); 3749 am65_cpsw_nuss_remove_tx_chns(common); 3750 3751 return 0; 3752 } 3753 3754 static int am65_cpsw_nuss_resume(struct device *dev) 3755 { 3756 struct am65_cpsw_common *common = dev_get_drvdata(dev); 3757 struct am65_cpsw_host *host_p = am65_common_get_host(common); 3758 struct am65_cpsw_port *port; 3759 struct net_device *ndev; 3760 int i, ret; 3761 3762 ret = am65_cpsw_nuss_init_tx_chns(common); 3763 if (ret) 3764 return ret; 3765 ret = am65_cpsw_nuss_init_rx_chns(common); 3766 if (ret) { 3767 am65_cpsw_nuss_remove_tx_chns(common); 3768 return ret; 3769 } 3770 3771 /* If RX IRQ was disabled before suspend, keep it disabled */ 3772 for (i = 0; i < common->rx_ch_num_flows; i++) { 3773 if (common->rx_chns.flows[i].irq_disabled) 3774 disable_irq(common->rx_chns.flows[i].irq); 3775 } 3776 3777 am65_cpts_resume(common->cpts); 3778 3779 for (i = 0; i < common->port_num; i++) { 3780 port = &common->ports[i]; 3781 ndev = port->ndev; 3782 3783 if (!ndev) 3784 continue; 3785 3786 if (netif_running(ndev)) { 3787 rtnl_lock(); 3788 ret = am65_cpsw_nuss_ndo_slave_open(ndev); 3789 rtnl_unlock(); 3790 if (ret < 0) { 3791 netdev_err(ndev, "failed to start: %d", ret); 3792 return ret; 3793 } 3794 } 3795 3796 netif_device_attach(ndev); 3797 writel(port->vid_context, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3798 } 3799 3800 writel(host_p->vid_context, host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET); 3801 cpsw_ale_restore(common->ale, common->ale_context); 3802 3803 return 0; 3804 } 3805 3806 static const struct dev_pm_ops am65_cpsw_nuss_dev_pm_ops = { 3807 SYSTEM_SLEEP_PM_OPS(am65_cpsw_nuss_suspend, am65_cpsw_nuss_resume) 3808 }; 3809 3810 static struct platform_driver am65_cpsw_nuss_driver = { 3811 .driver = { 3812 .name = AM65_CPSW_DRV_NAME, 3813 .of_match_table = am65_cpsw_nuss_of_mtable, 3814 .pm = &am65_cpsw_nuss_dev_pm_ops, 3815 }, 3816 .probe = am65_cpsw_nuss_probe, 3817 .remove = am65_cpsw_nuss_remove, 3818 }; 3819 3820 module_platform_driver(am65_cpsw_nuss_driver); 3821 3822 MODULE_LICENSE("GPL v2"); 3823 MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>"); 3824 MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver"); 3825