1 // SPDX-License-Identifier: GPL-2.0
2 /* Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver
3 *
4 * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
5 *
6 */
7
8 #include <linux/bpf_trace.h>
9 #include <linux/clk.h>
10 #include <linux/etherdevice.h>
11 #include <linux/if_vlan.h>
12 #include <linux/interrupt.h>
13 #include <linux/irqdomain.h>
14 #include <linux/kernel.h>
15 #include <linux/kmemleak.h>
16 #include <linux/module.h>
17 #include <linux/netdevice.h>
18 #include <linux/net_tstamp.h>
19 #include <linux/of.h>
20 #include <linux/of_mdio.h>
21 #include <linux/of_net.h>
22 #include <linux/of_device.h>
23 #include <linux/of_platform.h>
24 #include <linux/phylink.h>
25 #include <linux/phy/phy.h>
26 #include <linux/platform_device.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/regmap.h>
29 #include <linux/rtnetlink.h>
30 #include <linux/mfd/syscon.h>
31 #include <linux/sys_soc.h>
32 #include <linux/dma/ti-cppi5.h>
33 #include <linux/dma/k3-udma-glue.h>
34 #include <net/page_pool/helpers.h>
35 #include <net/dsa.h>
36 #include <net/switchdev.h>
37
38 #include "cpsw_ale.h"
39 #include "cpsw_sl.h"
40 #include "am65-cpsw-nuss.h"
41 #include "am65-cpsw-switchdev.h"
42 #include "k3-cppi-desc-pool.h"
43 #include "am65-cpts.h"
44
45 #define AM65_CPSW_SS_BASE 0x0
46 #define AM65_CPSW_SGMII_BASE 0x100
47 #define AM65_CPSW_XGMII_BASE 0x2100
48 #define AM65_CPSW_CPSW_NU_BASE 0x20000
49 #define AM65_CPSW_NU_PORTS_BASE 0x1000
50 #define AM65_CPSW_NU_FRAM_BASE 0x12000
51 #define AM65_CPSW_NU_STATS_BASE 0x1a000
52 #define AM65_CPSW_NU_ALE_BASE 0x1e000
53 #define AM65_CPSW_NU_CPTS_BASE 0x1d000
54
55 #define AM65_CPSW_NU_PORTS_OFFSET 0x1000
56 #define AM65_CPSW_NU_STATS_PORT_OFFSET 0x200
57 #define AM65_CPSW_NU_FRAM_PORT_OFFSET 0x200
58
59 #define AM65_CPSW_MAX_PORTS 8
60
61 #define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN
62 #define AM65_CPSW_MAX_PACKET_SIZE 2024
63
64 #define AM65_CPSW_REG_CTL 0x004
65 #define AM65_CPSW_REG_STAT_PORT_EN 0x014
66 #define AM65_CPSW_REG_PTYPE 0x018
67
68 #define AM65_CPSW_P0_REG_CTL 0x004
69 #define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET 0x008
70
71 #define AM65_CPSW_PORT_REG_PRI_CTL 0x01c
72 #define AM65_CPSW_PORT_REG_RX_PRI_MAP 0x020
73 #define AM65_CPSW_PORT_REG_RX_MAXLEN 0x024
74
75 #define AM65_CPSW_PORTN_REG_CTL 0x004
76 #define AM65_CPSW_PORTN_REG_DSCP_MAP 0x120
77 #define AM65_CPSW_PORTN_REG_SA_L 0x308
78 #define AM65_CPSW_PORTN_REG_SA_H 0x30c
79 #define AM65_CPSW_PORTN_REG_TS_CTL 0x310
80 #define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG 0x314
81 #define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG 0x318
82 #define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 0x31C
83
84 #define AM65_CPSW_SGMII_CONTROL_REG 0x010
85 #define AM65_CPSW_SGMII_MR_ADV_ABILITY_REG 0x018
86 #define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE BIT(0)
87
88 #define AM65_CPSW_CTL_VLAN_AWARE BIT(1)
89 #define AM65_CPSW_CTL_P0_ENABLE BIT(2)
90 #define AM65_CPSW_CTL_P0_TX_CRC_REMOVE BIT(13)
91 #define AM65_CPSW_CTL_P0_RX_PAD BIT(14)
92
93 /* AM65_CPSW_P0_REG_CTL */
94 #define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN BIT(0)
95 #define AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN BIT(16)
96
97 /* AM65_CPSW_PORT_REG_PRI_CTL */
98 #define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN BIT(8)
99
100 /* AM65_CPSW_PN_REG_CTL */
101 #define AM65_CPSW_PN_REG_CTL_DSCP_IPV4_EN BIT(1)
102 #define AM65_CPSW_PN_REG_CTL_DSCP_IPV6_EN BIT(2)
103
104 /* AM65_CPSW_PN_TS_CTL register fields */
105 #define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN BIT(4)
106 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN BIT(5)
107 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN BIT(6)
108 #define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN BIT(7)
109 #define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN BIT(10)
110 #define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN BIT(11)
111 #define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT 16
112
113 #define AM65_CPSW_PN_TS_CTL_RX_ANX_F_EN BIT(0)
114 #define AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN BIT(1)
115 #define AM65_CPSW_PN_TS_CTL_RX_VLAN_LT2_EN BIT(2)
116 #define AM65_CPSW_PN_TS_CTL_RX_ANX_D_EN BIT(3)
117 #define AM65_CPSW_PN_TS_CTL_RX_ANX_E_EN BIT(9)
118
119 /* AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG register fields */
120 #define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT 16
121
122 /* AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 */
123 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 BIT(16)
124 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 BIT(17)
125 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 BIT(18)
126 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 BIT(19)
127 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 BIT(20)
128 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 BIT(21)
129 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 BIT(22)
130 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23)
131
132 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
133 #define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
134
135 #define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e)
136
137 #define AM65_CPSW_TS_TX_ANX_ALL_EN \
138 (AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN | \
139 AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN | \
140 AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN)
141
142 #define AM65_CPSW_TS_RX_ANX_ALL_EN \
143 (AM65_CPSW_PN_TS_CTL_RX_ANX_D_EN | \
144 AM65_CPSW_PN_TS_CTL_RX_ANX_E_EN | \
145 AM65_CPSW_PN_TS_CTL_RX_ANX_F_EN)
146
147 #define AM65_CPSW_ALE_AGEOUT_DEFAULT 30
148 /* Number of TX/RX descriptors per channel/flow */
149 #define AM65_CPSW_MAX_TX_DESC 500
150 #define AM65_CPSW_MAX_RX_DESC 500
151
152 #define AM65_CPSW_NAV_PS_DATA_SIZE 16
153 #define AM65_CPSW_NAV_SW_DATA_SIZE 16
154
155 #define AM65_CPSW_DEBUG (NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | \
156 NETIF_MSG_IFUP | NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \
157 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
158
159 #define AM65_CPSW_DEFAULT_TX_CHNS 8
160 #define AM65_CPSW_DEFAULT_RX_CHN_FLOWS 1
161
162 /* CPPI streaming packet interface */
163 #define AM65_CPSW_CPPI_TX_FLOW_ID 0x3FFF
164 #define AM65_CPSW_CPPI_TX_PKT_TYPE 0x7
165
166 /* XDP */
167 #define AM65_CPSW_XDP_CONSUMED BIT(1)
168 #define AM65_CPSW_XDP_REDIRECT BIT(0)
169 #define AM65_CPSW_XDP_PASS 0
170
171 /* Include headroom compatible with both skb and xdpf */
172 #define AM65_CPSW_HEADROOM_NA (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
173 #define AM65_CPSW_HEADROOM ALIGN(AM65_CPSW_HEADROOM_NA, sizeof(long))
174
am65_cpsw_port_set_sl_mac(struct am65_cpsw_port * slave,const u8 * dev_addr)175 static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
176 const u8 *dev_addr)
177 {
178 u32 mac_hi = (dev_addr[0] << 0) | (dev_addr[1] << 8) |
179 (dev_addr[2] << 16) | (dev_addr[3] << 24);
180 u32 mac_lo = (dev_addr[4] << 0) | (dev_addr[5] << 8);
181
182 writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H);
183 writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L);
184 }
185
186 #define AM65_CPSW_DSCP_MAX GENMASK(5, 0)
187 #define AM65_CPSW_PRI_MAX GENMASK(2, 0)
188 #define AM65_CPSW_DSCP_PRI_PER_REG 8
189 #define AM65_CPSW_DSCP_PRI_SIZE 4 /* in bits */
am65_cpsw_port_set_dscp_map(struct am65_cpsw_port * slave,u8 dscp,u8 pri)190 static int am65_cpsw_port_set_dscp_map(struct am65_cpsw_port *slave, u8 dscp, u8 pri)
191 {
192 int reg_ofs;
193 int bit_ofs;
194 u32 val;
195
196 if (dscp > AM65_CPSW_DSCP_MAX)
197 return -EINVAL;
198
199 if (pri > AM65_CPSW_PRI_MAX)
200 return -EINVAL;
201
202 /* 32-bit register offset to this dscp */
203 reg_ofs = (dscp / AM65_CPSW_DSCP_PRI_PER_REG) * 4;
204 /* bit field offset to this dscp */
205 bit_ofs = AM65_CPSW_DSCP_PRI_SIZE * (dscp % AM65_CPSW_DSCP_PRI_PER_REG);
206
207 val = readl(slave->port_base + AM65_CPSW_PORTN_REG_DSCP_MAP + reg_ofs);
208 val &= ~(AM65_CPSW_PRI_MAX << bit_ofs); /* clear */
209 val |= pri << bit_ofs; /* set */
210 writel(val, slave->port_base + AM65_CPSW_PORTN_REG_DSCP_MAP + reg_ofs);
211
212 return 0;
213 }
214
am65_cpsw_port_enable_dscp_map(struct am65_cpsw_port * slave)215 static void am65_cpsw_port_enable_dscp_map(struct am65_cpsw_port *slave)
216 {
217 int dscp, pri;
218 u32 val;
219
220 /* Default DSCP to User Priority mapping as per:
221 * https://datatracker.ietf.org/doc/html/rfc8325#section-4.3
222 * and
223 * https://datatracker.ietf.org/doc/html/rfc8622#section-11
224 */
225 for (dscp = 0; dscp <= AM65_CPSW_DSCP_MAX; dscp++) {
226 switch (dscp) {
227 case 56: /* CS7 */
228 case 48: /* CS6 */
229 pri = 7;
230 break;
231 case 46: /* EF */
232 case 44: /* VA */
233 pri = 6;
234 break;
235 case 40: /* CS5 */
236 pri = 5;
237 break;
238 case 34: /* AF41 */
239 case 36: /* AF42 */
240 case 38: /* AF43 */
241 case 32: /* CS4 */
242 case 26: /* AF31 */
243 case 28: /* AF32 */
244 case 30: /* AF33 */
245 case 24: /* CS3 */
246 pri = 4;
247 break;
248 case 18: /* AF21 */
249 case 20: /* AF22 */
250 case 22: /* AF23 */
251 pri = 3;
252 break;
253 case 16: /* CS2 */
254 case 10: /* AF11 */
255 case 12: /* AF12 */
256 case 14: /* AF13 */
257 case 0: /* DF */
258 pri = 0;
259 break;
260 case 8: /* CS1 */
261 case 1: /* LE */
262 pri = 1;
263 break;
264 default:
265 pri = 0;
266 break;
267 }
268
269 am65_cpsw_port_set_dscp_map(slave, dscp, pri);
270 }
271
272 /* enable port IPV4 and IPV6 DSCP for this port */
273 val = readl(slave->port_base + AM65_CPSW_PORTN_REG_CTL);
274 val |= AM65_CPSW_PN_REG_CTL_DSCP_IPV4_EN |
275 AM65_CPSW_PN_REG_CTL_DSCP_IPV6_EN;
276 writel(val, slave->port_base + AM65_CPSW_PORTN_REG_CTL);
277 }
278
am65_cpsw_sl_ctl_reset(struct am65_cpsw_port * port)279 static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port)
280 {
281 cpsw_sl_reset(port->slave.mac_sl, 100);
282 /* Max length register has to be restored after MAC SL reset */
283 writel(AM65_CPSW_MAX_PACKET_SIZE,
284 port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
285 }
286
am65_cpsw_nuss_get_ver(struct am65_cpsw_common * common)287 static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common)
288 {
289 common->nuss_ver = readl(common->ss_base);
290 common->cpsw_ver = readl(common->cpsw_base);
291 dev_info(common->dev,
292 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n",
293 common->nuss_ver,
294 common->cpsw_ver,
295 common->port_num + 1,
296 common->pdata.quirks);
297 }
298
am65_cpsw_nuss_ndo_slave_add_vid(struct net_device * ndev,__be16 proto,u16 vid)299 static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
300 __be16 proto, u16 vid)
301 {
302 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
303 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
304 u32 port_mask, unreg_mcast = 0;
305 int ret;
306
307 if (!common->is_emac_mode)
308 return 0;
309
310 if (!netif_running(ndev) || !vid)
311 return 0;
312
313 ret = pm_runtime_resume_and_get(common->dev);
314 if (ret < 0)
315 return ret;
316
317 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
318 if (!vid)
319 unreg_mcast = port_mask;
320 dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
321 ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask,
322 unreg_mcast, port_mask, 0);
323
324 pm_runtime_put(common->dev);
325 return ret;
326 }
327
am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device * ndev,__be16 proto,u16 vid)328 static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev,
329 __be16 proto, u16 vid)
330 {
331 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
332 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
333 int ret;
334
335 if (!common->is_emac_mode)
336 return 0;
337
338 if (!netif_running(ndev) || !vid)
339 return 0;
340
341 ret = pm_runtime_resume_and_get(common->dev);
342 if (ret < 0)
343 return ret;
344
345 dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);
346 ret = cpsw_ale_del_vlan(common->ale, vid,
347 BIT(port->port_id) | ALE_PORT_HOST);
348
349 pm_runtime_put(common->dev);
350 return ret;
351 }
352
am65_cpsw_slave_set_promisc(struct am65_cpsw_port * port,bool promisc)353 static void am65_cpsw_slave_set_promisc(struct am65_cpsw_port *port,
354 bool promisc)
355 {
356 struct am65_cpsw_common *common = port->common;
357
358 if (promisc && !common->is_emac_mode) {
359 dev_dbg(common->dev, "promisc mode requested in switch mode");
360 return;
361 }
362
363 if (promisc) {
364 /* Enable promiscuous mode */
365 cpsw_ale_control_set(common->ale, port->port_id,
366 ALE_PORT_MACONLY_CAF, 1);
367 dev_dbg(common->dev, "promisc enabled\n");
368 } else {
369 /* Disable promiscuous mode */
370 cpsw_ale_control_set(common->ale, port->port_id,
371 ALE_PORT_MACONLY_CAF, 0);
372 dev_dbg(common->dev, "promisc disabled\n");
373 }
374 }
375
am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device * ndev)376 static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev)
377 {
378 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
379 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
380 u32 port_mask;
381 bool promisc;
382
383 promisc = !!(ndev->flags & IFF_PROMISC);
384 am65_cpsw_slave_set_promisc(port, promisc);
385
386 if (promisc)
387 return;
388
389 /* Restore allmulti on vlans if necessary */
390 cpsw_ale_set_allmulti(common->ale,
391 ndev->flags & IFF_ALLMULTI, port->port_id);
392
393 port_mask = ALE_PORT_HOST;
394 /* Clear all mcast from ALE */
395 cpsw_ale_flush_multicast(common->ale, port_mask, -1);
396
397 if (!netdev_mc_empty(ndev)) {
398 struct netdev_hw_addr *ha;
399
400 /* program multicast address list into ALE register */
401 netdev_for_each_mc_addr(ha, ndev) {
402 cpsw_ale_add_mcast(common->ale, ha->addr,
403 port_mask, 0, 0, 0);
404 }
405 }
406 }
407
am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device * ndev,unsigned int txqueue)408 static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev,
409 unsigned int txqueue)
410 {
411 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
412 struct am65_cpsw_tx_chn *tx_chn;
413 struct netdev_queue *netif_txq;
414 unsigned long trans_start;
415
416 netif_txq = netdev_get_tx_queue(ndev, txqueue);
417 tx_chn = &common->tx_chns[txqueue];
418 trans_start = READ_ONCE(netif_txq->trans_start);
419
420 netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
421 txqueue,
422 netif_tx_queue_stopped(netif_txq),
423 jiffies_to_msecs(jiffies - trans_start),
424 netdev_queue_dql_avail(netif_txq),
425 k3_cppi_desc_pool_avail(tx_chn->desc_pool));
426
427 if (netif_tx_queue_stopped(netif_txq)) {
428 /* try recover if stopped by us */
429 txq_trans_update(netif_txq);
430 netif_tx_wake_queue(netif_txq);
431 }
432 }
433
am65_cpsw_nuss_rx_push(struct am65_cpsw_common * common,struct page * page,u32 flow_idx)434 static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
435 struct page *page, u32 flow_idx)
436 {
437 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
438 struct cppi5_host_desc_t *desc_rx;
439 struct device *dev = common->dev;
440 struct am65_cpsw_swdata *swdata;
441 dma_addr_t desc_dma;
442 dma_addr_t buf_dma;
443
444 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
445 if (!desc_rx) {
446 dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
447 return -ENOMEM;
448 }
449 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
450
451 buf_dma = dma_map_single(rx_chn->dma_dev,
452 page_address(page) + AM65_CPSW_HEADROOM,
453 AM65_CPSW_MAX_PACKET_SIZE, DMA_FROM_DEVICE);
454 if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
455 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
456 dev_err(dev, "Failed to map rx buffer\n");
457 return -EINVAL;
458 }
459
460 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
461 AM65_CPSW_NAV_PS_DATA_SIZE);
462 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
463 cppi5_hdesc_attach_buf(desc_rx, buf_dma, AM65_CPSW_MAX_PACKET_SIZE,
464 buf_dma, AM65_CPSW_MAX_PACKET_SIZE);
465 swdata = cppi5_hdesc_get_swdata(desc_rx);
466 swdata->page = page;
467 swdata->flow_id = flow_idx;
468
469 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, flow_idx,
470 desc_rx, desc_dma);
471 }
472
am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common * common)473 void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common)
474 {
475 struct am65_cpsw_host *host_p = am65_common_get_host(common);
476 u32 val, pri_map;
477
478 /* P0 set Receive Priority Type */
479 val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
480
481 if (common->pf_p0_rx_ptype_rrobin) {
482 val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
483 /* Enet Ports fifos works in fixed priority mode only, so
484 * reset P0_Rx_Pri_Map so all packet will go in Enet fifo 0
485 */
486 pri_map = 0x0;
487 } else {
488 val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
489 /* restore P0_Rx_Pri_Map */
490 pri_map = 0x76543210;
491 }
492
493 writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);
494 writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
495 }
496
497 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common);
498 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
499 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
500 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
501 static inline void am65_cpsw_put_page(struct am65_cpsw_rx_flow *flow,
502 struct page *page,
503 bool allow_direct);
504 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
505 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
506
am65_cpsw_destroy_rxq(struct am65_cpsw_common * common,int id)507 static void am65_cpsw_destroy_rxq(struct am65_cpsw_common *common, int id)
508 {
509 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
510 struct am65_cpsw_rx_flow *flow;
511 struct xdp_rxq_info *rxq;
512 int port;
513
514 flow = &rx_chn->flows[id];
515 napi_disable(&flow->napi_rx);
516 hrtimer_cancel(&flow->rx_hrtimer);
517 k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, id, rx_chn,
518 am65_cpsw_nuss_rx_cleanup, !!id);
519
520 for (port = 0; port < common->port_num; port++) {
521 if (!common->ports[port].ndev)
522 continue;
523
524 rxq = &common->ports[port].xdp_rxq[id];
525
526 if (xdp_rxq_info_is_reg(rxq))
527 xdp_rxq_info_unreg(rxq);
528 }
529
530 if (flow->page_pool) {
531 page_pool_destroy(flow->page_pool);
532 flow->page_pool = NULL;
533 }
534 }
535
am65_cpsw_destroy_rxqs(struct am65_cpsw_common * common)536 static void am65_cpsw_destroy_rxqs(struct am65_cpsw_common *common)
537 {
538 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
539 int id;
540
541 reinit_completion(&common->tdown_complete);
542 k3_udma_glue_tdown_rx_chn(rx_chn->rx_chn, true);
543
544 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) {
545 id = wait_for_completion_timeout(&common->tdown_complete, msecs_to_jiffies(1000));
546 if (!id)
547 dev_err(common->dev, "rx teardown timeout\n");
548 }
549
550 for (id = common->rx_ch_num_flows - 1; id >= 0; id--)
551 am65_cpsw_destroy_rxq(common, id);
552
553 k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
554 }
555
am65_cpsw_create_rxq(struct am65_cpsw_common * common,int id)556 static int am65_cpsw_create_rxq(struct am65_cpsw_common *common, int id)
557 {
558 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
559 struct page_pool_params pp_params = {
560 .flags = PP_FLAG_DMA_MAP,
561 .order = 0,
562 .pool_size = AM65_CPSW_MAX_RX_DESC,
563 .nid = dev_to_node(common->dev),
564 .dev = common->dev,
565 .dma_dir = DMA_BIDIRECTIONAL,
566 /* .napi set dynamically */
567 };
568 struct am65_cpsw_rx_flow *flow;
569 struct xdp_rxq_info *rxq;
570 struct page_pool *pool;
571 struct page *page;
572 int port, ret, i;
573
574 flow = &rx_chn->flows[id];
575 pp_params.napi = &flow->napi_rx;
576 pool = page_pool_create(&pp_params);
577 if (IS_ERR(pool)) {
578 ret = PTR_ERR(pool);
579 return ret;
580 }
581
582 flow->page_pool = pool;
583
584 /* using same page pool is allowed as no running rx handlers
585 * simultaneously for both ndevs
586 */
587 for (port = 0; port < common->port_num; port++) {
588 if (!common->ports[port].ndev)
589 /* FIXME should we BUG here? */
590 continue;
591
592 rxq = &common->ports[port].xdp_rxq[id];
593 ret = xdp_rxq_info_reg(rxq, common->ports[port].ndev,
594 id, flow->napi_rx.napi_id);
595 if (ret)
596 goto err;
597
598 ret = xdp_rxq_info_reg_mem_model(rxq,
599 MEM_TYPE_PAGE_POOL,
600 pool);
601 if (ret)
602 goto err;
603 }
604
605 for (i = 0; i < AM65_CPSW_MAX_RX_DESC; i++) {
606 page = page_pool_dev_alloc_pages(flow->page_pool);
607 if (!page) {
608 dev_err(common->dev, "cannot allocate page in flow %d\n",
609 id);
610 ret = -ENOMEM;
611 goto err;
612 }
613
614 ret = am65_cpsw_nuss_rx_push(common, page, id);
615 if (ret < 0) {
616 dev_err(common->dev,
617 "cannot submit page to rx channel flow %d, error %d\n",
618 id, ret);
619 am65_cpsw_put_page(flow, page, false);
620 goto err;
621 }
622 }
623
624 napi_enable(&flow->napi_rx);
625 return 0;
626
627 err:
628 am65_cpsw_destroy_rxq(common, id);
629 return ret;
630 }
631
am65_cpsw_create_rxqs(struct am65_cpsw_common * common)632 static int am65_cpsw_create_rxqs(struct am65_cpsw_common *common)
633 {
634 int id, ret;
635
636 for (id = 0; id < common->rx_ch_num_flows; id++) {
637 ret = am65_cpsw_create_rxq(common, id);
638 if (ret) {
639 dev_err(common->dev, "couldn't create rxq %d: %d\n",
640 id, ret);
641 goto err;
642 }
643 }
644
645 ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);
646 if (ret) {
647 dev_err(common->dev, "couldn't enable rx chn: %d\n", ret);
648 goto err;
649 }
650
651 return 0;
652
653 err:
654 for (--id; id >= 0; id--)
655 am65_cpsw_destroy_rxq(common, id);
656
657 return ret;
658 }
659
am65_cpsw_destroy_txq(struct am65_cpsw_common * common,int id)660 static void am65_cpsw_destroy_txq(struct am65_cpsw_common *common, int id)
661 {
662 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];
663
664 napi_disable(&tx_chn->napi_tx);
665 hrtimer_cancel(&tx_chn->tx_hrtimer);
666 k3_udma_glue_reset_tx_chn(tx_chn->tx_chn, tx_chn,
667 am65_cpsw_nuss_tx_cleanup);
668 k3_udma_glue_disable_tx_chn(tx_chn->tx_chn);
669 }
670
am65_cpsw_destroy_txqs(struct am65_cpsw_common * common)671 static void am65_cpsw_destroy_txqs(struct am65_cpsw_common *common)
672 {
673 struct am65_cpsw_tx_chn *tx_chn = common->tx_chns;
674 int id;
675
676 /* shutdown tx channels */
677 atomic_set(&common->tdown_cnt, common->tx_ch_num);
678 /* ensure new tdown_cnt value is visible */
679 smp_mb__after_atomic();
680 reinit_completion(&common->tdown_complete);
681
682 for (id = 0; id < common->tx_ch_num; id++)
683 k3_udma_glue_tdown_tx_chn(tx_chn[id].tx_chn, false);
684
685 id = wait_for_completion_timeout(&common->tdown_complete,
686 msecs_to_jiffies(1000));
687 if (!id)
688 dev_err(common->dev, "tx teardown timeout\n");
689
690 for (id = common->tx_ch_num - 1; id >= 0; id--)
691 am65_cpsw_destroy_txq(common, id);
692 }
693
am65_cpsw_create_txq(struct am65_cpsw_common * common,int id)694 static int am65_cpsw_create_txq(struct am65_cpsw_common *common, int id)
695 {
696 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];
697 int ret;
698
699 ret = k3_udma_glue_enable_tx_chn(tx_chn->tx_chn);
700 if (ret)
701 return ret;
702
703 napi_enable(&tx_chn->napi_tx);
704
705 return 0;
706 }
707
am65_cpsw_create_txqs(struct am65_cpsw_common * common)708 static int am65_cpsw_create_txqs(struct am65_cpsw_common *common)
709 {
710 int id, ret;
711
712 for (id = 0; id < common->tx_ch_num; id++) {
713 ret = am65_cpsw_create_txq(common, id);
714 if (ret) {
715 dev_err(common->dev, "couldn't create txq %d: %d\n",
716 id, ret);
717 goto err;
718 }
719 }
720
721 return 0;
722
723 err:
724 for (--id; id >= 0; id--)
725 am65_cpsw_destroy_txq(common, id);
726
727 return ret;
728 }
729
am65_cpsw_nuss_desc_idx(struct k3_cppi_desc_pool * desc_pool,void * desc,unsigned char dsize_log2)730 static int am65_cpsw_nuss_desc_idx(struct k3_cppi_desc_pool *desc_pool,
731 void *desc,
732 unsigned char dsize_log2)
733 {
734 void *pool_addr = k3_cppi_desc_pool_cpuaddr(desc_pool);
735
736 return (desc - pool_addr) >> dsize_log2;
737 }
738
am65_cpsw_nuss_set_buf_type(struct am65_cpsw_tx_chn * tx_chn,struct cppi5_host_desc_t * desc,enum am65_cpsw_tx_buf_type buf_type)739 static void am65_cpsw_nuss_set_buf_type(struct am65_cpsw_tx_chn *tx_chn,
740 struct cppi5_host_desc_t *desc,
741 enum am65_cpsw_tx_buf_type buf_type)
742 {
743 int desc_idx;
744
745 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc,
746 tx_chn->dsize_log2);
747 k3_cppi_desc_pool_desc_info_set(tx_chn->desc_pool, desc_idx,
748 (void *)buf_type);
749 }
750
am65_cpsw_nuss_buf_type(struct am65_cpsw_tx_chn * tx_chn,dma_addr_t desc_dma)751 static enum am65_cpsw_tx_buf_type am65_cpsw_nuss_buf_type(struct am65_cpsw_tx_chn *tx_chn,
752 dma_addr_t desc_dma)
753 {
754 struct cppi5_host_desc_t *desc_tx;
755 int desc_idx;
756
757 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
758 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc_tx,
759 tx_chn->dsize_log2);
760
761 return (enum am65_cpsw_tx_buf_type)k3_cppi_desc_pool_desc_info(tx_chn->desc_pool,
762 desc_idx);
763 }
764
am65_cpsw_put_page(struct am65_cpsw_rx_flow * flow,struct page * page,bool allow_direct)765 static inline void am65_cpsw_put_page(struct am65_cpsw_rx_flow *flow,
766 struct page *page,
767 bool allow_direct)
768 {
769 page_pool_put_full_page(flow->page_pool, page, allow_direct);
770 }
771
am65_cpsw_nuss_rx_cleanup(void * data,dma_addr_t desc_dma)772 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
773 {
774 struct am65_cpsw_rx_chn *rx_chn = data;
775 struct cppi5_host_desc_t *desc_rx;
776 struct am65_cpsw_swdata *swdata;
777 dma_addr_t buf_dma;
778 struct page *page;
779 u32 buf_dma_len;
780 u32 flow_id;
781
782 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
783 swdata = cppi5_hdesc_get_swdata(desc_rx);
784 page = swdata->page;
785 flow_id = swdata->flow_id;
786 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
787 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
788 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
789 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
790 am65_cpsw_put_page(&rx_chn->flows[flow_id], page, false);
791 }
792
am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn * tx_chn,struct cppi5_host_desc_t * desc)793 static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
794 struct cppi5_host_desc_t *desc)
795 {
796 struct cppi5_host_desc_t *first_desc, *next_desc;
797 dma_addr_t buf_dma, next_desc_dma;
798 u32 buf_dma_len;
799
800 first_desc = desc;
801 next_desc = first_desc;
802
803 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
804 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
805
806 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
807
808 next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
809 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
810 while (next_desc_dma) {
811 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
812 next_desc_dma);
813 cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
814 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
815
816 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
817 DMA_TO_DEVICE);
818
819 next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
820 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
821
822 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
823 }
824
825 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
826 }
827
am65_cpsw_nuss_tx_cleanup(void * data,dma_addr_t desc_dma)828 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
829 {
830 struct am65_cpsw_tx_chn *tx_chn = data;
831 enum am65_cpsw_tx_buf_type buf_type;
832 struct cppi5_host_desc_t *desc_tx;
833 struct xdp_frame *xdpf;
834 struct sk_buff *skb;
835 void **swdata;
836
837 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
838 swdata = cppi5_hdesc_get_swdata(desc_tx);
839 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
840 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
841 skb = *(swdata);
842 dev_kfree_skb_any(skb);
843 } else {
844 xdpf = *(swdata);
845 xdp_return_frame(xdpf);
846 }
847
848 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
849 }
850
am65_cpsw_build_skb(void * page_addr,struct net_device * ndev,unsigned int len,unsigned int headroom)851 static struct sk_buff *am65_cpsw_build_skb(void *page_addr,
852 struct net_device *ndev,
853 unsigned int len,
854 unsigned int headroom)
855 {
856 struct sk_buff *skb;
857
858 len += AM65_CPSW_HEADROOM;
859
860 skb = build_skb(page_addr, len);
861 if (unlikely(!skb))
862 return NULL;
863
864 skb_reserve(skb, headroom);
865 skb->dev = ndev;
866
867 return skb;
868 }
869
am65_cpsw_nuss_common_open(struct am65_cpsw_common * common)870 static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
871 {
872 struct am65_cpsw_host *host_p = am65_common_get_host(common);
873 u32 val, port_mask;
874 int port_idx, ret;
875
876 if (common->usage_count)
877 return 0;
878
879 /* Control register */
880 writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE |
881 AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD,
882 common->cpsw_base + AM65_CPSW_REG_CTL);
883 /* Max length register */
884 writel(AM65_CPSW_MAX_PACKET_SIZE,
885 host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
886 /* set base flow_id */
887 writel(common->rx_flow_id_base,
888 host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
889 writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN | AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN,
890 host_p->port_base + AM65_CPSW_P0_REG_CTL);
891
892 am65_cpsw_nuss_set_p0_ptype(common);
893
894 /* enable statistic */
895 val = BIT(HOST_PORT_NUM);
896 for (port_idx = 0; port_idx < common->port_num; port_idx++) {
897 struct am65_cpsw_port *port = &common->ports[port_idx];
898
899 if (!port->disabled)
900 val |= BIT(port->port_id);
901 }
902 writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
903
904 /* disable priority elevation */
905 writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE);
906
907 cpsw_ale_start(common->ale);
908
909 /* limit to one RX flow only */
910 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
911 ALE_DEFAULT_THREAD_ID, 0);
912 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
913 ALE_DEFAULT_THREAD_ENABLE, 1);
914 /* switch to vlan aware mode */
915 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1);
916 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
917 ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
918
919 /* default vlan cfg: create mask based on enabled ports */
920 port_mask = GENMASK(common->port_num, 0) &
921 ~common->disabled_ports_mask;
922
923 cpsw_ale_add_vlan(common->ale, 0, port_mask,
924 port_mask, port_mask,
925 port_mask & ~ALE_PORT_HOST);
926
927 if (common->is_emac_mode)
928 am65_cpsw_init_host_port_emac(common);
929 else
930 am65_cpsw_init_host_port_switch(common);
931
932 am65_cpsw_qos_tx_p0_rate_init(common);
933
934 ret = am65_cpsw_create_rxqs(common);
935 if (ret)
936 return ret;
937
938 ret = am65_cpsw_create_txqs(common);
939 if (ret)
940 goto cleanup_rx;
941
942 dev_dbg(common->dev, "cpsw_nuss started\n");
943 return 0;
944
945 cleanup_rx:
946 am65_cpsw_destroy_rxqs(common);
947
948 return ret;
949 }
950
am65_cpsw_nuss_common_stop(struct am65_cpsw_common * common)951 static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
952 {
953 if (common->usage_count != 1)
954 return 0;
955
956 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
957 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
958
959 am65_cpsw_destroy_txqs(common);
960 am65_cpsw_destroy_rxqs(common);
961 cpsw_ale_stop(common->ale);
962
963 writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);
964 writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
965
966 dev_dbg(common->dev, "cpsw_nuss stopped\n");
967 return 0;
968 }
969
am65_cpsw_nuss_ndo_slave_stop(struct net_device * ndev)970 static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev)
971 {
972 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
973 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
974 int ret;
975
976 phylink_stop(port->slave.phylink);
977
978 netif_tx_stop_all_queues(ndev);
979
980 phylink_disconnect_phy(port->slave.phylink);
981
982 ret = am65_cpsw_nuss_common_stop(common);
983 if (ret)
984 return ret;
985
986 common->usage_count--;
987 pm_runtime_put(common->dev);
988 return 0;
989 }
990
cpsw_restore_vlans(struct net_device * vdev,int vid,void * arg)991 static int cpsw_restore_vlans(struct net_device *vdev, int vid, void *arg)
992 {
993 struct am65_cpsw_port *port = arg;
994
995 if (!vdev)
996 return 0;
997
998 return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid);
999 }
1000
am65_cpsw_nuss_ndo_slave_open(struct net_device * ndev)1001 static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
1002 {
1003 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1004 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1005 int ret, i;
1006 u32 reg;
1007
1008 ret = pm_runtime_resume_and_get(common->dev);
1009 if (ret < 0)
1010 return ret;
1011
1012 /* Idle MAC port */
1013 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
1014 cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
1015 cpsw_sl_ctl_reset(port->slave.mac_sl);
1016
1017 /* soft reset MAC */
1018 cpsw_sl_reg_write(port->slave.mac_sl, CPSW_SL_SOFT_RESET, 1);
1019 mdelay(1);
1020 reg = cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_SOFT_RESET);
1021 if (reg) {
1022 dev_err(common->dev, "soft RESET didn't complete\n");
1023 ret = -ETIMEDOUT;
1024 goto runtime_put;
1025 }
1026
1027 /* Notify the stack of the actual queue counts. */
1028 ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num);
1029 if (ret) {
1030 dev_err(common->dev, "cannot set real number of tx queues\n");
1031 goto runtime_put;
1032 }
1033
1034 ret = netif_set_real_num_rx_queues(ndev, common->rx_ch_num_flows);
1035 if (ret) {
1036 dev_err(common->dev, "cannot set real number of rx queues\n");
1037 goto runtime_put;
1038 }
1039
1040 for (i = 0; i < common->tx_ch_num; i++) {
1041 struct netdev_queue *txq = netdev_get_tx_queue(ndev, i);
1042
1043 netdev_tx_reset_queue(txq);
1044 txq->tx_maxrate = common->tx_chns[i].rate_mbps;
1045 }
1046
1047 ret = am65_cpsw_nuss_common_open(common);
1048 if (ret)
1049 goto runtime_put;
1050
1051 common->usage_count++;
1052
1053 /* VLAN aware CPSW mode is incompatible with some DSA tagging schemes.
1054 * Therefore disable VLAN_AWARE mode if any of the ports is a DSA Port.
1055 */
1056 if (netdev_uses_dsa(ndev)) {
1057 reg = readl(common->cpsw_base + AM65_CPSW_REG_CTL);
1058 reg &= ~AM65_CPSW_CTL_VLAN_AWARE;
1059 writel(reg, common->cpsw_base + AM65_CPSW_REG_CTL);
1060 }
1061
1062 am65_cpsw_port_set_sl_mac(port, ndev->dev_addr);
1063 am65_cpsw_port_enable_dscp_map(port);
1064
1065 if (common->is_emac_mode)
1066 am65_cpsw_init_port_emac_ale(port);
1067 else
1068 am65_cpsw_init_port_switch_ale(port);
1069
1070 /* mac_sl should be configured via phy-link interface */
1071 am65_cpsw_sl_ctl_reset(port);
1072
1073 ret = phylink_of_phy_connect(port->slave.phylink, port->slave.port_np, 0);
1074 if (ret)
1075 goto error_cleanup;
1076
1077 /* restore vlan configurations */
1078 vlan_for_each(ndev, cpsw_restore_vlans, port);
1079
1080 phylink_start(port->slave.phylink);
1081
1082 return 0;
1083
1084 error_cleanup:
1085 am65_cpsw_nuss_ndo_slave_stop(ndev);
1086 return ret;
1087
1088 runtime_put:
1089 pm_runtime_put(common->dev);
1090 return ret;
1091 }
1092
am65_cpsw_xdp_tx_frame(struct net_device * ndev,struct am65_cpsw_tx_chn * tx_chn,struct xdp_frame * xdpf,enum am65_cpsw_tx_buf_type buf_type)1093 static int am65_cpsw_xdp_tx_frame(struct net_device *ndev,
1094 struct am65_cpsw_tx_chn *tx_chn,
1095 struct xdp_frame *xdpf,
1096 enum am65_cpsw_tx_buf_type buf_type)
1097 {
1098 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1099 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1100 struct cppi5_host_desc_t *host_desc;
1101 struct netdev_queue *netif_txq;
1102 dma_addr_t dma_desc, dma_buf;
1103 u32 pkt_len = xdpf->len;
1104 void **swdata;
1105 int ret;
1106
1107 host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1108 if (unlikely(!host_desc)) {
1109 ndev->stats.tx_dropped++;
1110 return AM65_CPSW_XDP_CONSUMED; /* drop */
1111 }
1112
1113 am65_cpsw_nuss_set_buf_type(tx_chn, host_desc, buf_type);
1114
1115 dma_buf = dma_map_single(tx_chn->dma_dev, xdpf->data,
1116 pkt_len, DMA_TO_DEVICE);
1117 if (unlikely(dma_mapping_error(tx_chn->dma_dev, dma_buf))) {
1118 ndev->stats.tx_dropped++;
1119 ret = AM65_CPSW_XDP_CONSUMED; /* drop */
1120 goto pool_free;
1121 }
1122
1123 cppi5_hdesc_init(host_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1124 AM65_CPSW_NAV_PS_DATA_SIZE);
1125 cppi5_hdesc_set_pkttype(host_desc, AM65_CPSW_CPPI_TX_PKT_TYPE);
1126 cppi5_hdesc_set_pktlen(host_desc, pkt_len);
1127 cppi5_desc_set_pktids(&host_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID);
1128 cppi5_desc_set_tags_ids(&host_desc->hdr, 0, port->port_id);
1129
1130 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf);
1131 cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf, pkt_len);
1132
1133 swdata = cppi5_hdesc_get_swdata(host_desc);
1134 *(swdata) = xdpf;
1135
1136 /* Report BQL before sending the packet */
1137 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
1138 netdev_tx_sent_queue(netif_txq, pkt_len);
1139
1140 dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, host_desc);
1141 if (AM65_CPSW_IS_CPSW2G(common)) {
1142 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,
1143 dma_desc);
1144 } else {
1145 spin_lock_bh(&tx_chn->lock);
1146 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,
1147 dma_desc);
1148 spin_unlock_bh(&tx_chn->lock);
1149 }
1150 if (ret) {
1151 /* Inform BQL */
1152 netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1153 ndev->stats.tx_errors++;
1154 ret = AM65_CPSW_XDP_CONSUMED; /* drop */
1155 goto dma_unmap;
1156 }
1157
1158 return 0;
1159
1160 dma_unmap:
1161 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &dma_buf);
1162 dma_unmap_single(tx_chn->dma_dev, dma_buf, pkt_len, DMA_TO_DEVICE);
1163 pool_free:
1164 k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc);
1165 return ret;
1166 }
1167
am65_cpsw_run_xdp(struct am65_cpsw_rx_flow * flow,struct am65_cpsw_port * port,struct xdp_buff * xdp,int cpu,int * len)1168 static int am65_cpsw_run_xdp(struct am65_cpsw_rx_flow *flow,
1169 struct am65_cpsw_port *port,
1170 struct xdp_buff *xdp,
1171 int cpu, int *len)
1172 {
1173 struct am65_cpsw_common *common = flow->common;
1174 struct net_device *ndev = port->ndev;
1175 int ret = AM65_CPSW_XDP_CONSUMED;
1176 struct am65_cpsw_tx_chn *tx_chn;
1177 struct netdev_queue *netif_txq;
1178 struct xdp_frame *xdpf;
1179 struct bpf_prog *prog;
1180 struct page *page;
1181 int pkt_len;
1182 u32 act;
1183 int err;
1184
1185 pkt_len = *len;
1186 prog = READ_ONCE(port->xdp_prog);
1187 if (!prog)
1188 return AM65_CPSW_XDP_PASS;
1189
1190 act = bpf_prog_run_xdp(prog, xdp);
1191 /* XDP prog might have changed packet data and boundaries */
1192 *len = xdp->data_end - xdp->data;
1193
1194 switch (act) {
1195 case XDP_PASS:
1196 ret = AM65_CPSW_XDP_PASS;
1197 goto out;
1198 case XDP_TX:
1199 tx_chn = &common->tx_chns[cpu % AM65_CPSW_MAX_QUEUES];
1200 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
1201
1202 xdpf = xdp_convert_buff_to_frame(xdp);
1203 if (unlikely(!xdpf)) {
1204 ndev->stats.tx_dropped++;
1205 goto drop;
1206 }
1207
1208 __netif_tx_lock(netif_txq, cpu);
1209 err = am65_cpsw_xdp_tx_frame(ndev, tx_chn, xdpf,
1210 AM65_CPSW_TX_BUF_TYPE_XDP_TX);
1211 __netif_tx_unlock(netif_txq);
1212 if (err)
1213 goto drop;
1214
1215 dev_sw_netstats_rx_add(ndev, pkt_len);
1216 ret = AM65_CPSW_XDP_CONSUMED;
1217 goto out;
1218 case XDP_REDIRECT:
1219 if (unlikely(xdp_do_redirect(ndev, xdp, prog)))
1220 goto drop;
1221
1222 dev_sw_netstats_rx_add(ndev, pkt_len);
1223 ret = AM65_CPSW_XDP_REDIRECT;
1224 goto out;
1225 default:
1226 bpf_warn_invalid_xdp_action(ndev, prog, act);
1227 fallthrough;
1228 case XDP_ABORTED:
1229 drop:
1230 trace_xdp_exception(ndev, prog, act);
1231 fallthrough;
1232 case XDP_DROP:
1233 ndev->stats.rx_dropped++;
1234 }
1235
1236 page = virt_to_head_page(xdp->data);
1237 am65_cpsw_put_page(flow, page, true);
1238
1239 out:
1240 return ret;
1241 }
1242
1243 /* RX psdata[2] word format - checksum information */
1244 #define AM65_CPSW_RX_PSD_CSUM_ADD GENMASK(15, 0)
1245 #define AM65_CPSW_RX_PSD_CSUM_ERR BIT(16)
1246 #define AM65_CPSW_RX_PSD_IS_FRAGMENT BIT(17)
1247 #define AM65_CPSW_RX_PSD_IS_TCP BIT(18)
1248 #define AM65_CPSW_RX_PSD_IPV6_VALID BIT(19)
1249 #define AM65_CPSW_RX_PSD_IPV4_VALID BIT(20)
1250
am65_cpsw_nuss_rx_csum(struct sk_buff * skb,u32 csum_info)1251 static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
1252 {
1253 /* HW can verify IPv4/IPv6 TCP/UDP packets checksum
1254 * csum information provides in psdata[2] word:
1255 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error
1256 * AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID
1257 * bits - indicates IPv4/IPv6 packet
1258 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet
1259 * AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets
1260 * or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR
1261 */
1262 skb_checksum_none_assert(skb);
1263
1264 if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
1265 return;
1266
1267 if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
1268 AM65_CPSW_RX_PSD_IPV4_VALID)) &&
1269 !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
1270 /* csum for fragmented packets is unsupported */
1271 if (!(csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT))
1272 skb->ip_summed = CHECKSUM_UNNECESSARY;
1273 }
1274 }
1275
am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow * flow,int cpu,int * xdp_state)1276 static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow,
1277 int cpu, int *xdp_state)
1278 {
1279 struct am65_cpsw_rx_chn *rx_chn = &flow->common->rx_chns;
1280 u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
1281 struct am65_cpsw_common *common = flow->common;
1282 struct am65_cpsw_ndev_priv *ndev_priv;
1283 struct cppi5_host_desc_t *desc_rx;
1284 struct device *dev = common->dev;
1285 struct am65_cpsw_swdata *swdata;
1286 struct page *page, *new_page;
1287 dma_addr_t desc_dma, buf_dma;
1288 struct am65_cpsw_port *port;
1289 struct net_device *ndev;
1290 u32 flow_idx = flow->id;
1291 struct sk_buff *skb;
1292 struct xdp_buff xdp;
1293 int headroom, ret;
1294 void *page_addr;
1295 u32 *psdata;
1296
1297 *xdp_state = AM65_CPSW_XDP_PASS;
1298 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
1299 if (ret) {
1300 if (ret != -ENODATA)
1301 dev_err(dev, "RX: pop chn fail %d\n", ret);
1302 return ret;
1303 }
1304
1305 if (cppi5_desc_is_tdcm(desc_dma)) {
1306 dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
1307 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ)
1308 complete(&common->tdown_complete);
1309 return 0;
1310 }
1311
1312 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
1313 dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
1314 __func__, flow_idx, &desc_dma);
1315
1316 swdata = cppi5_hdesc_get_swdata(desc_rx);
1317 page = swdata->page;
1318 page_addr = page_address(page);
1319 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
1320 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
1321 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
1322 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
1323 dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
1324 port = am65_common_get_port(common, port_id);
1325 ndev = port->ndev;
1326 psdata = cppi5_hdesc_get_psdata(desc_rx);
1327 csum_info = psdata[2];
1328 dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
1329
1330 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
1331 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
1332
1333 if (port->xdp_prog) {
1334 xdp_init_buff(&xdp, PAGE_SIZE, &port->xdp_rxq[flow->id]);
1335 xdp_prepare_buff(&xdp, page_addr, AM65_CPSW_HEADROOM,
1336 pkt_len, false);
1337 *xdp_state = am65_cpsw_run_xdp(flow, port, &xdp,
1338 cpu, &pkt_len);
1339 if (*xdp_state != AM65_CPSW_XDP_PASS)
1340 goto allocate;
1341
1342 headroom = xdp.data - xdp.data_hard_start;
1343 } else {
1344 headroom = AM65_CPSW_HEADROOM;
1345 }
1346
1347 skb = am65_cpsw_build_skb(page_addr, ndev,
1348 AM65_CPSW_MAX_PACKET_SIZE, headroom);
1349 if (unlikely(!skb)) {
1350 new_page = page;
1351 goto requeue;
1352 }
1353
1354 ndev_priv = netdev_priv(ndev);
1355 am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark);
1356 skb_put(skb, pkt_len);
1357 if (port->rx_ts_enabled)
1358 am65_cpts_rx_timestamp(common->cpts, skb);
1359 skb_mark_for_recycle(skb);
1360 skb->protocol = eth_type_trans(skb, ndev);
1361 am65_cpsw_nuss_rx_csum(skb, csum_info);
1362 napi_gro_receive(&flow->napi_rx, skb);
1363
1364 dev_sw_netstats_rx_add(ndev, pkt_len);
1365
1366 allocate:
1367 new_page = page_pool_dev_alloc_pages(flow->page_pool);
1368 if (unlikely(!new_page)) {
1369 dev_err(dev, "page alloc failed\n");
1370 return -ENOMEM;
1371 }
1372
1373 if (netif_dormant(ndev)) {
1374 am65_cpsw_put_page(flow, new_page, true);
1375 ndev->stats.rx_dropped++;
1376 return 0;
1377 }
1378
1379 requeue:
1380 ret = am65_cpsw_nuss_rx_push(common, new_page, flow_idx);
1381 if (WARN_ON(ret < 0)) {
1382 am65_cpsw_put_page(flow, new_page, true);
1383 ndev->stats.rx_errors++;
1384 ndev->stats.rx_dropped++;
1385 }
1386
1387 return ret;
1388 }
1389
am65_cpsw_nuss_rx_timer_callback(struct hrtimer * timer)1390 static enum hrtimer_restart am65_cpsw_nuss_rx_timer_callback(struct hrtimer *timer)
1391 {
1392 struct am65_cpsw_rx_flow *flow = container_of(timer,
1393 struct am65_cpsw_rx_flow,
1394 rx_hrtimer);
1395
1396 enable_irq(flow->irq);
1397 return HRTIMER_NORESTART;
1398 }
1399
am65_cpsw_nuss_rx_poll(struct napi_struct * napi_rx,int budget)1400 static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
1401 {
1402 struct am65_cpsw_rx_flow *flow = am65_cpsw_napi_to_rx_flow(napi_rx);
1403 struct am65_cpsw_common *common = flow->common;
1404 int cpu = smp_processor_id();
1405 int xdp_state_or = 0;
1406 int cur_budget, ret;
1407 int xdp_state;
1408 int num_rx = 0;
1409
1410 /* process only this flow */
1411 cur_budget = budget;
1412 while (cur_budget--) {
1413 ret = am65_cpsw_nuss_rx_packets(flow, cpu, &xdp_state);
1414 xdp_state_or |= xdp_state;
1415 if (ret)
1416 break;
1417 num_rx++;
1418 }
1419
1420 if (xdp_state_or & AM65_CPSW_XDP_REDIRECT)
1421 xdp_do_flush();
1422
1423 dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
1424
1425 if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) {
1426 if (flow->irq_disabled) {
1427 flow->irq_disabled = false;
1428 if (unlikely(flow->rx_pace_timeout)) {
1429 hrtimer_start(&flow->rx_hrtimer,
1430 ns_to_ktime(flow->rx_pace_timeout),
1431 HRTIMER_MODE_REL_PINNED);
1432 } else {
1433 enable_irq(flow->irq);
1434 }
1435 }
1436 }
1437
1438 return num_rx;
1439 }
1440
1441 static struct sk_buff *
am65_cpsw_nuss_tx_compl_packet_skb(struct am65_cpsw_tx_chn * tx_chn,dma_addr_t desc_dma)1442 am65_cpsw_nuss_tx_compl_packet_skb(struct am65_cpsw_tx_chn *tx_chn,
1443 dma_addr_t desc_dma)
1444 {
1445 struct cppi5_host_desc_t *desc_tx;
1446 struct sk_buff *skb;
1447 void **swdata;
1448
1449 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
1450 desc_dma);
1451 swdata = cppi5_hdesc_get_swdata(desc_tx);
1452 skb = *(swdata);
1453 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
1454
1455 am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);
1456
1457 dev_sw_netstats_tx_add(skb->dev, 1, skb->len);
1458
1459 return skb;
1460 }
1461
1462 static struct xdp_frame *
am65_cpsw_nuss_tx_compl_packet_xdp(struct am65_cpsw_common * common,struct am65_cpsw_tx_chn * tx_chn,dma_addr_t desc_dma,struct net_device ** ndev)1463 am65_cpsw_nuss_tx_compl_packet_xdp(struct am65_cpsw_common *common,
1464 struct am65_cpsw_tx_chn *tx_chn,
1465 dma_addr_t desc_dma,
1466 struct net_device **ndev)
1467 {
1468 struct cppi5_host_desc_t *desc_tx;
1469 struct am65_cpsw_port *port;
1470 struct xdp_frame *xdpf;
1471 u32 port_id = 0;
1472 void **swdata;
1473
1474 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
1475 cppi5_desc_get_tags_ids(&desc_tx->hdr, NULL, &port_id);
1476 swdata = cppi5_hdesc_get_swdata(desc_tx);
1477 xdpf = *(swdata);
1478 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
1479
1480 port = am65_common_get_port(common, port_id);
1481 dev_sw_netstats_tx_add(port->ndev, 1, xdpf->len);
1482 *ndev = port->ndev;
1483
1484 return xdpf;
1485 }
1486
am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn * tx_chn,struct net_device * ndev,struct netdev_queue * netif_txq)1487 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,
1488 struct netdev_queue *netif_txq)
1489 {
1490 if (netif_tx_queue_stopped(netif_txq)) {
1491 /* Check whether the queue is stopped due to stalled
1492 * tx dma, if the queue is stopped then wake the queue
1493 * as we have free desc for tx
1494 */
1495 __netif_tx_lock(netif_txq, smp_processor_id());
1496 if (netif_running(ndev) &&
1497 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
1498 netif_tx_wake_queue(netif_txq);
1499
1500 __netif_tx_unlock(netif_txq);
1501 }
1502 }
1503
am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common * common,int chn,unsigned int budget,bool * tdown)1504 static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
1505 int chn, unsigned int budget, bool *tdown)
1506 {
1507 enum am65_cpsw_tx_buf_type buf_type;
1508 struct device *dev = common->dev;
1509 struct am65_cpsw_tx_chn *tx_chn;
1510 struct netdev_queue *netif_txq;
1511 unsigned int total_bytes = 0;
1512 struct net_device *ndev;
1513 struct xdp_frame *xdpf;
1514 struct sk_buff *skb;
1515 dma_addr_t desc_dma;
1516 int res, num_tx = 0;
1517
1518 tx_chn = &common->tx_chns[chn];
1519
1520 while (true) {
1521 spin_lock(&tx_chn->lock);
1522 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1523 spin_unlock(&tx_chn->lock);
1524 if (res == -ENODATA)
1525 break;
1526
1527 if (cppi5_desc_is_tdcm(desc_dma)) {
1528 if (atomic_dec_and_test(&common->tdown_cnt))
1529 complete(&common->tdown_complete);
1530 *tdown = true;
1531 break;
1532 }
1533
1534 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
1535 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
1536 skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);
1537 ndev = skb->dev;
1538 total_bytes = skb->len;
1539 napi_consume_skb(skb, budget);
1540 } else {
1541 xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,
1542 desc_dma, &ndev);
1543 total_bytes = xdpf->len;
1544 if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX)
1545 xdp_return_frame_rx_napi(xdpf);
1546 else
1547 xdp_return_frame(xdpf);
1548 }
1549 num_tx++;
1550
1551 netif_txq = netdev_get_tx_queue(ndev, chn);
1552
1553 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1554
1555 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1556 }
1557
1558 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1559
1560 return num_tx;
1561 }
1562
am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common * common,int chn,unsigned int budget,bool * tdown)1563 static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
1564 int chn, unsigned int budget, bool *tdown)
1565 {
1566 enum am65_cpsw_tx_buf_type buf_type;
1567 struct device *dev = common->dev;
1568 struct am65_cpsw_tx_chn *tx_chn;
1569 struct netdev_queue *netif_txq;
1570 unsigned int total_bytes = 0;
1571 struct net_device *ndev;
1572 struct xdp_frame *xdpf;
1573 struct sk_buff *skb;
1574 dma_addr_t desc_dma;
1575 int res, num_tx = 0;
1576
1577 tx_chn = &common->tx_chns[chn];
1578
1579 while (true) {
1580 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1581 if (res == -ENODATA)
1582 break;
1583
1584 if (cppi5_desc_is_tdcm(desc_dma)) {
1585 if (atomic_dec_and_test(&common->tdown_cnt))
1586 complete(&common->tdown_complete);
1587 *tdown = true;
1588 break;
1589 }
1590
1591 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
1592 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
1593 skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);
1594 ndev = skb->dev;
1595 total_bytes += skb->len;
1596 napi_consume_skb(skb, budget);
1597 } else {
1598 xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,
1599 desc_dma, &ndev);
1600 total_bytes += xdpf->len;
1601 if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX)
1602 xdp_return_frame_rx_napi(xdpf);
1603 else
1604 xdp_return_frame(xdpf);
1605 }
1606 num_tx++;
1607 }
1608
1609 if (!num_tx)
1610 return 0;
1611
1612 netif_txq = netdev_get_tx_queue(ndev, chn);
1613
1614 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1615
1616 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1617
1618 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1619
1620 return num_tx;
1621 }
1622
am65_cpsw_nuss_tx_timer_callback(struct hrtimer * timer)1623 static enum hrtimer_restart am65_cpsw_nuss_tx_timer_callback(struct hrtimer *timer)
1624 {
1625 struct am65_cpsw_tx_chn *tx_chns =
1626 container_of(timer, struct am65_cpsw_tx_chn, tx_hrtimer);
1627
1628 enable_irq(tx_chns->irq);
1629 return HRTIMER_NORESTART;
1630 }
1631
am65_cpsw_nuss_tx_poll(struct napi_struct * napi_tx,int budget)1632 static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
1633 {
1634 struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);
1635 bool tdown = false;
1636 int num_tx;
1637
1638 if (AM65_CPSW_IS_CPSW2G(tx_chn->common))
1639 num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id,
1640 budget, &tdown);
1641 else
1642 num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common,
1643 tx_chn->id, budget, &tdown);
1644
1645 if (num_tx >= budget)
1646 return budget;
1647
1648 if (napi_complete_done(napi_tx, num_tx)) {
1649 if (unlikely(tx_chn->tx_pace_timeout && !tdown)) {
1650 hrtimer_start(&tx_chn->tx_hrtimer,
1651 ns_to_ktime(tx_chn->tx_pace_timeout),
1652 HRTIMER_MODE_REL_PINNED);
1653 } else {
1654 enable_irq(tx_chn->irq);
1655 }
1656 }
1657
1658 return 0;
1659 }
1660
am65_cpsw_nuss_rx_irq(int irq,void * dev_id)1661 static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id)
1662 {
1663 struct am65_cpsw_rx_flow *flow = dev_id;
1664
1665 flow->irq_disabled = true;
1666 disable_irq_nosync(irq);
1667 napi_schedule(&flow->napi_rx);
1668
1669 return IRQ_HANDLED;
1670 }
1671
am65_cpsw_nuss_tx_irq(int irq,void * dev_id)1672 static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id)
1673 {
1674 struct am65_cpsw_tx_chn *tx_chn = dev_id;
1675
1676 disable_irq_nosync(irq);
1677 napi_schedule(&tx_chn->napi_tx);
1678
1679 return IRQ_HANDLED;
1680 }
1681
am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff * skb,struct net_device * ndev)1682 static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
1683 struct net_device *ndev)
1684 {
1685 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1686 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
1687 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1688 struct device *dev = common->dev;
1689 struct am65_cpsw_tx_chn *tx_chn;
1690 struct netdev_queue *netif_txq;
1691 dma_addr_t desc_dma, buf_dma;
1692 int ret, q_idx, i;
1693 void **swdata;
1694 u32 *psdata;
1695 u32 pkt_len;
1696
1697 /* padding enabled in hw */
1698 pkt_len = skb_headlen(skb);
1699
1700 /* SKB TX timestamp */
1701 if (port->tx_ts_enabled)
1702 am65_cpts_prep_tx_timestamp(common->cpts, skb);
1703
1704 q_idx = skb_get_queue_mapping(skb);
1705 dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
1706
1707 tx_chn = &common->tx_chns[q_idx];
1708 netif_txq = netdev_get_tx_queue(ndev, q_idx);
1709
1710 /* Map the linear buffer */
1711 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,
1712 DMA_TO_DEVICE);
1713 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1714 dev_err(dev, "Failed to map tx skb buffer\n");
1715 ndev->stats.tx_errors++;
1716 goto err_free_skb;
1717 }
1718
1719 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1720 if (!first_desc) {
1721 dev_dbg(dev, "Failed to allocate descriptor\n");
1722 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,
1723 DMA_TO_DEVICE);
1724 goto busy_stop_q;
1725 }
1726
1727 am65_cpsw_nuss_set_buf_type(tx_chn, first_desc,
1728 AM65_CPSW_TX_BUF_TYPE_SKB);
1729
1730 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1731 AM65_CPSW_NAV_PS_DATA_SIZE);
1732 cppi5_desc_set_pktids(&first_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID);
1733 cppi5_hdesc_set_pkttype(first_desc, AM65_CPSW_CPPI_TX_PKT_TYPE);
1734 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);
1735
1736 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1737 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
1738 swdata = cppi5_hdesc_get_swdata(first_desc);
1739 *(swdata) = skb;
1740 psdata = cppi5_hdesc_get_psdata(first_desc);
1741
1742 /* HW csum offload if enabled */
1743 psdata[2] = 0;
1744 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1745 unsigned int cs_start, cs_offset;
1746
1747 cs_start = skb_transport_offset(skb);
1748 cs_offset = cs_start + skb->csum_offset;
1749 /* HW numerates bytes starting from 1 */
1750 psdata[2] = ((cs_offset + 1) << 24) |
1751 ((cs_start + 1) << 16) | (skb->len - cs_start);
1752 dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
1753 }
1754
1755 if (!skb_is_nonlinear(skb))
1756 goto done_tx;
1757
1758 dev_dbg(dev, "fragmented SKB\n");
1759
1760 /* Handle the case where skb is fragmented in pages */
1761 cur_desc = first_desc;
1762 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1763 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1764 u32 frag_size = skb_frag_size(frag);
1765
1766 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1767 if (!next_desc) {
1768 dev_err(dev, "Failed to allocate descriptor\n");
1769 goto busy_free_descs;
1770 }
1771
1772 am65_cpsw_nuss_set_buf_type(tx_chn, next_desc,
1773 AM65_CPSW_TX_BUF_TYPE_SKB);
1774
1775 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
1776 DMA_TO_DEVICE);
1777 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1778 dev_err(dev, "Failed to map tx skb page\n");
1779 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
1780 ndev->stats.tx_errors++;
1781 goto err_free_descs;
1782 }
1783
1784 cppi5_hdesc_reset_hbdesc(next_desc);
1785 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1786 cppi5_hdesc_attach_buf(next_desc,
1787 buf_dma, frag_size, buf_dma, frag_size);
1788
1789 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
1790 next_desc);
1791 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);
1792 cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
1793
1794 pkt_len += frag_size;
1795 cur_desc = next_desc;
1796 }
1797 WARN_ON(pkt_len != skb->len);
1798
1799 done_tx:
1800 skb_tx_timestamp(skb);
1801
1802 /* report bql before sending packet */
1803 netdev_tx_sent_queue(netif_txq, pkt_len);
1804
1805 cppi5_hdesc_set_pktlen(first_desc, pkt_len);
1806 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
1807 if (AM65_CPSW_IS_CPSW2G(common)) {
1808 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1809 } else {
1810 spin_lock_bh(&tx_chn->lock);
1811 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1812 spin_unlock_bh(&tx_chn->lock);
1813 }
1814 if (ret) {
1815 dev_err(dev, "can't push desc %d\n", ret);
1816 /* inform bql */
1817 netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1818 ndev->stats.tx_errors++;
1819 goto err_free_descs;
1820 }
1821
1822 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
1823 netif_tx_stop_queue(netif_txq);
1824 /* Barrier, so that stop_queue visible to other cpus */
1825 smp_mb__after_atomic();
1826 dev_dbg(dev, "netif_tx_stop_queue %d\n", q_idx);
1827
1828 /* re-check for smp */
1829 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
1830 MAX_SKB_FRAGS) {
1831 netif_tx_wake_queue(netif_txq);
1832 dev_dbg(dev, "netif_tx_wake_queue %d\n", q_idx);
1833 }
1834 }
1835
1836 return NETDEV_TX_OK;
1837
1838 err_free_descs:
1839 am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1840 err_free_skb:
1841 ndev->stats.tx_dropped++;
1842 dev_kfree_skb_any(skb);
1843 return NETDEV_TX_OK;
1844
1845 busy_free_descs:
1846 am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1847 busy_stop_q:
1848 netif_tx_stop_queue(netif_txq);
1849 return NETDEV_TX_BUSY;
1850 }
1851
am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device * ndev,void * addr)1852 static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev,
1853 void *addr)
1854 {
1855 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1856 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1857 struct sockaddr *sockaddr = (struct sockaddr *)addr;
1858 int ret;
1859
1860 ret = eth_prepare_mac_addr_change(ndev, addr);
1861 if (ret < 0)
1862 return ret;
1863
1864 ret = pm_runtime_resume_and_get(common->dev);
1865 if (ret < 0)
1866 return ret;
1867
1868 cpsw_ale_del_ucast(common->ale, ndev->dev_addr,
1869 HOST_PORT_NUM, 0, 0);
1870 cpsw_ale_add_ucast(common->ale, sockaddr->sa_data,
1871 HOST_PORT_NUM, ALE_SECURE, 0);
1872
1873 am65_cpsw_port_set_sl_mac(port, addr);
1874 eth_commit_mac_addr_change(ndev, sockaddr);
1875
1876 pm_runtime_put(common->dev);
1877
1878 return 0;
1879 }
1880
am65_cpsw_nuss_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)1881 static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
1882 struct ifreq *ifr)
1883 {
1884 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1885 u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype;
1886 struct hwtstamp_config cfg;
1887
1888 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1889 return -EOPNOTSUPP;
1890
1891 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1892 return -EFAULT;
1893
1894 /* TX HW timestamp */
1895 switch (cfg.tx_type) {
1896 case HWTSTAMP_TX_OFF:
1897 case HWTSTAMP_TX_ON:
1898 break;
1899 default:
1900 return -ERANGE;
1901 }
1902
1903 switch (cfg.rx_filter) {
1904 case HWTSTAMP_FILTER_NONE:
1905 port->rx_ts_enabled = false;
1906 break;
1907 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1908 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1909 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1910 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1911 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1912 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1913 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1914 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1915 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1916 port->rx_ts_enabled = true;
1917 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1918 break;
1919 case HWTSTAMP_FILTER_ALL:
1920 case HWTSTAMP_FILTER_SOME:
1921 case HWTSTAMP_FILTER_NTP_ALL:
1922 return -EOPNOTSUPP;
1923 default:
1924 return -ERANGE;
1925 }
1926
1927 port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);
1928
1929 /* cfg TX timestamp */
1930 seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET <<
1931 AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588;
1932
1933 ts_vlan_ltype = ETH_P_8021Q;
1934
1935 ts_ctrl_ltype2 = ETH_P_1588 |
1936 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 |
1937 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 |
1938 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 |
1939 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 |
1940 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 |
1941 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 |
1942 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 |
1943 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO;
1944
1945 ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS <<
1946 AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT;
1947
1948 if (port->tx_ts_enabled)
1949 ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN |
1950 AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN;
1951
1952 if (port->rx_ts_enabled)
1953 ts_ctrl |= AM65_CPSW_TS_RX_ANX_ALL_EN |
1954 AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN;
1955
1956 writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);
1957 writel(ts_vlan_ltype, port->port_base +
1958 AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG);
1959 writel(ts_ctrl_ltype2, port->port_base +
1960 AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2);
1961 writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);
1962
1963 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1964 }
1965
am65_cpsw_nuss_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)1966 static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
1967 struct ifreq *ifr)
1968 {
1969 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1970 struct hwtstamp_config cfg;
1971
1972 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1973 return -EOPNOTSUPP;
1974
1975 cfg.flags = 0;
1976 cfg.tx_type = port->tx_ts_enabled ?
1977 HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1978 cfg.rx_filter = port->rx_ts_enabled ?
1979 HWTSTAMP_FILTER_PTP_V2_EVENT : HWTSTAMP_FILTER_NONE;
1980
1981 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1982 }
1983
am65_cpsw_nuss_ndo_slave_ioctl(struct net_device * ndev,struct ifreq * req,int cmd)1984 static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev,
1985 struct ifreq *req, int cmd)
1986 {
1987 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1988
1989 if (!netif_running(ndev))
1990 return -EINVAL;
1991
1992 switch (cmd) {
1993 case SIOCSHWTSTAMP:
1994 return am65_cpsw_nuss_hwtstamp_set(ndev, req);
1995 case SIOCGHWTSTAMP:
1996 return am65_cpsw_nuss_hwtstamp_get(ndev, req);
1997 }
1998
1999 return phylink_mii_ioctl(port->slave.phylink, req, cmd);
2000 }
2001
am65_cpsw_nuss_ndo_get_stats(struct net_device * dev,struct rtnl_link_stats64 * stats)2002 static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
2003 struct rtnl_link_stats64 *stats)
2004 {
2005 dev_fetch_sw_netstats(stats, dev->tstats);
2006
2007 stats->rx_errors = dev->stats.rx_errors;
2008 stats->rx_dropped = dev->stats.rx_dropped;
2009 stats->tx_dropped = dev->stats.tx_dropped;
2010 }
2011
am65_cpsw_xdp_prog_setup(struct net_device * ndev,struct bpf_prog * prog)2012 static int am65_cpsw_xdp_prog_setup(struct net_device *ndev,
2013 struct bpf_prog *prog)
2014 {
2015 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
2016 bool running = netif_running(ndev);
2017 struct bpf_prog *old_prog;
2018
2019 if (running)
2020 am65_cpsw_nuss_ndo_slave_stop(ndev);
2021
2022 old_prog = xchg(&port->xdp_prog, prog);
2023 if (old_prog)
2024 bpf_prog_put(old_prog);
2025
2026 if (running)
2027 return am65_cpsw_nuss_ndo_slave_open(ndev);
2028
2029 return 0;
2030 }
2031
am65_cpsw_ndo_bpf(struct net_device * ndev,struct netdev_bpf * bpf)2032 static int am65_cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2033 {
2034 switch (bpf->command) {
2035 case XDP_SETUP_PROG:
2036 return am65_cpsw_xdp_prog_setup(ndev, bpf->prog);
2037 default:
2038 return -EINVAL;
2039 }
2040 }
2041
am65_cpsw_ndo_xdp_xmit(struct net_device * ndev,int n,struct xdp_frame ** frames,u32 flags)2042 static int am65_cpsw_ndo_xdp_xmit(struct net_device *ndev, int n,
2043 struct xdp_frame **frames, u32 flags)
2044 {
2045 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2046 struct am65_cpsw_tx_chn *tx_chn;
2047 struct netdev_queue *netif_txq;
2048 int cpu = smp_processor_id();
2049 int i, nxmit = 0;
2050
2051 tx_chn = &common->tx_chns[cpu % common->tx_ch_num];
2052 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
2053
2054 __netif_tx_lock(netif_txq, cpu);
2055 for (i = 0; i < n; i++) {
2056 if (am65_cpsw_xdp_tx_frame(ndev, tx_chn, frames[i],
2057 AM65_CPSW_TX_BUF_TYPE_XDP_NDO))
2058 break;
2059 nxmit++;
2060 }
2061 __netif_tx_unlock(netif_txq);
2062
2063 return nxmit;
2064 }
2065
2066 static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
2067 .ndo_open = am65_cpsw_nuss_ndo_slave_open,
2068 .ndo_stop = am65_cpsw_nuss_ndo_slave_stop,
2069 .ndo_start_xmit = am65_cpsw_nuss_ndo_slave_xmit,
2070 .ndo_set_rx_mode = am65_cpsw_nuss_ndo_slave_set_rx_mode,
2071 .ndo_get_stats64 = am65_cpsw_nuss_ndo_get_stats,
2072 .ndo_validate_addr = eth_validate_addr,
2073 .ndo_set_mac_address = am65_cpsw_nuss_ndo_slave_set_mac_address,
2074 .ndo_tx_timeout = am65_cpsw_nuss_ndo_host_tx_timeout,
2075 .ndo_vlan_rx_add_vid = am65_cpsw_nuss_ndo_slave_add_vid,
2076 .ndo_vlan_rx_kill_vid = am65_cpsw_nuss_ndo_slave_kill_vid,
2077 .ndo_eth_ioctl = am65_cpsw_nuss_ndo_slave_ioctl,
2078 .ndo_setup_tc = am65_cpsw_qos_ndo_setup_tc,
2079 .ndo_set_tx_maxrate = am65_cpsw_qos_ndo_tx_p0_set_maxrate,
2080 .ndo_bpf = am65_cpsw_ndo_bpf,
2081 .ndo_xdp_xmit = am65_cpsw_ndo_xdp_xmit,
2082 };
2083
am65_cpsw_disable_phy(struct phy * phy)2084 static void am65_cpsw_disable_phy(struct phy *phy)
2085 {
2086 phy_power_off(phy);
2087 phy_exit(phy);
2088 }
2089
am65_cpsw_enable_phy(struct phy * phy)2090 static int am65_cpsw_enable_phy(struct phy *phy)
2091 {
2092 int ret;
2093
2094 ret = phy_init(phy);
2095 if (ret < 0)
2096 return ret;
2097
2098 ret = phy_power_on(phy);
2099 if (ret < 0) {
2100 phy_exit(phy);
2101 return ret;
2102 }
2103
2104 return 0;
2105 }
2106
am65_cpsw_disable_serdes_phy(struct am65_cpsw_common * common)2107 static void am65_cpsw_disable_serdes_phy(struct am65_cpsw_common *common)
2108 {
2109 struct am65_cpsw_port *port;
2110 struct phy *phy;
2111 int i;
2112
2113 for (i = 0; i < common->port_num; i++) {
2114 port = &common->ports[i];
2115 phy = port->slave.serdes_phy;
2116 if (phy)
2117 am65_cpsw_disable_phy(phy);
2118 }
2119 }
2120
am65_cpsw_init_serdes_phy(struct device * dev,struct device_node * port_np,struct am65_cpsw_port * port)2121 static int am65_cpsw_init_serdes_phy(struct device *dev, struct device_node *port_np,
2122 struct am65_cpsw_port *port)
2123 {
2124 const char *name = "serdes";
2125 struct phy *phy;
2126 int ret;
2127
2128 phy = devm_of_phy_optional_get(dev, port_np, name);
2129 if (IS_ERR_OR_NULL(phy))
2130 return PTR_ERR_OR_ZERO(phy);
2131
2132 /* Serdes PHY exists. Store it. */
2133 port->slave.serdes_phy = phy;
2134
2135 ret = am65_cpsw_enable_phy(phy);
2136 if (ret < 0)
2137 goto err_phy;
2138
2139 return 0;
2140
2141 err_phy:
2142 devm_phy_put(dev, phy);
2143 return ret;
2144 }
2145
am65_cpsw_nuss_mac_config(struct phylink_config * config,unsigned int mode,const struct phylink_link_state * state)2146 static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned int mode,
2147 const struct phylink_link_state *state)
2148 {
2149 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
2150 phylink_config);
2151 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
2152 struct am65_cpsw_common *common = port->common;
2153
2154 if (common->pdata.extra_modes & BIT(state->interface)) {
2155 if (state->interface == PHY_INTERFACE_MODE_SGMII) {
2156 writel(ADVERTISE_SGMII,
2157 port->sgmii_base + AM65_CPSW_SGMII_MR_ADV_ABILITY_REG);
2158 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
2159 } else {
2160 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
2161 }
2162
2163 if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
2164 cpsw_sl_ctl_set(port->slave.mac_sl,
2165 CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
2166 } else {
2167 cpsw_sl_ctl_clr(port->slave.mac_sl,
2168 CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
2169 }
2170
2171 writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE,
2172 port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG);
2173 }
2174 }
2175
am65_cpsw_nuss_mac_link_down(struct phylink_config * config,unsigned int mode,phy_interface_t interface)2176 static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode,
2177 phy_interface_t interface)
2178 {
2179 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
2180 phylink_config);
2181 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
2182 struct am65_cpsw_common *common = port->common;
2183 struct net_device *ndev = port->ndev;
2184 u32 mac_control;
2185 int tmo;
2186
2187 /* disable forwarding */
2188 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
2189
2190 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
2191
2192 tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
2193 dev_dbg(common->dev, "down msc_sl %08x tmo %d\n",
2194 cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo);
2195
2196 /* All the bits that am65_cpsw_nuss_mac_link_up() can possibly set */
2197 mac_control = CPSW_SL_CTL_GMII_EN | CPSW_SL_CTL_GIG | CPSW_SL_CTL_IFCTL_A |
2198 CPSW_SL_CTL_FULLDUPLEX | CPSW_SL_CTL_RX_FLOW_EN | CPSW_SL_CTL_TX_FLOW_EN;
2199 /* If interface mode is RGMII, CPSW_SL_CTL_EXT_EN might have been set for 10 Mbps */
2200 if (phy_interface_mode_is_rgmii(interface))
2201 mac_control |= CPSW_SL_CTL_EXT_EN;
2202 /* Only clear those bits that can be set by am65_cpsw_nuss_mac_link_up() */
2203 cpsw_sl_ctl_clr(port->slave.mac_sl, mac_control);
2204
2205 am65_cpsw_qos_link_down(ndev);
2206 netif_tx_stop_all_queues(ndev);
2207 }
2208
am65_cpsw_nuss_mac_link_up(struct phylink_config * config,struct phy_device * phy,unsigned int mode,phy_interface_t interface,int speed,int duplex,bool tx_pause,bool rx_pause)2209 static void am65_cpsw_nuss_mac_link_up(struct phylink_config *config, struct phy_device *phy,
2210 unsigned int mode, phy_interface_t interface, int speed,
2211 int duplex, bool tx_pause, bool rx_pause)
2212 {
2213 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
2214 phylink_config);
2215 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
2216 struct am65_cpsw_common *common = port->common;
2217 u32 mac_control = CPSW_SL_CTL_GMII_EN;
2218 struct net_device *ndev = port->ndev;
2219
2220 /* Bring the port out of idle state */
2221 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
2222
2223 if (speed == SPEED_1000)
2224 mac_control |= CPSW_SL_CTL_GIG;
2225 /* TODO: Verify whether in-band is necessary for 10 Mbps RGMII */
2226 if (speed == SPEED_10 && phy_interface_mode_is_rgmii(interface))
2227 /* Can be used with in band mode only */
2228 mac_control |= CPSW_SL_CTL_EXT_EN;
2229 if (speed == SPEED_100 && interface == PHY_INTERFACE_MODE_RMII)
2230 mac_control |= CPSW_SL_CTL_IFCTL_A;
2231 if (duplex)
2232 mac_control |= CPSW_SL_CTL_FULLDUPLEX;
2233
2234 /* rx_pause/tx_pause */
2235 if (rx_pause)
2236 mac_control |= CPSW_SL_CTL_TX_FLOW_EN;
2237
2238 if (tx_pause)
2239 mac_control |= CPSW_SL_CTL_RX_FLOW_EN;
2240
2241 cpsw_sl_ctl_set(port->slave.mac_sl, mac_control);
2242
2243 /* enable forwarding */
2244 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
2245
2246 am65_cpsw_qos_link_up(ndev, speed);
2247 netif_tx_wake_all_queues(ndev);
2248 }
2249
2250 static const struct phylink_mac_ops am65_cpsw_phylink_mac_ops = {
2251 .mac_config = am65_cpsw_nuss_mac_config,
2252 .mac_link_down = am65_cpsw_nuss_mac_link_down,
2253 .mac_link_up = am65_cpsw_nuss_mac_link_up,
2254 };
2255
am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port * port)2256 static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port)
2257 {
2258 struct am65_cpsw_common *common = port->common;
2259
2260 if (!port->disabled)
2261 return;
2262
2263 cpsw_ale_control_set(common->ale, port->port_id,
2264 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
2265
2266 cpsw_sl_reset(port->slave.mac_sl, 100);
2267 cpsw_sl_ctl_reset(port->slave.mac_sl);
2268 }
2269
am65_cpsw_nuss_free_tx_chns(void * data)2270 static void am65_cpsw_nuss_free_tx_chns(void *data)
2271 {
2272 struct am65_cpsw_common *common = data;
2273 int i;
2274
2275 for (i = 0; i < common->tx_ch_num; i++) {
2276 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2277
2278 if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
2279 k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
2280
2281 if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
2282 k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
2283
2284 memset(tx_chn, 0, sizeof(*tx_chn));
2285 }
2286 }
2287
am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common * common)2288 static void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common)
2289 {
2290 struct device *dev = common->dev;
2291 int i;
2292
2293 common->tx_ch_rate_msk = 0;
2294 for (i = 0; i < common->tx_ch_num; i++) {
2295 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2296
2297 if (tx_chn->irq > 0)
2298 devm_free_irq(dev, tx_chn->irq, tx_chn);
2299
2300 netif_napi_del(&tx_chn->napi_tx);
2301 }
2302
2303 am65_cpsw_nuss_free_tx_chns(common);
2304 }
2305
am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common * common)2306 static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common)
2307 {
2308 struct device *dev = common->dev;
2309 struct am65_cpsw_tx_chn *tx_chn;
2310 int i, ret = 0;
2311
2312 for (i = 0; i < common->tx_ch_num; i++) {
2313 tx_chn = &common->tx_chns[i];
2314
2315 hrtimer_init(&tx_chn->tx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
2316 tx_chn->tx_hrtimer.function = &am65_cpsw_nuss_tx_timer_callback;
2317
2318 netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx,
2319 am65_cpsw_nuss_tx_poll);
2320
2321 ret = devm_request_irq(dev, tx_chn->irq,
2322 am65_cpsw_nuss_tx_irq,
2323 IRQF_TRIGGER_HIGH,
2324 tx_chn->tx_chn_name, tx_chn);
2325 if (ret) {
2326 dev_err(dev, "failure requesting tx%u irq %u, %d\n",
2327 tx_chn->id, tx_chn->irq, ret);
2328 goto err;
2329 }
2330 }
2331
2332 return 0;
2333
2334 err:
2335 netif_napi_del(&tx_chn->napi_tx);
2336 for (--i; i >= 0; i--) {
2337 tx_chn = &common->tx_chns[i];
2338 devm_free_irq(dev, tx_chn->irq, tx_chn);
2339 netif_napi_del(&tx_chn->napi_tx);
2340 }
2341
2342 return ret;
2343 }
2344
am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common * common)2345 static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
2346 {
2347 u32 max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
2348 struct k3_udma_glue_tx_channel_cfg tx_cfg = { 0 };
2349 struct device *dev = common->dev;
2350 struct k3_ring_cfg ring_cfg = {
2351 .elm_size = K3_RINGACC_RING_ELSIZE_8,
2352 .mode = K3_RINGACC_RING_MODE_RING,
2353 .flags = 0
2354 };
2355 u32 hdesc_size, hdesc_size_out;
2356 int i, ret = 0;
2357
2358 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
2359 AM65_CPSW_NAV_SW_DATA_SIZE);
2360
2361 tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
2362 tx_cfg.tx_cfg = ring_cfg;
2363 tx_cfg.txcq_cfg = ring_cfg;
2364 tx_cfg.tx_cfg.size = max_desc_num;
2365 tx_cfg.txcq_cfg.size = max_desc_num;
2366
2367 for (i = 0; i < common->tx_ch_num; i++) {
2368 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2369
2370 snprintf(tx_chn->tx_chn_name,
2371 sizeof(tx_chn->tx_chn_name), "tx%d", i);
2372
2373 spin_lock_init(&tx_chn->lock);
2374 tx_chn->common = common;
2375 tx_chn->id = i;
2376 tx_chn->descs_num = max_desc_num;
2377
2378 tx_chn->tx_chn =
2379 k3_udma_glue_request_tx_chn(dev,
2380 tx_chn->tx_chn_name,
2381 &tx_cfg);
2382 if (IS_ERR(tx_chn->tx_chn)) {
2383 ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),
2384 "Failed to request tx dma channel\n");
2385 goto err;
2386 }
2387 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);
2388
2389 tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,
2390 tx_chn->descs_num,
2391 hdesc_size,
2392 tx_chn->tx_chn_name);
2393 if (IS_ERR(tx_chn->desc_pool)) {
2394 ret = PTR_ERR(tx_chn->desc_pool);
2395 dev_err(dev, "Failed to create poll %d\n", ret);
2396 goto err;
2397 }
2398
2399 hdesc_size_out = k3_cppi_desc_pool_desc_size(tx_chn->desc_pool);
2400 tx_chn->dsize_log2 = __fls(hdesc_size_out);
2401 WARN_ON(hdesc_size_out != (1 << tx_chn->dsize_log2));
2402
2403 tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
2404 if (tx_chn->irq < 0) {
2405 dev_err(dev, "Failed to get tx dma irq %d\n",
2406 tx_chn->irq);
2407 ret = tx_chn->irq;
2408 goto err;
2409 }
2410
2411 snprintf(tx_chn->tx_chn_name,
2412 sizeof(tx_chn->tx_chn_name), "%s-tx%d",
2413 dev_name(dev), tx_chn->id);
2414 }
2415
2416 ret = am65_cpsw_nuss_ndev_add_tx_napi(common);
2417 if (ret) {
2418 dev_err(dev, "Failed to add tx NAPI %d\n", ret);
2419 goto err;
2420 }
2421
2422 return 0;
2423
2424 err:
2425 am65_cpsw_nuss_free_tx_chns(common);
2426
2427 return ret;
2428 }
2429
am65_cpsw_nuss_free_rx_chns(void * data)2430 static void am65_cpsw_nuss_free_rx_chns(void *data)
2431 {
2432 struct am65_cpsw_common *common = data;
2433 struct am65_cpsw_rx_chn *rx_chn;
2434
2435 rx_chn = &common->rx_chns;
2436
2437 if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
2438 k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
2439
2440 if (!IS_ERR_OR_NULL(rx_chn->rx_chn))
2441 k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
2442 }
2443
am65_cpsw_nuss_remove_rx_chns(struct am65_cpsw_common * common)2444 static void am65_cpsw_nuss_remove_rx_chns(struct am65_cpsw_common *common)
2445 {
2446 struct device *dev = common->dev;
2447 struct am65_cpsw_rx_chn *rx_chn;
2448 struct am65_cpsw_rx_flow *flows;
2449 int i;
2450
2451 rx_chn = &common->rx_chns;
2452 flows = rx_chn->flows;
2453
2454 for (i = 0; i < common->rx_ch_num_flows; i++) {
2455 if (!(flows[i].irq < 0))
2456 devm_free_irq(dev, flows[i].irq, &flows[i]);
2457 netif_napi_del(&flows[i].napi_rx);
2458 }
2459
2460 am65_cpsw_nuss_free_rx_chns(common);
2461
2462 common->rx_flow_id_base = -1;
2463 }
2464
am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common * common)2465 static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
2466 {
2467 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
2468 struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 };
2469 u32 max_desc_num = AM65_CPSW_MAX_RX_DESC;
2470 struct device *dev = common->dev;
2471 struct am65_cpsw_rx_flow *flow;
2472 u32 hdesc_size, hdesc_size_out;
2473 u32 fdqring_id;
2474 int i, ret = 0;
2475
2476 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
2477 AM65_CPSW_NAV_SW_DATA_SIZE);
2478
2479 rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
2480 rx_cfg.flow_id_num = common->rx_ch_num_flows;
2481 rx_cfg.flow_id_base = common->rx_flow_id_base;
2482
2483 /* init all flows */
2484 rx_chn->dev = dev;
2485 rx_chn->descs_num = max_desc_num * rx_cfg.flow_id_num;
2486
2487 for (i = 0; i < common->rx_ch_num_flows; i++) {
2488 flow = &rx_chn->flows[i];
2489 flow->page_pool = NULL;
2490 }
2491
2492 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);
2493 if (IS_ERR(rx_chn->rx_chn)) {
2494 ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn),
2495 "Failed to request rx dma channel\n");
2496 goto err;
2497 }
2498 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);
2499
2500 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,
2501 rx_chn->descs_num,
2502 hdesc_size, "rx");
2503 if (IS_ERR(rx_chn->desc_pool)) {
2504 ret = PTR_ERR(rx_chn->desc_pool);
2505 dev_err(dev, "Failed to create rx poll %d\n", ret);
2506 goto err;
2507 }
2508
2509 hdesc_size_out = k3_cppi_desc_pool_desc_size(rx_chn->desc_pool);
2510 rx_chn->dsize_log2 = __fls(hdesc_size_out);
2511 WARN_ON(hdesc_size_out != (1 << rx_chn->dsize_log2));
2512
2513 common->rx_flow_id_base =
2514 k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
2515 dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
2516
2517 fdqring_id = K3_RINGACC_RING_ID_ANY;
2518 for (i = 0; i < rx_cfg.flow_id_num; i++) {
2519 struct k3_ring_cfg rxring_cfg = {
2520 .elm_size = K3_RINGACC_RING_ELSIZE_8,
2521 .mode = K3_RINGACC_RING_MODE_RING,
2522 .flags = 0,
2523 };
2524 struct k3_ring_cfg fdqring_cfg = {
2525 .elm_size = K3_RINGACC_RING_ELSIZE_8,
2526 .flags = K3_RINGACC_RING_SHARED,
2527 };
2528 struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
2529 .rx_cfg = rxring_cfg,
2530 .rxfdq_cfg = fdqring_cfg,
2531 .ring_rxq_id = K3_RINGACC_RING_ID_ANY,
2532 .src_tag_lo_sel =
2533 K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
2534 };
2535
2536 flow = &rx_chn->flows[i];
2537 flow->id = i;
2538 flow->common = common;
2539 flow->irq = -EINVAL;
2540
2541 rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
2542 rx_flow_cfg.rx_cfg.size = max_desc_num;
2543 /* share same FDQ for all flows */
2544 rx_flow_cfg.rxfdq_cfg.size = max_desc_num * rx_cfg.flow_id_num;
2545 rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode;
2546
2547 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,
2548 i, &rx_flow_cfg);
2549 if (ret) {
2550 dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
2551 goto err_flow;
2552 }
2553 if (!i)
2554 fdqring_id =
2555 k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,
2556 i);
2557
2558 flow->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);
2559 if (flow->irq <= 0) {
2560 dev_err(dev, "Failed to get rx dma irq %d\n",
2561 flow->irq);
2562 ret = flow->irq;
2563 goto err_flow;
2564 }
2565
2566 snprintf(flow->name,
2567 sizeof(flow->name), "%s-rx%d",
2568 dev_name(dev), i);
2569 hrtimer_init(&flow->rx_hrtimer, CLOCK_MONOTONIC,
2570 HRTIMER_MODE_REL_PINNED);
2571 flow->rx_hrtimer.function = &am65_cpsw_nuss_rx_timer_callback;
2572
2573 netif_napi_add(common->dma_ndev, &flow->napi_rx,
2574 am65_cpsw_nuss_rx_poll);
2575
2576 ret = devm_request_irq(dev, flow->irq,
2577 am65_cpsw_nuss_rx_irq,
2578 IRQF_TRIGGER_HIGH,
2579 flow->name, flow);
2580 if (ret) {
2581 dev_err(dev, "failure requesting rx %d irq %u, %d\n",
2582 i, flow->irq, ret);
2583 flow->irq = -EINVAL;
2584 goto err_request_irq;
2585 }
2586 }
2587
2588 /* setup classifier to route priorities to flows */
2589 cpsw_ale_classifier_setup_default(common->ale, common->rx_ch_num_flows);
2590
2591 return 0;
2592
2593 err_request_irq:
2594 netif_napi_del(&flow->napi_rx);
2595
2596 err_flow:
2597 for (--i; i >= 0; i--) {
2598 flow = &rx_chn->flows[i];
2599 devm_free_irq(dev, flow->irq, flow);
2600 netif_napi_del(&flow->napi_rx);
2601 }
2602
2603 err:
2604 am65_cpsw_nuss_free_rx_chns(common);
2605
2606 return ret;
2607 }
2608
am65_cpsw_nuss_init_host_p(struct am65_cpsw_common * common)2609 static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common)
2610 {
2611 struct am65_cpsw_host *host_p = am65_common_get_host(common);
2612
2613 host_p->common = common;
2614 host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;
2615 host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;
2616
2617 return 0;
2618 }
2619
am65_cpsw_am654_get_efuse_macid(struct device_node * of_node,int slave,u8 * mac_addr)2620 static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node,
2621 int slave, u8 *mac_addr)
2622 {
2623 u32 mac_lo, mac_hi, offset;
2624 struct regmap *syscon;
2625
2626 syscon = syscon_regmap_lookup_by_phandle_args(of_node, "ti,syscon-efuse",
2627 1, &offset);
2628 if (IS_ERR(syscon)) {
2629 if (PTR_ERR(syscon) == -ENODEV)
2630 return 0;
2631 return PTR_ERR(syscon);
2632 }
2633
2634 regmap_read(syscon, offset, &mac_lo);
2635 regmap_read(syscon, offset + 4, &mac_hi);
2636
2637 mac_addr[0] = (mac_hi >> 8) & 0xff;
2638 mac_addr[1] = mac_hi & 0xff;
2639 mac_addr[2] = (mac_lo >> 24) & 0xff;
2640 mac_addr[3] = (mac_lo >> 16) & 0xff;
2641 mac_addr[4] = (mac_lo >> 8) & 0xff;
2642 mac_addr[5] = mac_lo & 0xff;
2643
2644 return 0;
2645 }
2646
am65_cpsw_init_cpts(struct am65_cpsw_common * common)2647 static int am65_cpsw_init_cpts(struct am65_cpsw_common *common)
2648 {
2649 struct device *dev = common->dev;
2650 struct device_node *node;
2651 struct am65_cpts *cpts;
2652 void __iomem *reg_base;
2653
2654 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
2655 return 0;
2656
2657 node = of_get_child_by_name(dev->of_node, "cpts");
2658 if (!node) {
2659 dev_err(dev, "%s cpts not found\n", __func__);
2660 return -ENOENT;
2661 }
2662
2663 reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;
2664 cpts = am65_cpts_create(dev, reg_base, node);
2665 if (IS_ERR(cpts)) {
2666 int ret = PTR_ERR(cpts);
2667
2668 of_node_put(node);
2669 dev_err(dev, "cpts create err %d\n", ret);
2670 return ret;
2671 }
2672 common->cpts = cpts;
2673 /* Forbid PM runtime if CPTS is running.
2674 * K3 CPSWxG modules may completely lose context during ON->OFF
2675 * transitions depending on integration.
2676 * AM65x/J721E MCU CPSW2G: false
2677 * J721E MAIN_CPSW9G: true
2678 */
2679 pm_runtime_forbid(dev);
2680
2681 return 0;
2682 }
2683
am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common * common)2684 static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
2685 {
2686 struct device_node *node, *port_np;
2687 struct device *dev = common->dev;
2688 int ret;
2689
2690 node = of_get_child_by_name(dev->of_node, "ethernet-ports");
2691 if (!node)
2692 return -ENOENT;
2693
2694 for_each_child_of_node(node, port_np) {
2695 struct am65_cpsw_port *port;
2696 u32 port_id;
2697
2698 /* it is not a slave port node, continue */
2699 if (strcmp(port_np->name, "port"))
2700 continue;
2701
2702 ret = of_property_read_u32(port_np, "reg", &port_id);
2703 if (ret < 0) {
2704 dev_err(dev, "%pOF error reading port_id %d\n",
2705 port_np, ret);
2706 goto of_node_put;
2707 }
2708
2709 if (!port_id || port_id > common->port_num) {
2710 dev_err(dev, "%pOF has invalid port_id %u %s\n",
2711 port_np, port_id, port_np->name);
2712 ret = -EINVAL;
2713 goto of_node_put;
2714 }
2715
2716 port = am65_common_get_port(common, port_id);
2717 port->port_id = port_id;
2718 port->common = common;
2719 port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
2720 AM65_CPSW_NU_PORTS_OFFSET * (port_id);
2721 if (common->pdata.extra_modes)
2722 port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id);
2723 port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
2724 (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
2725 port->name = of_get_property(port_np, "label", NULL);
2726 port->fetch_ram_base =
2727 common->cpsw_base + AM65_CPSW_NU_FRAM_BASE +
2728 (AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1));
2729
2730 port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base);
2731 if (IS_ERR(port->slave.mac_sl)) {
2732 ret = PTR_ERR(port->slave.mac_sl);
2733 goto of_node_put;
2734 }
2735
2736 port->disabled = !of_device_is_available(port_np);
2737 if (port->disabled) {
2738 common->disabled_ports_mask |= BIT(port->port_id);
2739 continue;
2740 }
2741
2742 port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL);
2743 if (IS_ERR(port->slave.ifphy)) {
2744 ret = PTR_ERR(port->slave.ifphy);
2745 dev_err(dev, "%pOF error retrieving port phy: %d\n",
2746 port_np, ret);
2747 goto of_node_put;
2748 }
2749
2750 /* Initialize the Serdes PHY for the port */
2751 ret = am65_cpsw_init_serdes_phy(dev, port_np, port);
2752 if (ret)
2753 goto of_node_put;
2754
2755 port->slave.mac_only =
2756 of_property_read_bool(port_np, "ti,mac-only");
2757
2758 /* get phy/link info */
2759 port->slave.port_np = port_np;
2760 ret = of_get_phy_mode(port_np, &port->slave.phy_if);
2761 if (ret) {
2762 dev_err(dev, "%pOF read phy-mode err %d\n",
2763 port_np, ret);
2764 goto of_node_put;
2765 }
2766
2767 ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, port->slave.phy_if);
2768 if (ret)
2769 goto of_node_put;
2770
2771 ret = of_get_mac_address(port_np, port->slave.mac_addr);
2772 if (ret) {
2773 am65_cpsw_am654_get_efuse_macid(port_np,
2774 port->port_id,
2775 port->slave.mac_addr);
2776 if (!is_valid_ether_addr(port->slave.mac_addr)) {
2777 eth_random_addr(port->slave.mac_addr);
2778 dev_err(dev, "Use random MAC address\n");
2779 }
2780 }
2781
2782 /* Reset all Queue priorities to 0 */
2783 writel(0, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP);
2784 }
2785 of_node_put(node);
2786
2787 /* is there at least one ext.port */
2788 if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) {
2789 dev_err(dev, "No Ext. port are available\n");
2790 return -ENODEV;
2791 }
2792
2793 return 0;
2794
2795 of_node_put:
2796 of_node_put(port_np);
2797 of_node_put(node);
2798 return ret;
2799 }
2800
am65_cpsw_nuss_phylink_cleanup(struct am65_cpsw_common * common)2801 static void am65_cpsw_nuss_phylink_cleanup(struct am65_cpsw_common *common)
2802 {
2803 struct am65_cpsw_port *port;
2804 int i;
2805
2806 for (i = 0; i < common->port_num; i++) {
2807 port = &common->ports[i];
2808 if (port->slave.phylink)
2809 phylink_destroy(port->slave.phylink);
2810 }
2811 }
2812
2813 static int
am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common * common,u32 port_idx)2814 am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
2815 {
2816 struct am65_cpsw_ndev_priv *ndev_priv;
2817 struct device *dev = common->dev;
2818 struct am65_cpsw_port *port;
2819 struct phylink *phylink;
2820
2821 port = &common->ports[port_idx];
2822
2823 if (port->disabled)
2824 return 0;
2825
2826 /* alloc netdev */
2827 port->ndev = alloc_etherdev_mqs(sizeof(struct am65_cpsw_ndev_priv),
2828 AM65_CPSW_MAX_QUEUES,
2829 AM65_CPSW_MAX_QUEUES);
2830 if (!port->ndev) {
2831 dev_err(dev, "error allocating slave net_device %u\n",
2832 port->port_id);
2833 return -ENOMEM;
2834 }
2835
2836 ndev_priv = netdev_priv(port->ndev);
2837 ndev_priv->port = port;
2838 ndev_priv->msg_enable = AM65_CPSW_DEBUG;
2839 mutex_init(&ndev_priv->mm_lock);
2840 port->qos.link_speed = SPEED_UNKNOWN;
2841 SET_NETDEV_DEV(port->ndev, dev);
2842 port->ndev->dev.of_node = port->slave.port_np;
2843
2844 eth_hw_addr_set(port->ndev, port->slave.mac_addr);
2845
2846 port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;
2847 port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE -
2848 (VLAN_ETH_HLEN + ETH_FCS_LEN);
2849 port->ndev->hw_features = NETIF_F_SG |
2850 NETIF_F_RXCSUM |
2851 NETIF_F_HW_CSUM |
2852 NETIF_F_HW_TC;
2853 port->ndev->features = port->ndev->hw_features |
2854 NETIF_F_HW_VLAN_CTAG_FILTER;
2855 port->ndev->xdp_features = NETDEV_XDP_ACT_BASIC |
2856 NETDEV_XDP_ACT_REDIRECT |
2857 NETDEV_XDP_ACT_NDO_XMIT;
2858 port->ndev->vlan_features |= NETIF_F_SG;
2859 port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops;
2860 port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
2861
2862 /* Configuring Phylink */
2863 port->slave.phylink_config.dev = &port->ndev->dev;
2864 port->slave.phylink_config.type = PHYLINK_NETDEV;
2865 port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 |
2866 MAC_1000FD | MAC_5000FD;
2867 port->slave.phylink_config.mac_managed_pm = true; /* MAC does PM */
2868
2869 switch (port->slave.phy_if) {
2870 case PHY_INTERFACE_MODE_RGMII:
2871 case PHY_INTERFACE_MODE_RGMII_ID:
2872 case PHY_INTERFACE_MODE_RGMII_RXID:
2873 case PHY_INTERFACE_MODE_RGMII_TXID:
2874 phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces);
2875 break;
2876
2877 case PHY_INTERFACE_MODE_RMII:
2878 __set_bit(PHY_INTERFACE_MODE_RMII,
2879 port->slave.phylink_config.supported_interfaces);
2880 break;
2881
2882 case PHY_INTERFACE_MODE_QSGMII:
2883 case PHY_INTERFACE_MODE_SGMII:
2884 case PHY_INTERFACE_MODE_USXGMII:
2885 if (common->pdata.extra_modes & BIT(port->slave.phy_if)) {
2886 __set_bit(port->slave.phy_if,
2887 port->slave.phylink_config.supported_interfaces);
2888 } else {
2889 dev_err(dev, "selected phy-mode is not supported\n");
2890 return -EOPNOTSUPP;
2891 }
2892 break;
2893
2894 default:
2895 dev_err(dev, "selected phy-mode is not supported\n");
2896 return -EOPNOTSUPP;
2897 }
2898
2899 phylink = phylink_create(&port->slave.phylink_config,
2900 of_fwnode_handle(port->slave.port_np),
2901 port->slave.phy_if,
2902 &am65_cpsw_phylink_mac_ops);
2903 if (IS_ERR(phylink))
2904 return PTR_ERR(phylink);
2905
2906 port->slave.phylink = phylink;
2907
2908 /* Disable TX checksum offload by default due to HW bug */
2909 if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)
2910 port->ndev->features &= ~NETIF_F_HW_CSUM;
2911
2912 port->ndev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
2913 port->xdp_prog = NULL;
2914
2915 if (!common->dma_ndev)
2916 common->dma_ndev = port->ndev;
2917
2918 return 0;
2919 }
2920
am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common * common)2921 static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common)
2922 {
2923 int ret;
2924 int i;
2925
2926 for (i = 0; i < common->port_num; i++) {
2927 ret = am65_cpsw_nuss_init_port_ndev(common, i);
2928 if (ret)
2929 return ret;
2930 }
2931
2932 return ret;
2933 }
2934
am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common * common)2935 static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common)
2936 {
2937 struct am65_cpsw_port *port;
2938 int i;
2939
2940 for (i = 0; i < common->port_num; i++) {
2941 port = &common->ports[i];
2942 if (!port->ndev)
2943 continue;
2944 if (port->ndev->reg_state == NETREG_REGISTERED)
2945 unregister_netdev(port->ndev);
2946 free_netdev(port->ndev);
2947 port->ndev = NULL;
2948 }
2949 }
2950
am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common * common)2951 static void am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common *common)
2952 {
2953 int set_val = 0;
2954 int i;
2955
2956 if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask))
2957 set_val = 1;
2958
2959 dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val);
2960
2961 for (i = 1; i <= common->port_num; i++) {
2962 struct am65_cpsw_port *port = am65_common_get_port(common, i);
2963 struct am65_cpsw_ndev_priv *priv;
2964
2965 if (!port->ndev)
2966 continue;
2967
2968 priv = am65_ndev_to_priv(port->ndev);
2969 priv->offload_fwd_mark = set_val;
2970 }
2971 }
2972
am65_cpsw_port_dev_check(const struct net_device * ndev)2973 bool am65_cpsw_port_dev_check(const struct net_device *ndev)
2974 {
2975 if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) {
2976 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2977
2978 return !common->is_emac_mode;
2979 }
2980
2981 return false;
2982 }
2983
am65_cpsw_netdevice_port_link(struct net_device * ndev,struct net_device * br_ndev,struct netlink_ext_ack * extack)2984 static int am65_cpsw_netdevice_port_link(struct net_device *ndev,
2985 struct net_device *br_ndev,
2986 struct netlink_ext_ack *extack)
2987 {
2988 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2989 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2990 int err;
2991
2992 if (!common->br_members) {
2993 common->hw_bridge_dev = br_ndev;
2994 } else {
2995 /* This is adding the port to a second bridge, this is
2996 * unsupported
2997 */
2998 if (common->hw_bridge_dev != br_ndev)
2999 return -EOPNOTSUPP;
3000 }
3001
3002 err = switchdev_bridge_port_offload(ndev, ndev, NULL, NULL, NULL,
3003 false, extack);
3004 if (err)
3005 return err;
3006
3007 common->br_members |= BIT(priv->port->port_id);
3008
3009 am65_cpsw_port_offload_fwd_mark_update(common);
3010
3011 return NOTIFY_DONE;
3012 }
3013
am65_cpsw_netdevice_port_unlink(struct net_device * ndev)3014 static void am65_cpsw_netdevice_port_unlink(struct net_device *ndev)
3015 {
3016 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
3017 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
3018
3019 switchdev_bridge_port_unoffload(ndev, NULL, NULL, NULL);
3020
3021 common->br_members &= ~BIT(priv->port->port_id);
3022
3023 am65_cpsw_port_offload_fwd_mark_update(common);
3024
3025 if (!common->br_members)
3026 common->hw_bridge_dev = NULL;
3027 }
3028
3029 /* netdev notifier */
am65_cpsw_netdevice_event(struct notifier_block * unused,unsigned long event,void * ptr)3030 static int am65_cpsw_netdevice_event(struct notifier_block *unused,
3031 unsigned long event, void *ptr)
3032 {
3033 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
3034 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
3035 struct netdev_notifier_changeupper_info *info;
3036 int ret = NOTIFY_DONE;
3037
3038 if (!am65_cpsw_port_dev_check(ndev))
3039 return NOTIFY_DONE;
3040
3041 switch (event) {
3042 case NETDEV_CHANGEUPPER:
3043 info = ptr;
3044
3045 if (netif_is_bridge_master(info->upper_dev)) {
3046 if (info->linking)
3047 ret = am65_cpsw_netdevice_port_link(ndev,
3048 info->upper_dev,
3049 extack);
3050 else
3051 am65_cpsw_netdevice_port_unlink(ndev);
3052 }
3053 break;
3054 default:
3055 return NOTIFY_DONE;
3056 }
3057
3058 return notifier_from_errno(ret);
3059 }
3060
am65_cpsw_register_notifiers(struct am65_cpsw_common * cpsw)3061 static int am65_cpsw_register_notifiers(struct am65_cpsw_common *cpsw)
3062 {
3063 int ret = 0;
3064
3065 if (AM65_CPSW_IS_CPSW2G(cpsw) ||
3066 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
3067 return 0;
3068
3069 cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event;
3070 ret = register_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
3071 if (ret) {
3072 dev_err(cpsw->dev, "can't register netdevice notifier\n");
3073 return ret;
3074 }
3075
3076 ret = am65_cpsw_switchdev_register_notifiers(cpsw);
3077 if (ret)
3078 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
3079
3080 return ret;
3081 }
3082
am65_cpsw_unregister_notifiers(struct am65_cpsw_common * cpsw)3083 static void am65_cpsw_unregister_notifiers(struct am65_cpsw_common *cpsw)
3084 {
3085 if (AM65_CPSW_IS_CPSW2G(cpsw) ||
3086 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
3087 return;
3088
3089 am65_cpsw_switchdev_unregister_notifiers(cpsw);
3090 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
3091 }
3092
3093 static const struct devlink_ops am65_cpsw_devlink_ops = {};
3094
am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common * cpsw)3095 static void am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common *cpsw)
3096 {
3097 cpsw_ale_add_mcast(cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, 0,
3098 ALE_MCAST_BLOCK_LEARN_FWD);
3099 }
3100
am65_cpsw_init_host_port_switch(struct am65_cpsw_common * common)3101 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common)
3102 {
3103 struct am65_cpsw_host *host = am65_common_get_host(common);
3104
3105 writel(common->default_vlan, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3106
3107 am65_cpsw_init_stp_ale_entry(common);
3108
3109 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 1);
3110 dev_dbg(common->dev, "Set P0_UNI_FLOOD\n");
3111 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 0);
3112 }
3113
am65_cpsw_init_host_port_emac(struct am65_cpsw_common * common)3114 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common)
3115 {
3116 struct am65_cpsw_host *host = am65_common_get_host(common);
3117
3118 writel(0, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3119
3120 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 0);
3121 dev_dbg(common->dev, "unset P0_UNI_FLOOD\n");
3122
3123 /* learning make no sense in multi-mac mode */
3124 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 1);
3125 }
3126
am65_cpsw_dl_switch_mode_get(struct devlink * dl,u32 id,struct devlink_param_gset_ctx * ctx)3127 static int am65_cpsw_dl_switch_mode_get(struct devlink *dl, u32 id,
3128 struct devlink_param_gset_ctx *ctx)
3129 {
3130 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
3131 struct am65_cpsw_common *common = dl_priv->common;
3132
3133 dev_dbg(common->dev, "%s id:%u\n", __func__, id);
3134
3135 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
3136 return -EOPNOTSUPP;
3137
3138 ctx->val.vbool = !common->is_emac_mode;
3139
3140 return 0;
3141 }
3142
am65_cpsw_init_port_emac_ale(struct am65_cpsw_port * port)3143 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port)
3144 {
3145 struct am65_cpsw_slave_data *slave = &port->slave;
3146 struct am65_cpsw_common *common = port->common;
3147 u32 port_mask;
3148
3149 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3150
3151 if (slave->mac_only)
3152 /* enable mac-only mode on port */
3153 cpsw_ale_control_set(common->ale, port->port_id,
3154 ALE_PORT_MACONLY, 1);
3155
3156 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_NOLEARN, 1);
3157
3158 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
3159
3160 cpsw_ale_add_ucast(common->ale, port->ndev->dev_addr,
3161 HOST_PORT_NUM, ALE_SECURE, slave->port_vlan);
3162 cpsw_ale_add_mcast(common->ale, port->ndev->broadcast,
3163 port_mask, ALE_VLAN, slave->port_vlan, ALE_MCAST_FWD_2);
3164 }
3165
am65_cpsw_init_port_switch_ale(struct am65_cpsw_port * port)3166 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port)
3167 {
3168 struct am65_cpsw_slave_data *slave = &port->slave;
3169 struct am65_cpsw_common *cpsw = port->common;
3170 u32 port_mask;
3171
3172 cpsw_ale_control_set(cpsw->ale, port->port_id,
3173 ALE_PORT_NOLEARN, 0);
3174
3175 cpsw_ale_add_ucast(cpsw->ale, port->ndev->dev_addr,
3176 HOST_PORT_NUM, ALE_SECURE | ALE_BLOCKED | ALE_VLAN,
3177 slave->port_vlan);
3178
3179 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
3180
3181 cpsw_ale_add_mcast(cpsw->ale, port->ndev->broadcast,
3182 port_mask, ALE_VLAN, slave->port_vlan,
3183 ALE_MCAST_FWD_2);
3184
3185 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3186
3187 cpsw_ale_control_set(cpsw->ale, port->port_id,
3188 ALE_PORT_MACONLY, 0);
3189 }
3190
am65_cpsw_dl_switch_mode_set(struct devlink * dl,u32 id,struct devlink_param_gset_ctx * ctx,struct netlink_ext_ack * extack)3191 static int am65_cpsw_dl_switch_mode_set(struct devlink *dl, u32 id,
3192 struct devlink_param_gset_ctx *ctx,
3193 struct netlink_ext_ack *extack)
3194 {
3195 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
3196 struct am65_cpsw_common *cpsw = dl_priv->common;
3197 bool switch_en = ctx->val.vbool;
3198 bool if_running = false;
3199 int i;
3200
3201 dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id);
3202
3203 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
3204 return -EOPNOTSUPP;
3205
3206 if (switch_en == !cpsw->is_emac_mode)
3207 return 0;
3208
3209 if (!switch_en && cpsw->br_members) {
3210 dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n");
3211 return -EINVAL;
3212 }
3213
3214 rtnl_lock();
3215
3216 cpsw->is_emac_mode = !switch_en;
3217
3218 for (i = 0; i < cpsw->port_num; i++) {
3219 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3220
3221 if (!sl_ndev || !netif_running(sl_ndev))
3222 continue;
3223
3224 if_running = true;
3225 }
3226
3227 if (!if_running) {
3228 /* all ndevs are down */
3229 for (i = 0; i < cpsw->port_num; i++) {
3230 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3231 struct am65_cpsw_slave_data *slave;
3232
3233 if (!sl_ndev)
3234 continue;
3235
3236 slave = am65_ndev_to_slave(sl_ndev);
3237 if (switch_en)
3238 slave->port_vlan = cpsw->default_vlan;
3239 else
3240 slave->port_vlan = 0;
3241 }
3242
3243 goto exit;
3244 }
3245
3246 cpsw_ale_control_set(cpsw->ale, 0, ALE_BYPASS, 1);
3247 /* clean up ALE table */
3248 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_CLEAR, 1);
3249 cpsw_ale_control_get(cpsw->ale, HOST_PORT_NUM, ALE_AGEOUT);
3250
3251 if (switch_en) {
3252 dev_info(cpsw->dev, "Enable switch mode\n");
3253
3254 am65_cpsw_init_host_port_switch(cpsw);
3255
3256 for (i = 0; i < cpsw->port_num; i++) {
3257 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3258 struct am65_cpsw_slave_data *slave;
3259 struct am65_cpsw_port *port;
3260
3261 if (!sl_ndev)
3262 continue;
3263
3264 port = am65_ndev_to_port(sl_ndev);
3265 slave = am65_ndev_to_slave(sl_ndev);
3266 slave->port_vlan = cpsw->default_vlan;
3267
3268 if (netif_running(sl_ndev))
3269 am65_cpsw_init_port_switch_ale(port);
3270 }
3271
3272 } else {
3273 dev_info(cpsw->dev, "Disable switch mode\n");
3274
3275 am65_cpsw_init_host_port_emac(cpsw);
3276
3277 for (i = 0; i < cpsw->port_num; i++) {
3278 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3279 struct am65_cpsw_port *port;
3280
3281 if (!sl_ndev)
3282 continue;
3283
3284 port = am65_ndev_to_port(sl_ndev);
3285 port->slave.port_vlan = 0;
3286 if (netif_running(sl_ndev))
3287 am65_cpsw_init_port_emac_ale(port);
3288 }
3289 }
3290 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_BYPASS, 0);
3291 exit:
3292 rtnl_unlock();
3293
3294 return 0;
3295 }
3296
3297 static const struct devlink_param am65_cpsw_devlink_params[] = {
3298 DEVLINK_PARAM_DRIVER(AM65_CPSW_DL_PARAM_SWITCH_MODE, "switch_mode",
3299 DEVLINK_PARAM_TYPE_BOOL,
3300 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
3301 am65_cpsw_dl_switch_mode_get,
3302 am65_cpsw_dl_switch_mode_set, NULL),
3303 };
3304
am65_cpsw_nuss_register_devlink(struct am65_cpsw_common * common)3305 static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common)
3306 {
3307 struct devlink_port_attrs attrs = {};
3308 struct am65_cpsw_devlink *dl_priv;
3309 struct device *dev = common->dev;
3310 struct devlink_port *dl_port;
3311 struct am65_cpsw_port *port;
3312 int ret = 0;
3313 int i;
3314
3315 common->devlink =
3316 devlink_alloc(&am65_cpsw_devlink_ops, sizeof(*dl_priv), dev);
3317 if (!common->devlink)
3318 return -ENOMEM;
3319
3320 dl_priv = devlink_priv(common->devlink);
3321 dl_priv->common = common;
3322
3323 /* Provide devlink hook to switch mode when multiple external ports
3324 * are present NUSS switchdev driver is enabled.
3325 */
3326 if (!AM65_CPSW_IS_CPSW2G(common) &&
3327 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
3328 ret = devlink_params_register(common->devlink,
3329 am65_cpsw_devlink_params,
3330 ARRAY_SIZE(am65_cpsw_devlink_params));
3331 if (ret) {
3332 dev_err(dev, "devlink params reg fail ret:%d\n", ret);
3333 goto dl_unreg;
3334 }
3335 }
3336
3337 for (i = 1; i <= common->port_num; i++) {
3338 port = am65_common_get_port(common, i);
3339 dl_port = &port->devlink_port;
3340
3341 if (port->ndev)
3342 attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
3343 else
3344 attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED;
3345 attrs.phys.port_number = port->port_id;
3346 attrs.switch_id.id_len = sizeof(resource_size_t);
3347 memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len);
3348 devlink_port_attrs_set(dl_port, &attrs);
3349
3350 ret = devlink_port_register(common->devlink, dl_port, port->port_id);
3351 if (ret) {
3352 dev_err(dev, "devlink_port reg fail for port %d, ret:%d\n",
3353 port->port_id, ret);
3354 goto dl_port_unreg;
3355 }
3356 }
3357 devlink_register(common->devlink);
3358 return ret;
3359
3360 dl_port_unreg:
3361 for (i = i - 1; i >= 1; i--) {
3362 port = am65_common_get_port(common, i);
3363 dl_port = &port->devlink_port;
3364
3365 devlink_port_unregister(dl_port);
3366 }
3367 dl_unreg:
3368 devlink_free(common->devlink);
3369 return ret;
3370 }
3371
am65_cpsw_unregister_devlink(struct am65_cpsw_common * common)3372 static void am65_cpsw_unregister_devlink(struct am65_cpsw_common *common)
3373 {
3374 struct devlink_port *dl_port;
3375 struct am65_cpsw_port *port;
3376 int i;
3377
3378 devlink_unregister(common->devlink);
3379
3380 for (i = 1; i <= common->port_num; i++) {
3381 port = am65_common_get_port(common, i);
3382 dl_port = &port->devlink_port;
3383
3384 devlink_port_unregister(dl_port);
3385 }
3386
3387 if (!AM65_CPSW_IS_CPSW2G(common) &&
3388 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
3389 devlink_params_unregister(common->devlink,
3390 am65_cpsw_devlink_params,
3391 ARRAY_SIZE(am65_cpsw_devlink_params));
3392
3393 devlink_free(common->devlink);
3394 }
3395
am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common * common)3396 static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common)
3397 {
3398 struct am65_cpsw_rx_chn *rx_chan = &common->rx_chns;
3399 struct am65_cpsw_tx_chn *tx_chan = common->tx_chns;
3400 struct device *dev = common->dev;
3401 struct am65_cpsw_port *port;
3402 int ret = 0, i;
3403
3404 /* init tx channels */
3405 ret = am65_cpsw_nuss_init_tx_chns(common);
3406 if (ret)
3407 return ret;
3408 ret = am65_cpsw_nuss_init_rx_chns(common);
3409 if (ret)
3410 goto err_remove_tx;
3411
3412 /* The DMA Channels are not guaranteed to be in a clean state.
3413 * Reset and disable them to ensure that they are back to the
3414 * clean state and ready to be used.
3415 */
3416 for (i = 0; i < common->tx_ch_num; i++) {
3417 k3_udma_glue_reset_tx_chn(tx_chan[i].tx_chn, &tx_chan[i],
3418 am65_cpsw_nuss_tx_cleanup);
3419 k3_udma_glue_disable_tx_chn(tx_chan[i].tx_chn);
3420 }
3421
3422 for (i = 0; i < common->rx_ch_num_flows; i++)
3423 k3_udma_glue_reset_rx_chn(rx_chan->rx_chn, i,
3424 rx_chan,
3425 am65_cpsw_nuss_rx_cleanup, !!i);
3426
3427 k3_udma_glue_disable_rx_chn(rx_chan->rx_chn);
3428
3429 ret = am65_cpsw_nuss_register_devlink(common);
3430 if (ret)
3431 goto err_remove_rx;
3432
3433 for (i = 0; i < common->port_num; i++) {
3434 port = &common->ports[i];
3435
3436 if (!port->ndev)
3437 continue;
3438
3439 SET_NETDEV_DEVLINK_PORT(port->ndev, &port->devlink_port);
3440
3441 ret = register_netdev(port->ndev);
3442 if (ret) {
3443 dev_err(dev, "error registering slave net device%i %d\n",
3444 i, ret);
3445 goto err_cleanup_ndev;
3446 }
3447 }
3448
3449 ret = am65_cpsw_register_notifiers(common);
3450 if (ret)
3451 goto err_cleanup_ndev;
3452
3453 /* can't auto unregister ndev using devm_add_action() due to
3454 * devres release sequence in DD core for DMA
3455 */
3456
3457 return 0;
3458
3459 err_cleanup_ndev:
3460 am65_cpsw_nuss_cleanup_ndev(common);
3461 am65_cpsw_unregister_devlink(common);
3462 err_remove_rx:
3463 am65_cpsw_nuss_remove_rx_chns(common);
3464 err_remove_tx:
3465 am65_cpsw_nuss_remove_tx_chns(common);
3466
3467 return ret;
3468 }
3469
am65_cpsw_nuss_update_tx_rx_chns(struct am65_cpsw_common * common,int num_tx,int num_rx)3470 int am65_cpsw_nuss_update_tx_rx_chns(struct am65_cpsw_common *common,
3471 int num_tx, int num_rx)
3472 {
3473 int ret;
3474
3475 am65_cpsw_nuss_remove_tx_chns(common);
3476 am65_cpsw_nuss_remove_rx_chns(common);
3477
3478 common->tx_ch_num = num_tx;
3479 common->rx_ch_num_flows = num_rx;
3480 ret = am65_cpsw_nuss_init_tx_chns(common);
3481 if (ret)
3482 return ret;
3483
3484 ret = am65_cpsw_nuss_init_rx_chns(common);
3485 if (ret)
3486 am65_cpsw_nuss_remove_tx_chns(common);
3487
3488 return ret;
3489 }
3490
3491 struct am65_cpsw_soc_pdata {
3492 u32 quirks_dis;
3493 };
3494
3495 static const struct am65_cpsw_soc_pdata am65x_soc_sr2_0 = {
3496 .quirks_dis = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
3497 };
3498
3499 static const struct soc_device_attribute am65_cpsw_socinfo[] = {
3500 { .family = "AM65X",
3501 .revision = "SR2.0",
3502 .data = &am65x_soc_sr2_0
3503 },
3504 {/* sentinel */}
3505 };
3506
3507 static const struct am65_cpsw_pdata am65x_sr1_0 = {
3508 .quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
3509 .ale_dev_id = "am65x-cpsw2g",
3510 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3511 };
3512
3513 static const struct am65_cpsw_pdata j721e_pdata = {
3514 .quirks = 0,
3515 .ale_dev_id = "am65x-cpsw2g",
3516 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3517 };
3518
3519 static const struct am65_cpsw_pdata am64x_cpswxg_pdata = {
3520 .quirks = AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ,
3521 .ale_dev_id = "am64-cpswxg",
3522 .fdqring_mode = K3_RINGACC_RING_MODE_RING,
3523 };
3524
3525 static const struct am65_cpsw_pdata j7200_cpswxg_pdata = {
3526 .quirks = 0,
3527 .ale_dev_id = "am64-cpswxg",
3528 .fdqring_mode = K3_RINGACC_RING_MODE_RING,
3529 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII) |
3530 BIT(PHY_INTERFACE_MODE_USXGMII),
3531 };
3532
3533 static const struct am65_cpsw_pdata j721e_cpswxg_pdata = {
3534 .quirks = 0,
3535 .ale_dev_id = "am64-cpswxg",
3536 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3537 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
3538 };
3539
3540 static const struct am65_cpsw_pdata j784s4_cpswxg_pdata = {
3541 .quirks = 0,
3542 .ale_dev_id = "am64-cpswxg",
3543 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3544 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII) |
3545 BIT(PHY_INTERFACE_MODE_USXGMII),
3546 };
3547
3548 static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
3549 { .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
3550 { .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
3551 { .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
3552 { .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
3553 { .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata},
3554 { .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata},
3555 { /* sentinel */ },
3556 };
3557 MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
3558
am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common * common)3559 static void am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common *common)
3560 {
3561 const struct soc_device_attribute *soc;
3562
3563 soc = soc_device_match(am65_cpsw_socinfo);
3564 if (soc && soc->data) {
3565 const struct am65_cpsw_soc_pdata *socdata = soc->data;
3566
3567 /* disable quirks */
3568 common->pdata.quirks &= ~socdata->quirks_dis;
3569 }
3570 }
3571
am65_cpsw_nuss_probe(struct platform_device * pdev)3572 static int am65_cpsw_nuss_probe(struct platform_device *pdev)
3573 {
3574 struct cpsw_ale_params ale_params = { 0 };
3575 const struct of_device_id *of_id;
3576 struct device *dev = &pdev->dev;
3577 struct am65_cpsw_common *common;
3578 struct device_node *node;
3579 struct resource *res;
3580 struct clk *clk;
3581 int ale_entries;
3582 __be64 id_temp;
3583 int ret, i;
3584
3585 common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
3586 if (!common)
3587 return -ENOMEM;
3588 common->dev = dev;
3589
3590 of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev);
3591 if (!of_id)
3592 return -EINVAL;
3593 common->pdata = *(const struct am65_cpsw_pdata *)of_id->data;
3594
3595 am65_cpsw_nuss_apply_socinfo(common);
3596
3597 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss");
3598 common->ss_base = devm_ioremap_resource(&pdev->dev, res);
3599 if (IS_ERR(common->ss_base))
3600 return PTR_ERR(common->ss_base);
3601 common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;
3602 /* Use device's physical base address as switch id */
3603 id_temp = cpu_to_be64(res->start);
3604 memcpy(common->switch_id, &id_temp, sizeof(res->start));
3605
3606 node = of_get_child_by_name(dev->of_node, "ethernet-ports");
3607 if (!node)
3608 return -ENOENT;
3609 common->port_num = of_get_child_count(node);
3610 of_node_put(node);
3611 if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS)
3612 return -ENOENT;
3613
3614 common->rx_flow_id_base = -1;
3615 init_completion(&common->tdown_complete);
3616 common->tx_ch_num = AM65_CPSW_DEFAULT_TX_CHNS;
3617 common->rx_ch_num_flows = AM65_CPSW_DEFAULT_RX_CHN_FLOWS;
3618 common->pf_p0_rx_ptype_rrobin = true;
3619 common->default_vlan = 1;
3620
3621 common->ports = devm_kcalloc(dev, common->port_num,
3622 sizeof(*common->ports),
3623 GFP_KERNEL);
3624 if (!common->ports)
3625 return -ENOMEM;
3626
3627 clk = devm_clk_get(dev, "fck");
3628 if (IS_ERR(clk))
3629 return dev_err_probe(dev, PTR_ERR(clk), "getting fck clock\n");
3630 common->bus_freq = clk_get_rate(clk);
3631
3632 pm_runtime_enable(dev);
3633 ret = pm_runtime_resume_and_get(dev);
3634 if (ret < 0) {
3635 pm_runtime_disable(dev);
3636 return ret;
3637 }
3638
3639 node = of_get_child_by_name(dev->of_node, "mdio");
3640 if (!node) {
3641 dev_warn(dev, "MDIO node not found\n");
3642 } else if (of_device_is_available(node)) {
3643 struct platform_device *mdio_pdev;
3644
3645 mdio_pdev = of_platform_device_create(node, NULL, dev);
3646 if (!mdio_pdev) {
3647 ret = -ENODEV;
3648 goto err_pm_clear;
3649 }
3650
3651 common->mdio_dev = &mdio_pdev->dev;
3652 }
3653 of_node_put(node);
3654
3655 am65_cpsw_nuss_get_ver(common);
3656
3657 ret = am65_cpsw_nuss_init_host_p(common);
3658 if (ret)
3659 goto err_of_clear;
3660
3661 ret = am65_cpsw_nuss_init_slave_ports(common);
3662 if (ret)
3663 goto err_of_clear;
3664
3665 /* init common data */
3666 ale_params.dev = dev;
3667 ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT;
3668 ale_params.ale_ports = common->port_num + 1;
3669 ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;
3670 ale_params.dev_id = common->pdata.ale_dev_id;
3671 ale_params.bus_freq = common->bus_freq;
3672
3673 common->ale = cpsw_ale_create(&ale_params);
3674 if (IS_ERR(common->ale)) {
3675 dev_err(dev, "error initializing ale engine\n");
3676 ret = PTR_ERR(common->ale);
3677 goto err_of_clear;
3678 }
3679
3680 ale_entries = common->ale->params.ale_entries;
3681 common->ale_context = devm_kzalloc(dev,
3682 ale_entries * ALE_ENTRY_WORDS * sizeof(u32),
3683 GFP_KERNEL);
3684 ret = am65_cpsw_init_cpts(common);
3685 if (ret)
3686 goto err_of_clear;
3687
3688 /* init ports */
3689 for (i = 0; i < common->port_num; i++)
3690 am65_cpsw_nuss_slave_disable_unused(&common->ports[i]);
3691
3692 dev_set_drvdata(dev, common);
3693
3694 common->is_emac_mode = true;
3695
3696 ret = am65_cpsw_nuss_init_ndevs(common);
3697 if (ret)
3698 goto err_ndevs_clear;
3699
3700 ret = am65_cpsw_nuss_register_ndevs(common);
3701 if (ret)
3702 goto err_ndevs_clear;
3703
3704 pm_runtime_put(dev);
3705 return 0;
3706
3707 err_ndevs_clear:
3708 am65_cpsw_nuss_cleanup_ndev(common);
3709 am65_cpsw_nuss_phylink_cleanup(common);
3710 am65_cpts_release(common->cpts);
3711 err_of_clear:
3712 if (common->mdio_dev)
3713 of_platform_device_destroy(common->mdio_dev, NULL);
3714 err_pm_clear:
3715 pm_runtime_put_sync(dev);
3716 pm_runtime_disable(dev);
3717 return ret;
3718 }
3719
am65_cpsw_nuss_remove(struct platform_device * pdev)3720 static void am65_cpsw_nuss_remove(struct platform_device *pdev)
3721 {
3722 struct device *dev = &pdev->dev;
3723 struct am65_cpsw_common *common;
3724 int ret;
3725
3726 common = dev_get_drvdata(dev);
3727
3728 ret = pm_runtime_resume_and_get(&pdev->dev);
3729 if (ret < 0) {
3730 /* Note, if this error path is taken, we're leaking some
3731 * resources.
3732 */
3733 dev_err(&pdev->dev, "Failed to resume device (%pe)\n",
3734 ERR_PTR(ret));
3735 return;
3736 }
3737
3738 am65_cpsw_unregister_notifiers(common);
3739
3740 /* must unregister ndevs here because DD release_driver routine calls
3741 * dma_deconfigure(dev) before devres_release_all(dev)
3742 */
3743 am65_cpsw_nuss_cleanup_ndev(common);
3744 am65_cpsw_unregister_devlink(common);
3745 am65_cpsw_nuss_remove_rx_chns(common);
3746 am65_cpsw_nuss_remove_tx_chns(common);
3747 am65_cpsw_nuss_phylink_cleanup(common);
3748 am65_cpts_release(common->cpts);
3749 am65_cpsw_disable_serdes_phy(common);
3750
3751 if (common->mdio_dev)
3752 of_platform_device_destroy(common->mdio_dev, NULL);
3753
3754 pm_runtime_put_sync(&pdev->dev);
3755 pm_runtime_disable(&pdev->dev);
3756 }
3757
am65_cpsw_nuss_suspend(struct device * dev)3758 static int am65_cpsw_nuss_suspend(struct device *dev)
3759 {
3760 struct am65_cpsw_common *common = dev_get_drvdata(dev);
3761 struct am65_cpsw_host *host_p = am65_common_get_host(common);
3762 struct am65_cpsw_port *port;
3763 struct net_device *ndev;
3764 int i, ret;
3765
3766 cpsw_ale_dump(common->ale, common->ale_context);
3767 host_p->vid_context = readl(host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3768 for (i = 0; i < common->port_num; i++) {
3769 port = &common->ports[i];
3770 ndev = port->ndev;
3771
3772 if (!ndev)
3773 continue;
3774
3775 port->vid_context = readl(port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3776 netif_device_detach(ndev);
3777 if (netif_running(ndev)) {
3778 rtnl_lock();
3779 ret = am65_cpsw_nuss_ndo_slave_stop(ndev);
3780 rtnl_unlock();
3781 if (ret < 0) {
3782 netdev_err(ndev, "failed to stop: %d", ret);
3783 return ret;
3784 }
3785 }
3786 }
3787
3788 am65_cpts_suspend(common->cpts);
3789
3790 am65_cpsw_nuss_remove_rx_chns(common);
3791 am65_cpsw_nuss_remove_tx_chns(common);
3792
3793 return 0;
3794 }
3795
am65_cpsw_nuss_resume(struct device * dev)3796 static int am65_cpsw_nuss_resume(struct device *dev)
3797 {
3798 struct am65_cpsw_common *common = dev_get_drvdata(dev);
3799 struct am65_cpsw_host *host_p = am65_common_get_host(common);
3800 struct am65_cpsw_port *port;
3801 struct net_device *ndev;
3802 int i, ret;
3803
3804 ret = am65_cpsw_nuss_init_tx_chns(common);
3805 if (ret)
3806 return ret;
3807 ret = am65_cpsw_nuss_init_rx_chns(common);
3808 if (ret) {
3809 am65_cpsw_nuss_remove_tx_chns(common);
3810 return ret;
3811 }
3812
3813 /* If RX IRQ was disabled before suspend, keep it disabled */
3814 for (i = 0; i < common->rx_ch_num_flows; i++) {
3815 if (common->rx_chns.flows[i].irq_disabled)
3816 disable_irq(common->rx_chns.flows[i].irq);
3817 }
3818
3819 am65_cpts_resume(common->cpts);
3820
3821 for (i = 0; i < common->port_num; i++) {
3822 port = &common->ports[i];
3823 ndev = port->ndev;
3824
3825 if (!ndev)
3826 continue;
3827
3828 if (netif_running(ndev)) {
3829 rtnl_lock();
3830 ret = am65_cpsw_nuss_ndo_slave_open(ndev);
3831 rtnl_unlock();
3832 if (ret < 0) {
3833 netdev_err(ndev, "failed to start: %d", ret);
3834 return ret;
3835 }
3836 }
3837
3838 netif_device_attach(ndev);
3839 writel(port->vid_context, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3840 }
3841
3842 writel(host_p->vid_context, host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3843 cpsw_ale_restore(common->ale, common->ale_context);
3844
3845 return 0;
3846 }
3847
3848 static const struct dev_pm_ops am65_cpsw_nuss_dev_pm_ops = {
3849 SYSTEM_SLEEP_PM_OPS(am65_cpsw_nuss_suspend, am65_cpsw_nuss_resume)
3850 };
3851
3852 static struct platform_driver am65_cpsw_nuss_driver = {
3853 .driver = {
3854 .name = AM65_CPSW_DRV_NAME,
3855 .of_match_table = am65_cpsw_nuss_of_mtable,
3856 .pm = &am65_cpsw_nuss_dev_pm_ops,
3857 },
3858 .probe = am65_cpsw_nuss_probe,
3859 .remove = am65_cpsw_nuss_remove,
3860 };
3861
3862 module_platform_driver(am65_cpsw_nuss_driver);
3863
3864 MODULE_LICENSE("GPL v2");
3865 MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
3866 MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");
3867