1 // SPDX-License-Identifier: GPL-2.0
2 /* Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver
3 *
4 * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
5 *
6 */
7
8 #include <linux/bpf_trace.h>
9 #include <linux/clk.h>
10 #include <linux/etherdevice.h>
11 #include <linux/if_vlan.h>
12 #include <linux/interrupt.h>
13 #include <linux/irqdomain.h>
14 #include <linux/kernel.h>
15 #include <linux/kmemleak.h>
16 #include <linux/module.h>
17 #include <linux/netdevice.h>
18 #include <linux/net_tstamp.h>
19 #include <linux/of.h>
20 #include <linux/of_mdio.h>
21 #include <linux/of_net.h>
22 #include <linux/of_device.h>
23 #include <linux/of_platform.h>
24 #include <linux/phylink.h>
25 #include <linux/phy/phy.h>
26 #include <linux/platform_device.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/regmap.h>
29 #include <linux/rtnetlink.h>
30 #include <linux/mfd/syscon.h>
31 #include <linux/sys_soc.h>
32 #include <linux/dma/ti-cppi5.h>
33 #include <linux/dma/k3-udma-glue.h>
34 #include <net/page_pool/helpers.h>
35 #include <net/dsa.h>
36 #include <net/switchdev.h>
37
38 #include "cpsw_ale.h"
39 #include "cpsw_sl.h"
40 #include "am65-cpsw-nuss.h"
41 #include "am65-cpsw-switchdev.h"
42 #include "k3-cppi-desc-pool.h"
43 #include "am65-cpts.h"
44
45 #define AM65_CPSW_SS_BASE 0x0
46 #define AM65_CPSW_SGMII_BASE 0x100
47 #define AM65_CPSW_XGMII_BASE 0x2100
48 #define AM65_CPSW_CPSW_NU_BASE 0x20000
49 #define AM65_CPSW_NU_PORTS_BASE 0x1000
50 #define AM65_CPSW_NU_FRAM_BASE 0x12000
51 #define AM65_CPSW_NU_STATS_BASE 0x1a000
52 #define AM65_CPSW_NU_ALE_BASE 0x1e000
53 #define AM65_CPSW_NU_CPTS_BASE 0x1d000
54
55 #define AM65_CPSW_NU_PORTS_OFFSET 0x1000
56 #define AM65_CPSW_NU_STATS_PORT_OFFSET 0x200
57 #define AM65_CPSW_NU_FRAM_PORT_OFFSET 0x200
58
59 #define AM65_CPSW_MAX_PORTS 8
60
61 #define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN
62 #define AM65_CPSW_MAX_PACKET_SIZE 2024
63
64 #define AM65_CPSW_REG_CTL 0x004
65 #define AM65_CPSW_REG_STAT_PORT_EN 0x014
66 #define AM65_CPSW_REG_PTYPE 0x018
67
68 #define AM65_CPSW_P0_REG_CTL 0x004
69 #define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET 0x008
70
71 #define AM65_CPSW_PORT_REG_PRI_CTL 0x01c
72 #define AM65_CPSW_PORT_REG_RX_PRI_MAP 0x020
73 #define AM65_CPSW_PORT_REG_RX_MAXLEN 0x024
74
75 #define AM65_CPSW_PORTN_REG_CTL 0x004
76 #define AM65_CPSW_PORTN_REG_DSCP_MAP 0x120
77 #define AM65_CPSW_PORTN_REG_SA_L 0x308
78 #define AM65_CPSW_PORTN_REG_SA_H 0x30c
79 #define AM65_CPSW_PORTN_REG_TS_CTL 0x310
80 #define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG 0x314
81 #define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG 0x318
82 #define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 0x31C
83
84 #define AM65_CPSW_SGMII_CONTROL_REG 0x010
85 #define AM65_CPSW_SGMII_MR_ADV_ABILITY_REG 0x018
86 #define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE BIT(0)
87
88 #define AM65_CPSW_CTL_VLAN_AWARE BIT(1)
89 #define AM65_CPSW_CTL_P0_ENABLE BIT(2)
90 #define AM65_CPSW_CTL_P0_TX_CRC_REMOVE BIT(13)
91 #define AM65_CPSW_CTL_P0_RX_PAD BIT(14)
92
93 /* AM65_CPSW_P0_REG_CTL */
94 #define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN BIT(0)
95 #define AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN BIT(16)
96
97 /* AM65_CPSW_PORT_REG_PRI_CTL */
98 #define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN BIT(8)
99
100 /* AM65_CPSW_PN_REG_CTL */
101 #define AM65_CPSW_PN_REG_CTL_DSCP_IPV4_EN BIT(1)
102 #define AM65_CPSW_PN_REG_CTL_DSCP_IPV6_EN BIT(2)
103
104 /* AM65_CPSW_PN_TS_CTL register fields */
105 #define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN BIT(4)
106 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN BIT(5)
107 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN BIT(6)
108 #define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN BIT(7)
109 #define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN BIT(10)
110 #define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN BIT(11)
111 #define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT 16
112
113 #define AM65_CPSW_PN_TS_CTL_RX_ANX_F_EN BIT(0)
114 #define AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN BIT(1)
115 #define AM65_CPSW_PN_TS_CTL_RX_VLAN_LT2_EN BIT(2)
116 #define AM65_CPSW_PN_TS_CTL_RX_ANX_D_EN BIT(3)
117 #define AM65_CPSW_PN_TS_CTL_RX_ANX_E_EN BIT(9)
118
119 /* AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG register fields */
120 #define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT 16
121
122 /* AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 */
123 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 BIT(16)
124 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 BIT(17)
125 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 BIT(18)
126 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 BIT(19)
127 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 BIT(20)
128 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 BIT(21)
129 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 BIT(22)
130 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23)
131
132 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
133 #define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
134
135 #define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e)
136
137 #define AM65_CPSW_TS_TX_ANX_ALL_EN \
138 (AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN | \
139 AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN | \
140 AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN)
141
142 #define AM65_CPSW_TS_RX_ANX_ALL_EN \
143 (AM65_CPSW_PN_TS_CTL_RX_ANX_D_EN | \
144 AM65_CPSW_PN_TS_CTL_RX_ANX_E_EN | \
145 AM65_CPSW_PN_TS_CTL_RX_ANX_F_EN)
146
147 #define AM65_CPSW_ALE_AGEOUT_DEFAULT 30
148 /* Number of TX/RX descriptors per channel/flow */
149 #define AM65_CPSW_MAX_TX_DESC 500
150 #define AM65_CPSW_MAX_RX_DESC 500
151
152 #define AM65_CPSW_NAV_PS_DATA_SIZE 16
153 #define AM65_CPSW_NAV_SW_DATA_SIZE 16
154
155 #define AM65_CPSW_DEBUG (NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | \
156 NETIF_MSG_IFUP | NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \
157 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
158
159 #define AM65_CPSW_DEFAULT_TX_CHNS 8
160 #define AM65_CPSW_DEFAULT_RX_CHN_FLOWS 1
161
162 /* CPPI streaming packet interface */
163 #define AM65_CPSW_CPPI_TX_FLOW_ID 0x3FFF
164 #define AM65_CPSW_CPPI_TX_PKT_TYPE 0x7
165
166 /* XDP */
167 #define AM65_CPSW_XDP_CONSUMED BIT(1)
168 #define AM65_CPSW_XDP_REDIRECT BIT(0)
169 #define AM65_CPSW_XDP_PASS 0
170
171 /* Include headroom compatible with both skb and xdpf */
172 #define AM65_CPSW_HEADROOM_NA (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
173 #define AM65_CPSW_HEADROOM ALIGN(AM65_CPSW_HEADROOM_NA, sizeof(long))
174
am65_cpsw_port_set_sl_mac(struct am65_cpsw_port * slave,const u8 * dev_addr)175 static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
176 const u8 *dev_addr)
177 {
178 u32 mac_hi = (dev_addr[0] << 0) | (dev_addr[1] << 8) |
179 (dev_addr[2] << 16) | (dev_addr[3] << 24);
180 u32 mac_lo = (dev_addr[4] << 0) | (dev_addr[5] << 8);
181
182 writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H);
183 writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L);
184 }
185
186 #define AM65_CPSW_DSCP_MAX GENMASK(5, 0)
187 #define AM65_CPSW_PRI_MAX GENMASK(2, 0)
188 #define AM65_CPSW_DSCP_PRI_PER_REG 8
189 #define AM65_CPSW_DSCP_PRI_SIZE 4 /* in bits */
am65_cpsw_port_set_dscp_map(struct am65_cpsw_port * slave,u8 dscp,u8 pri)190 static int am65_cpsw_port_set_dscp_map(struct am65_cpsw_port *slave, u8 dscp, u8 pri)
191 {
192 int reg_ofs;
193 int bit_ofs;
194 u32 val;
195
196 if (dscp > AM65_CPSW_DSCP_MAX)
197 return -EINVAL;
198
199 if (pri > AM65_CPSW_PRI_MAX)
200 return -EINVAL;
201
202 /* 32-bit register offset to this dscp */
203 reg_ofs = (dscp / AM65_CPSW_DSCP_PRI_PER_REG) * 4;
204 /* bit field offset to this dscp */
205 bit_ofs = AM65_CPSW_DSCP_PRI_SIZE * (dscp % AM65_CPSW_DSCP_PRI_PER_REG);
206
207 val = readl(slave->port_base + AM65_CPSW_PORTN_REG_DSCP_MAP + reg_ofs);
208 val &= ~(AM65_CPSW_PRI_MAX << bit_ofs); /* clear */
209 val |= pri << bit_ofs; /* set */
210 writel(val, slave->port_base + AM65_CPSW_PORTN_REG_DSCP_MAP + reg_ofs);
211
212 return 0;
213 }
214
am65_cpsw_port_enable_dscp_map(struct am65_cpsw_port * slave)215 static void am65_cpsw_port_enable_dscp_map(struct am65_cpsw_port *slave)
216 {
217 int dscp, pri;
218 u32 val;
219
220 /* Default DSCP to User Priority mapping as per:
221 * https://datatracker.ietf.org/doc/html/rfc8325#section-4.3
222 * and
223 * https://datatracker.ietf.org/doc/html/rfc8622#section-11
224 */
225 for (dscp = 0; dscp <= AM65_CPSW_DSCP_MAX; dscp++) {
226 switch (dscp) {
227 case 56: /* CS7 */
228 case 48: /* CS6 */
229 pri = 7;
230 break;
231 case 46: /* EF */
232 case 44: /* VA */
233 pri = 6;
234 break;
235 case 40: /* CS5 */
236 pri = 5;
237 break;
238 case 34: /* AF41 */
239 case 36: /* AF42 */
240 case 38: /* AF43 */
241 case 32: /* CS4 */
242 case 26: /* AF31 */
243 case 28: /* AF32 */
244 case 30: /* AF33 */
245 case 24: /* CS3 */
246 pri = 4;
247 break;
248 case 18: /* AF21 */
249 case 20: /* AF22 */
250 case 22: /* AF23 */
251 pri = 3;
252 break;
253 case 16: /* CS2 */
254 case 10: /* AF11 */
255 case 12: /* AF12 */
256 case 14: /* AF13 */
257 case 0: /* DF */
258 pri = 0;
259 break;
260 case 8: /* CS1 */
261 case 1: /* LE */
262 pri = 1;
263 break;
264 default:
265 pri = 0;
266 break;
267 }
268
269 am65_cpsw_port_set_dscp_map(slave, dscp, pri);
270 }
271
272 /* enable port IPV4 and IPV6 DSCP for this port */
273 val = readl(slave->port_base + AM65_CPSW_PORTN_REG_CTL);
274 val |= AM65_CPSW_PN_REG_CTL_DSCP_IPV4_EN |
275 AM65_CPSW_PN_REG_CTL_DSCP_IPV6_EN;
276 writel(val, slave->port_base + AM65_CPSW_PORTN_REG_CTL);
277 }
278
am65_cpsw_sl_ctl_reset(struct am65_cpsw_port * port)279 static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port)
280 {
281 cpsw_sl_reset(port->slave.mac_sl, 100);
282 /* Max length register has to be restored after MAC SL reset */
283 writel(AM65_CPSW_MAX_PACKET_SIZE,
284 port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
285 }
286
am65_cpsw_nuss_get_ver(struct am65_cpsw_common * common)287 static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common)
288 {
289 common->nuss_ver = readl(common->ss_base);
290 common->cpsw_ver = readl(common->cpsw_base);
291 dev_info(common->dev,
292 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n",
293 common->nuss_ver,
294 common->cpsw_ver,
295 common->port_num + 1,
296 common->pdata.quirks);
297 }
298
am65_cpsw_nuss_ndo_slave_add_vid(struct net_device * ndev,__be16 proto,u16 vid)299 static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
300 __be16 proto, u16 vid)
301 {
302 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
303 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
304 u32 port_mask, unreg_mcast = 0;
305 int ret;
306
307 if (!common->is_emac_mode)
308 return 0;
309
310 if (!netif_running(ndev) || !vid)
311 return 0;
312
313 ret = pm_runtime_resume_and_get(common->dev);
314 if (ret < 0)
315 return ret;
316
317 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
318 if (!vid)
319 unreg_mcast = port_mask;
320 dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
321 ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask,
322 unreg_mcast, port_mask, 0);
323
324 pm_runtime_put(common->dev);
325 return ret;
326 }
327
am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device * ndev,__be16 proto,u16 vid)328 static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev,
329 __be16 proto, u16 vid)
330 {
331 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
332 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
333 int ret;
334
335 if (!common->is_emac_mode)
336 return 0;
337
338 if (!netif_running(ndev) || !vid)
339 return 0;
340
341 ret = pm_runtime_resume_and_get(common->dev);
342 if (ret < 0)
343 return ret;
344
345 dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);
346 ret = cpsw_ale_del_vlan(common->ale, vid,
347 BIT(port->port_id) | ALE_PORT_HOST);
348
349 pm_runtime_put(common->dev);
350 return ret;
351 }
352
am65_cpsw_slave_set_promisc(struct am65_cpsw_port * port,bool promisc)353 static void am65_cpsw_slave_set_promisc(struct am65_cpsw_port *port,
354 bool promisc)
355 {
356 struct am65_cpsw_common *common = port->common;
357
358 if (promisc && !common->is_emac_mode) {
359 dev_dbg(common->dev, "promisc mode requested in switch mode");
360 return;
361 }
362
363 if (promisc) {
364 /* Enable promiscuous mode */
365 cpsw_ale_control_set(common->ale, port->port_id,
366 ALE_PORT_MACONLY_CAF, 1);
367 dev_dbg(common->dev, "promisc enabled\n");
368 } else {
369 /* Disable promiscuous mode */
370 cpsw_ale_control_set(common->ale, port->port_id,
371 ALE_PORT_MACONLY_CAF, 0);
372 dev_dbg(common->dev, "promisc disabled\n");
373 }
374 }
375
am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device * ndev)376 static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev)
377 {
378 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
379 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
380 u32 port_mask;
381 bool promisc;
382
383 promisc = !!(ndev->flags & IFF_PROMISC);
384 am65_cpsw_slave_set_promisc(port, promisc);
385
386 if (promisc)
387 return;
388
389 /* Restore allmulti on vlans if necessary */
390 cpsw_ale_set_allmulti(common->ale,
391 ndev->flags & IFF_ALLMULTI, port->port_id);
392
393 port_mask = ALE_PORT_HOST;
394 /* Clear all mcast from ALE */
395 cpsw_ale_flush_multicast(common->ale, port_mask, -1);
396
397 if (!netdev_mc_empty(ndev)) {
398 struct netdev_hw_addr *ha;
399
400 /* program multicast address list into ALE register */
401 netdev_for_each_mc_addr(ha, ndev) {
402 cpsw_ale_add_mcast(common->ale, ha->addr,
403 port_mask, 0, 0, 0);
404 }
405 }
406 }
407
am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device * ndev,unsigned int txqueue)408 static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev,
409 unsigned int txqueue)
410 {
411 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
412 struct am65_cpsw_tx_chn *tx_chn;
413 struct netdev_queue *netif_txq;
414 unsigned long trans_start;
415
416 netif_txq = netdev_get_tx_queue(ndev, txqueue);
417 tx_chn = &common->tx_chns[txqueue];
418 trans_start = READ_ONCE(netif_txq->trans_start);
419
420 netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
421 txqueue,
422 netif_tx_queue_stopped(netif_txq),
423 jiffies_to_msecs(jiffies - trans_start),
424 netdev_queue_dql_avail(netif_txq),
425 k3_cppi_desc_pool_avail(tx_chn->desc_pool));
426
427 if (netif_tx_queue_stopped(netif_txq)) {
428 /* try recover if stopped by us */
429 txq_trans_update(netif_txq);
430 netif_tx_wake_queue(netif_txq);
431 }
432 }
433
am65_cpsw_nuss_rx_push(struct am65_cpsw_common * common,struct page * page,u32 flow_idx)434 static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
435 struct page *page, u32 flow_idx)
436 {
437 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
438 struct cppi5_host_desc_t *desc_rx;
439 struct device *dev = common->dev;
440 struct am65_cpsw_swdata *swdata;
441 dma_addr_t desc_dma;
442 dma_addr_t buf_dma;
443
444 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
445 if (!desc_rx) {
446 dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
447 return -ENOMEM;
448 }
449 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
450
451 buf_dma = dma_map_single(rx_chn->dma_dev,
452 page_address(page) + AM65_CPSW_HEADROOM,
453 AM65_CPSW_MAX_PACKET_SIZE, DMA_FROM_DEVICE);
454 if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
455 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
456 dev_err(dev, "Failed to map rx buffer\n");
457 return -EINVAL;
458 }
459
460 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
461 AM65_CPSW_NAV_PS_DATA_SIZE);
462 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
463 cppi5_hdesc_attach_buf(desc_rx, buf_dma, AM65_CPSW_MAX_PACKET_SIZE,
464 buf_dma, AM65_CPSW_MAX_PACKET_SIZE);
465 swdata = cppi5_hdesc_get_swdata(desc_rx);
466 swdata->page = page;
467 swdata->flow_id = flow_idx;
468
469 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, flow_idx,
470 desc_rx, desc_dma);
471 }
472
am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common * common)473 void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common)
474 {
475 struct am65_cpsw_host *host_p = am65_common_get_host(common);
476 u32 val, pri_map;
477
478 /* P0 set Receive Priority Type */
479 val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
480
481 if (common->pf_p0_rx_ptype_rrobin) {
482 val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
483 /* Enet Ports fifos works in fixed priority mode only, so
484 * reset P0_Rx_Pri_Map so all packet will go in Enet fifo 0
485 */
486 pri_map = 0x0;
487 } else {
488 val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
489 /* restore P0_Rx_Pri_Map */
490 pri_map = 0x76543210;
491 }
492
493 writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);
494 writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
495 }
496
497 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common);
498 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
499 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
500 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
501 static inline void am65_cpsw_put_page(struct am65_cpsw_rx_flow *flow,
502 struct page *page,
503 bool allow_direct);
504 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
505 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
506
am65_cpsw_destroy_rxq(struct am65_cpsw_common * common,int id)507 static void am65_cpsw_destroy_rxq(struct am65_cpsw_common *common, int id)
508 {
509 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
510 struct am65_cpsw_rx_flow *flow;
511 struct xdp_rxq_info *rxq;
512 int port;
513
514 flow = &rx_chn->flows[id];
515 napi_disable(&flow->napi_rx);
516 hrtimer_cancel(&flow->rx_hrtimer);
517 k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, id, rx_chn,
518 am65_cpsw_nuss_rx_cleanup, !!id);
519
520 for (port = 0; port < common->port_num; port++) {
521 if (!common->ports[port].ndev)
522 continue;
523
524 rxq = &common->ports[port].xdp_rxq[id];
525
526 if (xdp_rxq_info_is_reg(rxq))
527 xdp_rxq_info_unreg(rxq);
528 }
529
530 if (flow->page_pool) {
531 page_pool_destroy(flow->page_pool);
532 flow->page_pool = NULL;
533 }
534 }
535
am65_cpsw_destroy_rxqs(struct am65_cpsw_common * common)536 static void am65_cpsw_destroy_rxqs(struct am65_cpsw_common *common)
537 {
538 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
539 int id;
540
541 reinit_completion(&common->tdown_complete);
542 k3_udma_glue_tdown_rx_chn(rx_chn->rx_chn, true);
543
544 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) {
545 id = wait_for_completion_timeout(&common->tdown_complete, msecs_to_jiffies(1000));
546 if (!id)
547 dev_err(common->dev, "rx teardown timeout\n");
548 }
549
550 for (id = common->rx_ch_num_flows - 1; id >= 0; id--)
551 am65_cpsw_destroy_rxq(common, id);
552
553 k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
554 }
555
am65_cpsw_create_rxq(struct am65_cpsw_common * common,int id)556 static int am65_cpsw_create_rxq(struct am65_cpsw_common *common, int id)
557 {
558 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
559 struct page_pool_params pp_params = {
560 .flags = PP_FLAG_DMA_MAP,
561 .order = 0,
562 .pool_size = AM65_CPSW_MAX_RX_DESC,
563 .nid = dev_to_node(common->dev),
564 .dev = common->dev,
565 .dma_dir = DMA_BIDIRECTIONAL,
566 /* .napi set dynamically */
567 };
568 struct am65_cpsw_rx_flow *flow;
569 struct xdp_rxq_info *rxq;
570 struct page_pool *pool;
571 struct page *page;
572 int port, ret, i;
573
574 flow = &rx_chn->flows[id];
575 pp_params.napi = &flow->napi_rx;
576 pool = page_pool_create(&pp_params);
577 if (IS_ERR(pool)) {
578 ret = PTR_ERR(pool);
579 return ret;
580 }
581
582 flow->page_pool = pool;
583
584 /* using same page pool is allowed as no running rx handlers
585 * simultaneously for both ndevs
586 */
587 for (port = 0; port < common->port_num; port++) {
588 if (!common->ports[port].ndev)
589 /* FIXME should we BUG here? */
590 continue;
591
592 rxq = &common->ports[port].xdp_rxq[id];
593 ret = xdp_rxq_info_reg(rxq, common->ports[port].ndev,
594 id, flow->napi_rx.napi_id);
595 if (ret)
596 goto err;
597
598 ret = xdp_rxq_info_reg_mem_model(rxq,
599 MEM_TYPE_PAGE_POOL,
600 pool);
601 if (ret)
602 goto err;
603 }
604
605 for (i = 0; i < AM65_CPSW_MAX_RX_DESC; i++) {
606 page = page_pool_dev_alloc_pages(flow->page_pool);
607 if (!page) {
608 dev_err(common->dev, "cannot allocate page in flow %d\n",
609 id);
610 ret = -ENOMEM;
611 goto err;
612 }
613
614 ret = am65_cpsw_nuss_rx_push(common, page, id);
615 if (ret < 0) {
616 dev_err(common->dev,
617 "cannot submit page to rx channel flow %d, error %d\n",
618 id, ret);
619 am65_cpsw_put_page(flow, page, false);
620 goto err;
621 }
622 }
623
624 napi_enable(&flow->napi_rx);
625 return 0;
626
627 err:
628 am65_cpsw_destroy_rxq(common, id);
629 return ret;
630 }
631
am65_cpsw_create_rxqs(struct am65_cpsw_common * common)632 static int am65_cpsw_create_rxqs(struct am65_cpsw_common *common)
633 {
634 int id, ret;
635
636 for (id = 0; id < common->rx_ch_num_flows; id++) {
637 ret = am65_cpsw_create_rxq(common, id);
638 if (ret) {
639 dev_err(common->dev, "couldn't create rxq %d: %d\n",
640 id, ret);
641 goto err;
642 }
643 }
644
645 ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);
646 if (ret) {
647 dev_err(common->dev, "couldn't enable rx chn: %d\n", ret);
648 goto err;
649 }
650
651 return 0;
652
653 err:
654 for (--id; id >= 0; id--)
655 am65_cpsw_destroy_rxq(common, id);
656
657 return ret;
658 }
659
am65_cpsw_destroy_txq(struct am65_cpsw_common * common,int id)660 static void am65_cpsw_destroy_txq(struct am65_cpsw_common *common, int id)
661 {
662 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];
663
664 napi_disable(&tx_chn->napi_tx);
665 hrtimer_cancel(&tx_chn->tx_hrtimer);
666 k3_udma_glue_reset_tx_chn(tx_chn->tx_chn, tx_chn,
667 am65_cpsw_nuss_tx_cleanup);
668 k3_udma_glue_disable_tx_chn(tx_chn->tx_chn);
669 }
670
am65_cpsw_destroy_txqs(struct am65_cpsw_common * common)671 static void am65_cpsw_destroy_txqs(struct am65_cpsw_common *common)
672 {
673 struct am65_cpsw_tx_chn *tx_chn = common->tx_chns;
674 int id;
675
676 /* shutdown tx channels */
677 atomic_set(&common->tdown_cnt, common->tx_ch_num);
678 /* ensure new tdown_cnt value is visible */
679 smp_mb__after_atomic();
680 reinit_completion(&common->tdown_complete);
681
682 for (id = 0; id < common->tx_ch_num; id++)
683 k3_udma_glue_tdown_tx_chn(tx_chn[id].tx_chn, false);
684
685 id = wait_for_completion_timeout(&common->tdown_complete,
686 msecs_to_jiffies(1000));
687 if (!id)
688 dev_err(common->dev, "tx teardown timeout\n");
689
690 for (id = common->tx_ch_num - 1; id >= 0; id--)
691 am65_cpsw_destroy_txq(common, id);
692 }
693
am65_cpsw_create_txq(struct am65_cpsw_common * common,int id)694 static int am65_cpsw_create_txq(struct am65_cpsw_common *common, int id)
695 {
696 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[id];
697 int ret;
698
699 ret = k3_udma_glue_enable_tx_chn(tx_chn->tx_chn);
700 if (ret)
701 return ret;
702
703 napi_enable(&tx_chn->napi_tx);
704
705 return 0;
706 }
707
am65_cpsw_create_txqs(struct am65_cpsw_common * common)708 static int am65_cpsw_create_txqs(struct am65_cpsw_common *common)
709 {
710 int id, ret;
711
712 for (id = 0; id < common->tx_ch_num; id++) {
713 ret = am65_cpsw_create_txq(common, id);
714 if (ret) {
715 dev_err(common->dev, "couldn't create txq %d: %d\n",
716 id, ret);
717 goto err;
718 }
719 }
720
721 return 0;
722
723 err:
724 for (--id; id >= 0; id--)
725 am65_cpsw_destroy_txq(common, id);
726
727 return ret;
728 }
729
am65_cpsw_nuss_desc_idx(struct k3_cppi_desc_pool * desc_pool,void * desc,unsigned char dsize_log2)730 static int am65_cpsw_nuss_desc_idx(struct k3_cppi_desc_pool *desc_pool,
731 void *desc,
732 unsigned char dsize_log2)
733 {
734 void *pool_addr = k3_cppi_desc_pool_cpuaddr(desc_pool);
735
736 return (desc - pool_addr) >> dsize_log2;
737 }
738
am65_cpsw_nuss_set_buf_type(struct am65_cpsw_tx_chn * tx_chn,struct cppi5_host_desc_t * desc,enum am65_cpsw_tx_buf_type buf_type)739 static void am65_cpsw_nuss_set_buf_type(struct am65_cpsw_tx_chn *tx_chn,
740 struct cppi5_host_desc_t *desc,
741 enum am65_cpsw_tx_buf_type buf_type)
742 {
743 int desc_idx;
744
745 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc,
746 tx_chn->dsize_log2);
747 k3_cppi_desc_pool_desc_info_set(tx_chn->desc_pool, desc_idx,
748 (void *)buf_type);
749 }
750
am65_cpsw_nuss_buf_type(struct am65_cpsw_tx_chn * tx_chn,dma_addr_t desc_dma)751 static enum am65_cpsw_tx_buf_type am65_cpsw_nuss_buf_type(struct am65_cpsw_tx_chn *tx_chn,
752 dma_addr_t desc_dma)
753 {
754 struct cppi5_host_desc_t *desc_tx;
755 int desc_idx;
756
757 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
758 desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc_tx,
759 tx_chn->dsize_log2);
760
761 return (enum am65_cpsw_tx_buf_type)k3_cppi_desc_pool_desc_info(tx_chn->desc_pool,
762 desc_idx);
763 }
764
am65_cpsw_put_page(struct am65_cpsw_rx_flow * flow,struct page * page,bool allow_direct)765 static inline void am65_cpsw_put_page(struct am65_cpsw_rx_flow *flow,
766 struct page *page,
767 bool allow_direct)
768 {
769 page_pool_put_full_page(flow->page_pool, page, allow_direct);
770 }
771
am65_cpsw_nuss_rx_cleanup(void * data,dma_addr_t desc_dma)772 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
773 {
774 struct am65_cpsw_rx_chn *rx_chn = data;
775 struct cppi5_host_desc_t *desc_rx;
776 struct am65_cpsw_swdata *swdata;
777 dma_addr_t buf_dma;
778 struct page *page;
779 u32 buf_dma_len;
780 u32 flow_id;
781
782 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
783 swdata = cppi5_hdesc_get_swdata(desc_rx);
784 page = swdata->page;
785 flow_id = swdata->flow_id;
786 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
787 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
788 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
789 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
790 am65_cpsw_put_page(&rx_chn->flows[flow_id], page, false);
791 }
792
am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn * tx_chn,struct cppi5_host_desc_t * desc)793 static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
794 struct cppi5_host_desc_t *desc)
795 {
796 struct cppi5_host_desc_t *first_desc, *next_desc;
797 dma_addr_t buf_dma, next_desc_dma;
798 u32 buf_dma_len;
799
800 first_desc = desc;
801 next_desc = first_desc;
802
803 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
804 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
805
806 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
807
808 next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
809 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
810 while (next_desc_dma) {
811 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
812 next_desc_dma);
813 cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
814 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
815
816 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
817 DMA_TO_DEVICE);
818
819 next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
820 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
821
822 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
823 }
824
825 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
826 }
827
am65_cpsw_nuss_tx_cleanup(void * data,dma_addr_t desc_dma)828 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
829 {
830 struct am65_cpsw_tx_chn *tx_chn = data;
831 enum am65_cpsw_tx_buf_type buf_type;
832 struct cppi5_host_desc_t *desc_tx;
833 struct xdp_frame *xdpf;
834 struct sk_buff *skb;
835 void **swdata;
836
837 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
838 swdata = cppi5_hdesc_get_swdata(desc_tx);
839 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
840 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
841 skb = *(swdata);
842 dev_kfree_skb_any(skb);
843 } else {
844 xdpf = *(swdata);
845 xdp_return_frame(xdpf);
846 }
847
848 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
849 }
850
am65_cpsw_build_skb(void * page_addr,struct net_device * ndev,unsigned int len,unsigned int headroom)851 static struct sk_buff *am65_cpsw_build_skb(void *page_addr,
852 struct net_device *ndev,
853 unsigned int len,
854 unsigned int headroom)
855 {
856 struct sk_buff *skb;
857
858 len += AM65_CPSW_HEADROOM;
859
860 skb = build_skb(page_addr, len);
861 if (unlikely(!skb))
862 return NULL;
863
864 skb_reserve(skb, headroom);
865 skb->dev = ndev;
866
867 return skb;
868 }
869
am65_cpsw_nuss_common_open(struct am65_cpsw_common * common)870 static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
871 {
872 struct am65_cpsw_host *host_p = am65_common_get_host(common);
873 u32 val, port_mask;
874 int port_idx, ret;
875
876 if (common->usage_count)
877 return 0;
878
879 /* Control register */
880 writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE |
881 AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD,
882 common->cpsw_base + AM65_CPSW_REG_CTL);
883 /* Max length register */
884 writel(AM65_CPSW_MAX_PACKET_SIZE,
885 host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
886 /* set base flow_id */
887 writel(common->rx_flow_id_base,
888 host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
889 writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN | AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN,
890 host_p->port_base + AM65_CPSW_P0_REG_CTL);
891
892 am65_cpsw_nuss_set_p0_ptype(common);
893
894 /* enable statistic */
895 val = BIT(HOST_PORT_NUM);
896 for (port_idx = 0; port_idx < common->port_num; port_idx++) {
897 struct am65_cpsw_port *port = &common->ports[port_idx];
898
899 if (!port->disabled)
900 val |= BIT(port->port_id);
901 }
902 writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
903
904 /* disable priority elevation */
905 writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE);
906
907 cpsw_ale_start(common->ale);
908
909 /* limit to one RX flow only */
910 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
911 ALE_DEFAULT_THREAD_ID, 0);
912 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
913 ALE_DEFAULT_THREAD_ENABLE, 1);
914 /* switch to vlan aware mode */
915 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1);
916 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
917 ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
918
919 /* default vlan cfg: create mask based on enabled ports */
920 port_mask = GENMASK(common->port_num, 0) &
921 ~common->disabled_ports_mask;
922
923 cpsw_ale_add_vlan(common->ale, 0, port_mask,
924 port_mask, port_mask,
925 port_mask & ~ALE_PORT_HOST);
926
927 if (common->is_emac_mode)
928 am65_cpsw_init_host_port_emac(common);
929 else
930 am65_cpsw_init_host_port_switch(common);
931
932 am65_cpsw_qos_tx_p0_rate_init(common);
933
934 ret = am65_cpsw_create_rxqs(common);
935 if (ret)
936 return ret;
937
938 ret = am65_cpsw_create_txqs(common);
939 if (ret)
940 goto cleanup_rx;
941
942 dev_dbg(common->dev, "cpsw_nuss started\n");
943 return 0;
944
945 cleanup_rx:
946 am65_cpsw_destroy_rxqs(common);
947
948 return ret;
949 }
950
am65_cpsw_nuss_common_stop(struct am65_cpsw_common * common)951 static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
952 {
953 if (common->usage_count != 1)
954 return 0;
955
956 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
957 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
958
959 am65_cpsw_destroy_txqs(common);
960 am65_cpsw_destroy_rxqs(common);
961 cpsw_ale_stop(common->ale);
962
963 writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);
964 writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
965
966 dev_dbg(common->dev, "cpsw_nuss stopped\n");
967 return 0;
968 }
969
am65_cpsw_nuss_ndo_slave_stop(struct net_device * ndev)970 static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev)
971 {
972 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
973 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
974 int ret;
975
976 phylink_stop(port->slave.phylink);
977
978 netif_tx_stop_all_queues(ndev);
979
980 phylink_disconnect_phy(port->slave.phylink);
981
982 ret = am65_cpsw_nuss_common_stop(common);
983 if (ret)
984 return ret;
985
986 common->usage_count--;
987 pm_runtime_put(common->dev);
988 return 0;
989 }
990
cpsw_restore_vlans(struct net_device * vdev,int vid,void * arg)991 static int cpsw_restore_vlans(struct net_device *vdev, int vid, void *arg)
992 {
993 struct am65_cpsw_port *port = arg;
994
995 if (!vdev)
996 return 0;
997
998 return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid);
999 }
1000
am65_cpsw_nuss_ndo_slave_open(struct net_device * ndev)1001 static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
1002 {
1003 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1004 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1005 int ret, i;
1006 u32 reg;
1007
1008 ret = pm_runtime_resume_and_get(common->dev);
1009 if (ret < 0)
1010 return ret;
1011
1012 /* Idle MAC port */
1013 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
1014 cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
1015 cpsw_sl_ctl_reset(port->slave.mac_sl);
1016
1017 /* soft reset MAC */
1018 cpsw_sl_reg_write(port->slave.mac_sl, CPSW_SL_SOFT_RESET, 1);
1019 mdelay(1);
1020 reg = cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_SOFT_RESET);
1021 if (reg) {
1022 dev_err(common->dev, "soft RESET didn't complete\n");
1023 ret = -ETIMEDOUT;
1024 goto runtime_put;
1025 }
1026
1027 /* Notify the stack of the actual queue counts. */
1028 ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num);
1029 if (ret) {
1030 dev_err(common->dev, "cannot set real number of tx queues\n");
1031 goto runtime_put;
1032 }
1033
1034 ret = netif_set_real_num_rx_queues(ndev, common->rx_ch_num_flows);
1035 if (ret) {
1036 dev_err(common->dev, "cannot set real number of rx queues\n");
1037 goto runtime_put;
1038 }
1039
1040 for (i = 0; i < common->tx_ch_num; i++) {
1041 struct netdev_queue *txq = netdev_get_tx_queue(ndev, i);
1042
1043 netdev_tx_reset_queue(txq);
1044 txq->tx_maxrate = common->tx_chns[i].rate_mbps;
1045 }
1046
1047 ret = am65_cpsw_nuss_common_open(common);
1048 if (ret)
1049 goto runtime_put;
1050
1051 common->usage_count++;
1052
1053 /* VLAN aware CPSW mode is incompatible with some DSA tagging schemes.
1054 * Therefore disable VLAN_AWARE mode if any of the ports is a DSA Port.
1055 */
1056 if (netdev_uses_dsa(ndev)) {
1057 reg = readl(common->cpsw_base + AM65_CPSW_REG_CTL);
1058 reg &= ~AM65_CPSW_CTL_VLAN_AWARE;
1059 writel(reg, common->cpsw_base + AM65_CPSW_REG_CTL);
1060 }
1061
1062 am65_cpsw_port_set_sl_mac(port, ndev->dev_addr);
1063 am65_cpsw_port_enable_dscp_map(port);
1064
1065 if (common->is_emac_mode)
1066 am65_cpsw_init_port_emac_ale(port);
1067 else
1068 am65_cpsw_init_port_switch_ale(port);
1069
1070 /* mac_sl should be configured via phy-link interface */
1071 am65_cpsw_sl_ctl_reset(port);
1072
1073 ret = phylink_of_phy_connect(port->slave.phylink, port->slave.port_np, 0);
1074 if (ret)
1075 goto error_cleanup;
1076
1077 /* restore vlan configurations */
1078 vlan_for_each(ndev, cpsw_restore_vlans, port);
1079
1080 phylink_start(port->slave.phylink);
1081
1082 return 0;
1083
1084 error_cleanup:
1085 am65_cpsw_nuss_ndo_slave_stop(ndev);
1086 return ret;
1087
1088 runtime_put:
1089 pm_runtime_put(common->dev);
1090 return ret;
1091 }
1092
am65_cpsw_xdp_tx_frame(struct net_device * ndev,struct am65_cpsw_tx_chn * tx_chn,struct xdp_frame * xdpf,enum am65_cpsw_tx_buf_type buf_type)1093 static int am65_cpsw_xdp_tx_frame(struct net_device *ndev,
1094 struct am65_cpsw_tx_chn *tx_chn,
1095 struct xdp_frame *xdpf,
1096 enum am65_cpsw_tx_buf_type buf_type)
1097 {
1098 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1099 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1100 struct cppi5_host_desc_t *host_desc;
1101 struct netdev_queue *netif_txq;
1102 dma_addr_t dma_desc, dma_buf;
1103 u32 pkt_len = xdpf->len;
1104 void **swdata;
1105 int ret;
1106
1107 host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1108 if (unlikely(!host_desc)) {
1109 ndev->stats.tx_dropped++;
1110 return AM65_CPSW_XDP_CONSUMED; /* drop */
1111 }
1112
1113 am65_cpsw_nuss_set_buf_type(tx_chn, host_desc, buf_type);
1114
1115 dma_buf = dma_map_single(tx_chn->dma_dev, xdpf->data,
1116 pkt_len, DMA_TO_DEVICE);
1117 if (unlikely(dma_mapping_error(tx_chn->dma_dev, dma_buf))) {
1118 ndev->stats.tx_dropped++;
1119 ret = AM65_CPSW_XDP_CONSUMED; /* drop */
1120 goto pool_free;
1121 }
1122
1123 cppi5_hdesc_init(host_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1124 AM65_CPSW_NAV_PS_DATA_SIZE);
1125 cppi5_hdesc_set_pkttype(host_desc, AM65_CPSW_CPPI_TX_PKT_TYPE);
1126 cppi5_hdesc_set_pktlen(host_desc, pkt_len);
1127 cppi5_desc_set_pktids(&host_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID);
1128 cppi5_desc_set_tags_ids(&host_desc->hdr, 0, port->port_id);
1129
1130 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf);
1131 cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf, pkt_len);
1132
1133 swdata = cppi5_hdesc_get_swdata(host_desc);
1134 *(swdata) = xdpf;
1135
1136 /* Report BQL before sending the packet */
1137 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
1138 netdev_tx_sent_queue(netif_txq, pkt_len);
1139
1140 dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, host_desc);
1141 if (AM65_CPSW_IS_CPSW2G(common)) {
1142 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,
1143 dma_desc);
1144 } else {
1145 spin_lock_bh(&tx_chn->lock);
1146 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,
1147 dma_desc);
1148 spin_unlock_bh(&tx_chn->lock);
1149 }
1150 if (ret) {
1151 /* Inform BQL */
1152 netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1153 ndev->stats.tx_errors++;
1154 ret = AM65_CPSW_XDP_CONSUMED; /* drop */
1155 goto dma_unmap;
1156 }
1157
1158 return 0;
1159
1160 dma_unmap:
1161 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &dma_buf);
1162 dma_unmap_single(tx_chn->dma_dev, dma_buf, pkt_len, DMA_TO_DEVICE);
1163 pool_free:
1164 k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc);
1165 return ret;
1166 }
1167
am65_cpsw_run_xdp(struct am65_cpsw_rx_flow * flow,struct am65_cpsw_port * port,struct xdp_buff * xdp,int cpu,int * len)1168 static int am65_cpsw_run_xdp(struct am65_cpsw_rx_flow *flow,
1169 struct am65_cpsw_port *port,
1170 struct xdp_buff *xdp,
1171 int cpu, int *len)
1172 {
1173 struct am65_cpsw_common *common = flow->common;
1174 struct net_device *ndev = port->ndev;
1175 int ret = AM65_CPSW_XDP_CONSUMED;
1176 struct am65_cpsw_tx_chn *tx_chn;
1177 struct netdev_queue *netif_txq;
1178 struct xdp_frame *xdpf;
1179 struct bpf_prog *prog;
1180 struct page *page;
1181 int pkt_len;
1182 u32 act;
1183 int err;
1184
1185 pkt_len = *len;
1186 prog = READ_ONCE(port->xdp_prog);
1187 if (!prog)
1188 return AM65_CPSW_XDP_PASS;
1189
1190 act = bpf_prog_run_xdp(prog, xdp);
1191 /* XDP prog might have changed packet data and boundaries */
1192 *len = xdp->data_end - xdp->data;
1193
1194 switch (act) {
1195 case XDP_PASS:
1196 ret = AM65_CPSW_XDP_PASS;
1197 goto out;
1198 case XDP_TX:
1199 tx_chn = &common->tx_chns[cpu % AM65_CPSW_MAX_QUEUES];
1200 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
1201
1202 xdpf = xdp_convert_buff_to_frame(xdp);
1203 if (unlikely(!xdpf)) {
1204 ndev->stats.tx_dropped++;
1205 goto drop;
1206 }
1207
1208 __netif_tx_lock(netif_txq, cpu);
1209 err = am65_cpsw_xdp_tx_frame(ndev, tx_chn, xdpf,
1210 AM65_CPSW_TX_BUF_TYPE_XDP_TX);
1211 __netif_tx_unlock(netif_txq);
1212 if (err)
1213 goto drop;
1214
1215 dev_sw_netstats_rx_add(ndev, pkt_len);
1216 ret = AM65_CPSW_XDP_CONSUMED;
1217 goto out;
1218 case XDP_REDIRECT:
1219 if (unlikely(xdp_do_redirect(ndev, xdp, prog)))
1220 goto drop;
1221
1222 dev_sw_netstats_rx_add(ndev, pkt_len);
1223 ret = AM65_CPSW_XDP_REDIRECT;
1224 goto out;
1225 default:
1226 bpf_warn_invalid_xdp_action(ndev, prog, act);
1227 fallthrough;
1228 case XDP_ABORTED:
1229 drop:
1230 trace_xdp_exception(ndev, prog, act);
1231 fallthrough;
1232 case XDP_DROP:
1233 ndev->stats.rx_dropped++;
1234 }
1235
1236 page = virt_to_head_page(xdp->data);
1237 am65_cpsw_put_page(flow, page, true);
1238
1239 out:
1240 return ret;
1241 }
1242
1243 /* RX psdata[2] word format - checksum information */
1244 #define AM65_CPSW_RX_PSD_CSUM_ADD GENMASK(15, 0)
1245 #define AM65_CPSW_RX_PSD_CSUM_ERR BIT(16)
1246 #define AM65_CPSW_RX_PSD_IS_FRAGMENT BIT(17)
1247 #define AM65_CPSW_RX_PSD_IS_TCP BIT(18)
1248 #define AM65_CPSW_RX_PSD_IPV6_VALID BIT(19)
1249 #define AM65_CPSW_RX_PSD_IPV4_VALID BIT(20)
1250
am65_cpsw_nuss_rx_csum(struct sk_buff * skb,u32 csum_info)1251 static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
1252 {
1253 /* HW can verify IPv4/IPv6 TCP/UDP packets checksum
1254 * csum information provides in psdata[2] word:
1255 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error
1256 * AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID
1257 * bits - indicates IPv4/IPv6 packet
1258 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet
1259 * AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets
1260 * or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR
1261 */
1262 skb_checksum_none_assert(skb);
1263
1264 if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
1265 return;
1266
1267 if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
1268 AM65_CPSW_RX_PSD_IPV4_VALID)) &&
1269 !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
1270 /* csum for fragmented packets is unsupported */
1271 if (!(csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT))
1272 skb->ip_summed = CHECKSUM_UNNECESSARY;
1273 }
1274 }
1275
am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow * flow,int cpu,int * xdp_state)1276 static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow,
1277 int cpu, int *xdp_state)
1278 {
1279 struct am65_cpsw_rx_chn *rx_chn = &flow->common->rx_chns;
1280 u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
1281 struct am65_cpsw_common *common = flow->common;
1282 struct am65_cpsw_ndev_priv *ndev_priv;
1283 struct cppi5_host_desc_t *desc_rx;
1284 struct device *dev = common->dev;
1285 struct am65_cpsw_swdata *swdata;
1286 struct page *page, *new_page;
1287 dma_addr_t desc_dma, buf_dma;
1288 struct am65_cpsw_port *port;
1289 struct net_device *ndev;
1290 u32 flow_idx = flow->id;
1291 struct sk_buff *skb;
1292 struct xdp_buff xdp;
1293 int headroom, ret;
1294 void *page_addr;
1295 u32 *psdata;
1296
1297 *xdp_state = AM65_CPSW_XDP_PASS;
1298 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
1299 if (ret) {
1300 if (ret != -ENODATA)
1301 dev_err(dev, "RX: pop chn fail %d\n", ret);
1302 return ret;
1303 }
1304
1305 if (cppi5_desc_is_tdcm(desc_dma)) {
1306 dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
1307 if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ)
1308 complete(&common->tdown_complete);
1309 return 0;
1310 }
1311
1312 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
1313 dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
1314 __func__, flow_idx, &desc_dma);
1315
1316 swdata = cppi5_hdesc_get_swdata(desc_rx);
1317 page = swdata->page;
1318 page_addr = page_address(page);
1319 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
1320 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
1321 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
1322 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
1323 dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
1324 port = am65_common_get_port(common, port_id);
1325 ndev = port->ndev;
1326 psdata = cppi5_hdesc_get_psdata(desc_rx);
1327 csum_info = psdata[2];
1328 dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
1329
1330 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
1331 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
1332
1333 if (port->xdp_prog) {
1334 xdp_init_buff(&xdp, PAGE_SIZE, &port->xdp_rxq[flow->id]);
1335 xdp_prepare_buff(&xdp, page_addr, AM65_CPSW_HEADROOM,
1336 pkt_len, false);
1337 *xdp_state = am65_cpsw_run_xdp(flow, port, &xdp,
1338 cpu, &pkt_len);
1339 if (*xdp_state != AM65_CPSW_XDP_PASS)
1340 goto allocate;
1341
1342 headroom = xdp.data - xdp.data_hard_start;
1343 } else {
1344 headroom = AM65_CPSW_HEADROOM;
1345 }
1346
1347 skb = am65_cpsw_build_skb(page_addr, ndev,
1348 AM65_CPSW_MAX_PACKET_SIZE, headroom);
1349 if (unlikely(!skb)) {
1350 new_page = page;
1351 goto requeue;
1352 }
1353
1354 ndev_priv = netdev_priv(ndev);
1355 am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark);
1356 skb_put(skb, pkt_len);
1357 if (port->rx_ts_enabled)
1358 am65_cpts_rx_timestamp(common->cpts, skb);
1359 skb_mark_for_recycle(skb);
1360 skb->protocol = eth_type_trans(skb, ndev);
1361 am65_cpsw_nuss_rx_csum(skb, csum_info);
1362 napi_gro_receive(&flow->napi_rx, skb);
1363
1364 dev_sw_netstats_rx_add(ndev, pkt_len);
1365
1366 allocate:
1367 new_page = page_pool_dev_alloc_pages(flow->page_pool);
1368 if (unlikely(!new_page)) {
1369 dev_err(dev, "page alloc failed\n");
1370 return -ENOMEM;
1371 }
1372
1373 if (netif_dormant(ndev)) {
1374 am65_cpsw_put_page(flow, new_page, true);
1375 ndev->stats.rx_dropped++;
1376 return 0;
1377 }
1378
1379 requeue:
1380 ret = am65_cpsw_nuss_rx_push(common, new_page, flow_idx);
1381 if (WARN_ON(ret < 0)) {
1382 am65_cpsw_put_page(flow, new_page, true);
1383 ndev->stats.rx_errors++;
1384 ndev->stats.rx_dropped++;
1385 }
1386
1387 return ret;
1388 }
1389
am65_cpsw_nuss_rx_timer_callback(struct hrtimer * timer)1390 static enum hrtimer_restart am65_cpsw_nuss_rx_timer_callback(struct hrtimer *timer)
1391 {
1392 struct am65_cpsw_rx_flow *flow = container_of(timer,
1393 struct am65_cpsw_rx_flow,
1394 rx_hrtimer);
1395
1396 enable_irq(flow->irq);
1397 return HRTIMER_NORESTART;
1398 }
1399
am65_cpsw_nuss_rx_poll(struct napi_struct * napi_rx,int budget)1400 static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
1401 {
1402 struct am65_cpsw_rx_flow *flow = am65_cpsw_napi_to_rx_flow(napi_rx);
1403 struct am65_cpsw_common *common = flow->common;
1404 int cpu = smp_processor_id();
1405 int xdp_state_or = 0;
1406 int cur_budget, ret;
1407 int xdp_state;
1408 int num_rx = 0;
1409
1410 /* process only this flow */
1411 cur_budget = budget;
1412 while (cur_budget--) {
1413 ret = am65_cpsw_nuss_rx_packets(flow, cpu, &xdp_state);
1414 xdp_state_or |= xdp_state;
1415 if (ret)
1416 break;
1417 num_rx++;
1418 }
1419
1420 if (xdp_state_or & AM65_CPSW_XDP_REDIRECT)
1421 xdp_do_flush();
1422
1423 dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
1424
1425 if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) {
1426 if (flow->irq_disabled) {
1427 flow->irq_disabled = false;
1428 if (unlikely(flow->rx_pace_timeout)) {
1429 hrtimer_start(&flow->rx_hrtimer,
1430 ns_to_ktime(flow->rx_pace_timeout),
1431 HRTIMER_MODE_REL_PINNED);
1432 } else {
1433 enable_irq(flow->irq);
1434 }
1435 }
1436 }
1437
1438 return num_rx;
1439 }
1440
1441 static struct sk_buff *
am65_cpsw_nuss_tx_compl_packet_skb(struct am65_cpsw_tx_chn * tx_chn,dma_addr_t desc_dma)1442 am65_cpsw_nuss_tx_compl_packet_skb(struct am65_cpsw_tx_chn *tx_chn,
1443 dma_addr_t desc_dma)
1444 {
1445 struct cppi5_host_desc_t *desc_tx;
1446 struct sk_buff *skb;
1447 void **swdata;
1448
1449 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
1450 desc_dma);
1451 swdata = cppi5_hdesc_get_swdata(desc_tx);
1452 skb = *(swdata);
1453 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
1454
1455 am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);
1456
1457 dev_sw_netstats_tx_add(skb->dev, 1, skb->len);
1458
1459 return skb;
1460 }
1461
1462 static struct xdp_frame *
am65_cpsw_nuss_tx_compl_packet_xdp(struct am65_cpsw_common * common,struct am65_cpsw_tx_chn * tx_chn,dma_addr_t desc_dma,struct net_device ** ndev)1463 am65_cpsw_nuss_tx_compl_packet_xdp(struct am65_cpsw_common *common,
1464 struct am65_cpsw_tx_chn *tx_chn,
1465 dma_addr_t desc_dma,
1466 struct net_device **ndev)
1467 {
1468 struct cppi5_host_desc_t *desc_tx;
1469 struct am65_cpsw_port *port;
1470 struct xdp_frame *xdpf;
1471 u32 port_id = 0;
1472 void **swdata;
1473
1474 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
1475 cppi5_desc_get_tags_ids(&desc_tx->hdr, NULL, &port_id);
1476 swdata = cppi5_hdesc_get_swdata(desc_tx);
1477 xdpf = *(swdata);
1478 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
1479
1480 port = am65_common_get_port(common, port_id);
1481 dev_sw_netstats_tx_add(port->ndev, 1, xdpf->len);
1482 *ndev = port->ndev;
1483
1484 return xdpf;
1485 }
1486
am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn * tx_chn,struct net_device * ndev,struct netdev_queue * netif_txq)1487 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,
1488 struct netdev_queue *netif_txq)
1489 {
1490 if (netif_tx_queue_stopped(netif_txq)) {
1491 /* Check whether the queue is stopped due to stalled
1492 * tx dma, if the queue is stopped then wake the queue
1493 * as we have free desc for tx
1494 */
1495 __netif_tx_lock(netif_txq, smp_processor_id());
1496 if (netif_running(ndev) &&
1497 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
1498 netif_tx_wake_queue(netif_txq);
1499
1500 __netif_tx_unlock(netif_txq);
1501 }
1502 }
1503
am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common * common,int chn,unsigned int budget,bool * tdown)1504 static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
1505 int chn, unsigned int budget, bool *tdown)
1506 {
1507 enum am65_cpsw_tx_buf_type buf_type;
1508 struct device *dev = common->dev;
1509 struct am65_cpsw_tx_chn *tx_chn;
1510 struct netdev_queue *netif_txq;
1511 unsigned int total_bytes = 0;
1512 struct net_device *ndev;
1513 struct xdp_frame *xdpf;
1514 struct sk_buff *skb;
1515 dma_addr_t desc_dma;
1516 int res, num_tx = 0;
1517
1518 tx_chn = &common->tx_chns[chn];
1519
1520 while (true) {
1521 spin_lock(&tx_chn->lock);
1522 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1523 spin_unlock(&tx_chn->lock);
1524 if (res == -ENODATA)
1525 break;
1526
1527 if (cppi5_desc_is_tdcm(desc_dma)) {
1528 if (atomic_dec_and_test(&common->tdown_cnt))
1529 complete(&common->tdown_complete);
1530 *tdown = true;
1531 break;
1532 }
1533
1534 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
1535 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
1536 skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);
1537 ndev = skb->dev;
1538 total_bytes = skb->len;
1539 napi_consume_skb(skb, budget);
1540 } else {
1541 xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,
1542 desc_dma, &ndev);
1543 total_bytes = xdpf->len;
1544 if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX)
1545 xdp_return_frame_rx_napi(xdpf);
1546 else
1547 xdp_return_frame(xdpf);
1548 }
1549 num_tx++;
1550
1551 netif_txq = netdev_get_tx_queue(ndev, chn);
1552
1553 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1554
1555 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1556 }
1557
1558 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1559
1560 return num_tx;
1561 }
1562
am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common * common,int chn,unsigned int budget,bool * tdown)1563 static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
1564 int chn, unsigned int budget, bool *tdown)
1565 {
1566 enum am65_cpsw_tx_buf_type buf_type;
1567 struct device *dev = common->dev;
1568 struct am65_cpsw_tx_chn *tx_chn;
1569 struct netdev_queue *netif_txq;
1570 unsigned int total_bytes = 0;
1571 struct net_device *ndev;
1572 struct xdp_frame *xdpf;
1573 struct sk_buff *skb;
1574 dma_addr_t desc_dma;
1575 int res, num_tx = 0;
1576
1577 tx_chn = &common->tx_chns[chn];
1578
1579 while (true) {
1580 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1581 if (res == -ENODATA)
1582 break;
1583
1584 if (cppi5_desc_is_tdcm(desc_dma)) {
1585 if (atomic_dec_and_test(&common->tdown_cnt))
1586 complete(&common->tdown_complete);
1587 *tdown = true;
1588 break;
1589 }
1590
1591 buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
1592 if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
1593 skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);
1594 ndev = skb->dev;
1595 total_bytes += skb->len;
1596 napi_consume_skb(skb, budget);
1597 } else {
1598 xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,
1599 desc_dma, &ndev);
1600 total_bytes += xdpf->len;
1601 if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX)
1602 xdp_return_frame_rx_napi(xdpf);
1603 else
1604 xdp_return_frame(xdpf);
1605 }
1606 num_tx++;
1607 }
1608
1609 if (!num_tx)
1610 return 0;
1611
1612 netif_txq = netdev_get_tx_queue(ndev, chn);
1613
1614 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1615
1616 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1617
1618 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1619
1620 return num_tx;
1621 }
1622
am65_cpsw_nuss_tx_timer_callback(struct hrtimer * timer)1623 static enum hrtimer_restart am65_cpsw_nuss_tx_timer_callback(struct hrtimer *timer)
1624 {
1625 struct am65_cpsw_tx_chn *tx_chns =
1626 container_of(timer, struct am65_cpsw_tx_chn, tx_hrtimer);
1627
1628 enable_irq(tx_chns->irq);
1629 return HRTIMER_NORESTART;
1630 }
1631
am65_cpsw_nuss_tx_poll(struct napi_struct * napi_tx,int budget)1632 static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
1633 {
1634 struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);
1635 bool tdown = false;
1636 int num_tx;
1637
1638 if (AM65_CPSW_IS_CPSW2G(tx_chn->common))
1639 num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id,
1640 budget, &tdown);
1641 else
1642 num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common,
1643 tx_chn->id, budget, &tdown);
1644
1645 if (num_tx >= budget)
1646 return budget;
1647
1648 if (napi_complete_done(napi_tx, num_tx)) {
1649 if (unlikely(tx_chn->tx_pace_timeout && !tdown)) {
1650 hrtimer_start(&tx_chn->tx_hrtimer,
1651 ns_to_ktime(tx_chn->tx_pace_timeout),
1652 HRTIMER_MODE_REL_PINNED);
1653 } else {
1654 enable_irq(tx_chn->irq);
1655 }
1656 }
1657
1658 return 0;
1659 }
1660
am65_cpsw_nuss_rx_irq(int irq,void * dev_id)1661 static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id)
1662 {
1663 struct am65_cpsw_rx_flow *flow = dev_id;
1664
1665 flow->irq_disabled = true;
1666 disable_irq_nosync(irq);
1667 napi_schedule(&flow->napi_rx);
1668
1669 return IRQ_HANDLED;
1670 }
1671
am65_cpsw_nuss_tx_irq(int irq,void * dev_id)1672 static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id)
1673 {
1674 struct am65_cpsw_tx_chn *tx_chn = dev_id;
1675
1676 disable_irq_nosync(irq);
1677 napi_schedule(&tx_chn->napi_tx);
1678
1679 return IRQ_HANDLED;
1680 }
1681
am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff * skb,struct net_device * ndev)1682 static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
1683 struct net_device *ndev)
1684 {
1685 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1686 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
1687 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1688 struct device *dev = common->dev;
1689 struct am65_cpsw_tx_chn *tx_chn;
1690 struct netdev_queue *netif_txq;
1691 dma_addr_t desc_dma, buf_dma;
1692 int ret, q_idx, i;
1693 void **swdata;
1694 u32 *psdata;
1695 u32 pkt_len;
1696
1697 /* padding enabled in hw */
1698 pkt_len = skb_headlen(skb);
1699
1700 /* SKB TX timestamp */
1701 if (port->tx_ts_enabled)
1702 am65_cpts_prep_tx_timestamp(common->cpts, skb);
1703
1704 q_idx = skb_get_queue_mapping(skb);
1705 dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
1706
1707 tx_chn = &common->tx_chns[q_idx];
1708 netif_txq = netdev_get_tx_queue(ndev, q_idx);
1709
1710 /* Map the linear buffer */
1711 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,
1712 DMA_TO_DEVICE);
1713 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1714 dev_err(dev, "Failed to map tx skb buffer\n");
1715 ndev->stats.tx_errors++;
1716 goto err_free_skb;
1717 }
1718
1719 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1720 if (!first_desc) {
1721 dev_dbg(dev, "Failed to allocate descriptor\n");
1722 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,
1723 DMA_TO_DEVICE);
1724 goto busy_stop_q;
1725 }
1726
1727 am65_cpsw_nuss_set_buf_type(tx_chn, first_desc,
1728 AM65_CPSW_TX_BUF_TYPE_SKB);
1729
1730 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1731 AM65_CPSW_NAV_PS_DATA_SIZE);
1732 cppi5_desc_set_pktids(&first_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID);
1733 cppi5_hdesc_set_pkttype(first_desc, AM65_CPSW_CPPI_TX_PKT_TYPE);
1734 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);
1735
1736 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1737 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
1738 swdata = cppi5_hdesc_get_swdata(first_desc);
1739 *(swdata) = skb;
1740 psdata = cppi5_hdesc_get_psdata(first_desc);
1741
1742 /* HW csum offload if enabled */
1743 psdata[2] = 0;
1744 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1745 unsigned int cs_start, cs_offset;
1746
1747 cs_start = skb_transport_offset(skb);
1748 cs_offset = cs_start + skb->csum_offset;
1749 /* HW numerates bytes starting from 1 */
1750 psdata[2] = ((cs_offset + 1) << 24) |
1751 ((cs_start + 1) << 16) | (skb->len - cs_start);
1752 dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
1753 }
1754
1755 if (!skb_is_nonlinear(skb))
1756 goto done_tx;
1757
1758 dev_dbg(dev, "fragmented SKB\n");
1759
1760 /* Handle the case where skb is fragmented in pages */
1761 cur_desc = first_desc;
1762 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1763 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1764 u32 frag_size = skb_frag_size(frag);
1765
1766 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1767 if (!next_desc) {
1768 dev_err(dev, "Failed to allocate descriptor\n");
1769 goto busy_free_descs;
1770 }
1771
1772 am65_cpsw_nuss_set_buf_type(tx_chn, next_desc,
1773 AM65_CPSW_TX_BUF_TYPE_SKB);
1774
1775 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
1776 DMA_TO_DEVICE);
1777 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1778 dev_err(dev, "Failed to map tx skb page\n");
1779 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
1780 ndev->stats.tx_errors++;
1781 goto err_free_descs;
1782 }
1783
1784 cppi5_hdesc_reset_hbdesc(next_desc);
1785 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1786 cppi5_hdesc_attach_buf(next_desc,
1787 buf_dma, frag_size, buf_dma, frag_size);
1788
1789 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
1790 next_desc);
1791 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);
1792 cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
1793
1794 pkt_len += frag_size;
1795 cur_desc = next_desc;
1796 }
1797 WARN_ON(pkt_len != skb->len);
1798
1799 done_tx:
1800 skb_tx_timestamp(skb);
1801
1802 /* report bql before sending packet */
1803 netdev_tx_sent_queue(netif_txq, pkt_len);
1804
1805 cppi5_hdesc_set_pktlen(first_desc, pkt_len);
1806 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
1807 if (AM65_CPSW_IS_CPSW2G(common)) {
1808 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1809 } else {
1810 spin_lock_bh(&tx_chn->lock);
1811 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1812 spin_unlock_bh(&tx_chn->lock);
1813 }
1814 if (ret) {
1815 dev_err(dev, "can't push desc %d\n", ret);
1816 /* inform bql */
1817 netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1818 ndev->stats.tx_errors++;
1819 goto err_free_descs;
1820 }
1821
1822 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
1823 netif_tx_stop_queue(netif_txq);
1824 /* Barrier, so that stop_queue visible to other cpus */
1825 smp_mb__after_atomic();
1826 dev_dbg(dev, "netif_tx_stop_queue %d\n", q_idx);
1827
1828 /* re-check for smp */
1829 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
1830 MAX_SKB_FRAGS) {
1831 netif_tx_wake_queue(netif_txq);
1832 dev_dbg(dev, "netif_tx_wake_queue %d\n", q_idx);
1833 }
1834 }
1835
1836 return NETDEV_TX_OK;
1837
1838 err_free_descs:
1839 am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1840 err_free_skb:
1841 ndev->stats.tx_dropped++;
1842 dev_kfree_skb_any(skb);
1843 return NETDEV_TX_OK;
1844
1845 busy_free_descs:
1846 am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1847 busy_stop_q:
1848 netif_tx_stop_queue(netif_txq);
1849 return NETDEV_TX_BUSY;
1850 }
1851
am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device * ndev,void * addr)1852 static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev,
1853 void *addr)
1854 {
1855 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1856 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1857 struct sockaddr *sockaddr = (struct sockaddr *)addr;
1858 int ret;
1859
1860 ret = eth_prepare_mac_addr_change(ndev, addr);
1861 if (ret < 0)
1862 return ret;
1863
1864 ret = pm_runtime_resume_and_get(common->dev);
1865 if (ret < 0)
1866 return ret;
1867
1868 cpsw_ale_del_ucast(common->ale, ndev->dev_addr,
1869 HOST_PORT_NUM, 0, 0);
1870 cpsw_ale_add_ucast(common->ale, sockaddr->sa_data,
1871 HOST_PORT_NUM, ALE_SECURE, 0);
1872
1873 am65_cpsw_port_set_sl_mac(port, addr);
1874 eth_commit_mac_addr_change(ndev, sockaddr);
1875
1876 pm_runtime_put(common->dev);
1877
1878 return 0;
1879 }
1880
am65_cpsw_nuss_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)1881 static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
1882 struct ifreq *ifr)
1883 {
1884 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1885 u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype;
1886 struct hwtstamp_config cfg;
1887
1888 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1889 return -EOPNOTSUPP;
1890
1891 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1892 return -EFAULT;
1893
1894 /* TX HW timestamp */
1895 switch (cfg.tx_type) {
1896 case HWTSTAMP_TX_OFF:
1897 case HWTSTAMP_TX_ON:
1898 break;
1899 default:
1900 return -ERANGE;
1901 }
1902
1903 switch (cfg.rx_filter) {
1904 case HWTSTAMP_FILTER_NONE:
1905 port->rx_ts_enabled = false;
1906 break;
1907 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1908 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1909 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1910 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1911 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1912 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1913 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1914 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1915 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1916 port->rx_ts_enabled = true;
1917 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1918 break;
1919 case HWTSTAMP_FILTER_ALL:
1920 case HWTSTAMP_FILTER_SOME:
1921 case HWTSTAMP_FILTER_NTP_ALL:
1922 return -EOPNOTSUPP;
1923 default:
1924 return -ERANGE;
1925 }
1926
1927 port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);
1928
1929 /* cfg TX timestamp */
1930 seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET <<
1931 AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588;
1932
1933 ts_vlan_ltype = ETH_P_8021Q;
1934
1935 ts_ctrl_ltype2 = ETH_P_1588 |
1936 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 |
1937 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 |
1938 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 |
1939 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 |
1940 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 |
1941 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 |
1942 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 |
1943 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO;
1944
1945 ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS <<
1946 AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT;
1947
1948 if (port->tx_ts_enabled)
1949 ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN |
1950 AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN;
1951
1952 if (port->rx_ts_enabled)
1953 ts_ctrl |= AM65_CPSW_TS_RX_ANX_ALL_EN |
1954 AM65_CPSW_PN_TS_CTL_RX_VLAN_LT1_EN;
1955
1956 writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);
1957 writel(ts_vlan_ltype, port->port_base +
1958 AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG);
1959 writel(ts_ctrl_ltype2, port->port_base +
1960 AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2);
1961 writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);
1962
1963 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1964 }
1965
am65_cpsw_nuss_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)1966 static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
1967 struct ifreq *ifr)
1968 {
1969 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1970 struct hwtstamp_config cfg;
1971
1972 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1973 return -EOPNOTSUPP;
1974
1975 cfg.flags = 0;
1976 cfg.tx_type = port->tx_ts_enabled ?
1977 HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1978 cfg.rx_filter = port->rx_ts_enabled ?
1979 HWTSTAMP_FILTER_PTP_V2_EVENT : HWTSTAMP_FILTER_NONE;
1980
1981 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1982 }
1983
am65_cpsw_nuss_ndo_slave_ioctl(struct net_device * ndev,struct ifreq * req,int cmd)1984 static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev,
1985 struct ifreq *req, int cmd)
1986 {
1987 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1988
1989 if (!netif_running(ndev))
1990 return -EINVAL;
1991
1992 switch (cmd) {
1993 case SIOCSHWTSTAMP:
1994 return am65_cpsw_nuss_hwtstamp_set(ndev, req);
1995 case SIOCGHWTSTAMP:
1996 return am65_cpsw_nuss_hwtstamp_get(ndev, req);
1997 }
1998
1999 return phylink_mii_ioctl(port->slave.phylink, req, cmd);
2000 }
2001
am65_cpsw_nuss_ndo_get_stats(struct net_device * dev,struct rtnl_link_stats64 * stats)2002 static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
2003 struct rtnl_link_stats64 *stats)
2004 {
2005 dev_fetch_sw_netstats(stats, dev->tstats);
2006
2007 stats->rx_errors = dev->stats.rx_errors;
2008 stats->rx_dropped = dev->stats.rx_dropped;
2009 stats->tx_dropped = dev->stats.tx_dropped;
2010 }
2011
am65_cpsw_xdp_prog_setup(struct net_device * ndev,struct bpf_prog * prog)2012 static int am65_cpsw_xdp_prog_setup(struct net_device *ndev,
2013 struct bpf_prog *prog)
2014 {
2015 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
2016 bool running = netif_running(ndev);
2017 struct bpf_prog *old_prog;
2018
2019 if (running)
2020 am65_cpsw_nuss_ndo_slave_stop(ndev);
2021
2022 old_prog = xchg(&port->xdp_prog, prog);
2023 if (old_prog)
2024 bpf_prog_put(old_prog);
2025
2026 if (running)
2027 return am65_cpsw_nuss_ndo_slave_open(ndev);
2028
2029 return 0;
2030 }
2031
am65_cpsw_ndo_bpf(struct net_device * ndev,struct netdev_bpf * bpf)2032 static int am65_cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2033 {
2034 switch (bpf->command) {
2035 case XDP_SETUP_PROG:
2036 return am65_cpsw_xdp_prog_setup(ndev, bpf->prog);
2037 default:
2038 return -EINVAL;
2039 }
2040 }
2041
am65_cpsw_ndo_xdp_xmit(struct net_device * ndev,int n,struct xdp_frame ** frames,u32 flags)2042 static int am65_cpsw_ndo_xdp_xmit(struct net_device *ndev, int n,
2043 struct xdp_frame **frames, u32 flags)
2044 {
2045 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2046 struct am65_cpsw_tx_chn *tx_chn;
2047 struct netdev_queue *netif_txq;
2048 int cpu = smp_processor_id();
2049 int i, nxmit = 0;
2050
2051 tx_chn = &common->tx_chns[cpu % common->tx_ch_num];
2052 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
2053
2054 __netif_tx_lock(netif_txq, cpu);
2055 for (i = 0; i < n; i++) {
2056 if (am65_cpsw_xdp_tx_frame(ndev, tx_chn, frames[i],
2057 AM65_CPSW_TX_BUF_TYPE_XDP_NDO))
2058 break;
2059 nxmit++;
2060 }
2061 __netif_tx_unlock(netif_txq);
2062
2063 return nxmit;
2064 }
2065
2066 static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
2067 .ndo_open = am65_cpsw_nuss_ndo_slave_open,
2068 .ndo_stop = am65_cpsw_nuss_ndo_slave_stop,
2069 .ndo_start_xmit = am65_cpsw_nuss_ndo_slave_xmit,
2070 .ndo_set_rx_mode = am65_cpsw_nuss_ndo_slave_set_rx_mode,
2071 .ndo_get_stats64 = am65_cpsw_nuss_ndo_get_stats,
2072 .ndo_validate_addr = eth_validate_addr,
2073 .ndo_set_mac_address = am65_cpsw_nuss_ndo_slave_set_mac_address,
2074 .ndo_tx_timeout = am65_cpsw_nuss_ndo_host_tx_timeout,
2075 .ndo_vlan_rx_add_vid = am65_cpsw_nuss_ndo_slave_add_vid,
2076 .ndo_vlan_rx_kill_vid = am65_cpsw_nuss_ndo_slave_kill_vid,
2077 .ndo_eth_ioctl = am65_cpsw_nuss_ndo_slave_ioctl,
2078 .ndo_setup_tc = am65_cpsw_qos_ndo_setup_tc,
2079 .ndo_set_tx_maxrate = am65_cpsw_qos_ndo_tx_p0_set_maxrate,
2080 .ndo_bpf = am65_cpsw_ndo_bpf,
2081 .ndo_xdp_xmit = am65_cpsw_ndo_xdp_xmit,
2082 };
2083
am65_cpsw_disable_phy(struct phy * phy)2084 static void am65_cpsw_disable_phy(struct phy *phy)
2085 {
2086 phy_power_off(phy);
2087 phy_exit(phy);
2088 }
2089
am65_cpsw_enable_phy(struct phy * phy)2090 static int am65_cpsw_enable_phy(struct phy *phy)
2091 {
2092 int ret;
2093
2094 ret = phy_init(phy);
2095 if (ret < 0)
2096 return ret;
2097
2098 ret = phy_power_on(phy);
2099 if (ret < 0) {
2100 phy_exit(phy);
2101 return ret;
2102 }
2103
2104 return 0;
2105 }
2106
am65_cpsw_disable_serdes_phy(struct am65_cpsw_common * common)2107 static void am65_cpsw_disable_serdes_phy(struct am65_cpsw_common *common)
2108 {
2109 struct am65_cpsw_port *port;
2110 struct phy *phy;
2111 int i;
2112
2113 for (i = 0; i < common->port_num; i++) {
2114 port = &common->ports[i];
2115 phy = port->slave.serdes_phy;
2116 if (phy)
2117 am65_cpsw_disable_phy(phy);
2118 }
2119 }
2120
am65_cpsw_init_serdes_phy(struct device * dev,struct device_node * port_np,struct am65_cpsw_port * port)2121 static int am65_cpsw_init_serdes_phy(struct device *dev, struct device_node *port_np,
2122 struct am65_cpsw_port *port)
2123 {
2124 const char *name = "serdes";
2125 struct phy *phy;
2126 int ret;
2127
2128 phy = devm_of_phy_optional_get(dev, port_np, name);
2129 if (IS_ERR_OR_NULL(phy))
2130 return PTR_ERR_OR_ZERO(phy);
2131
2132 /* Serdes PHY exists. Store it. */
2133 port->slave.serdes_phy = phy;
2134
2135 ret = am65_cpsw_enable_phy(phy);
2136 if (ret < 0)
2137 goto err_phy;
2138
2139 return 0;
2140
2141 err_phy:
2142 devm_phy_put(dev, phy);
2143 return ret;
2144 }
2145
am65_cpsw_nuss_mac_config(struct phylink_config * config,unsigned int mode,const struct phylink_link_state * state)2146 static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned int mode,
2147 const struct phylink_link_state *state)
2148 {
2149 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
2150 phylink_config);
2151 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
2152 struct am65_cpsw_common *common = port->common;
2153
2154 if (common->pdata.extra_modes & BIT(state->interface)) {
2155 if (state->interface == PHY_INTERFACE_MODE_SGMII) {
2156 writel(ADVERTISE_SGMII,
2157 port->sgmii_base + AM65_CPSW_SGMII_MR_ADV_ABILITY_REG);
2158 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
2159 } else {
2160 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
2161 }
2162
2163 if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
2164 cpsw_sl_ctl_set(port->slave.mac_sl,
2165 CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
2166 } else {
2167 cpsw_sl_ctl_clr(port->slave.mac_sl,
2168 CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
2169 }
2170
2171 writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE,
2172 port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG);
2173 }
2174 }
2175
am65_cpsw_nuss_mac_link_down(struct phylink_config * config,unsigned int mode,phy_interface_t interface)2176 static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode,
2177 phy_interface_t interface)
2178 {
2179 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
2180 phylink_config);
2181 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
2182 struct am65_cpsw_common *common = port->common;
2183 struct net_device *ndev = port->ndev;
2184 u32 mac_control;
2185 int tmo;
2186
2187 /* disable forwarding */
2188 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
2189
2190 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
2191
2192 tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
2193 dev_dbg(common->dev, "down msc_sl %08x tmo %d\n",
2194 cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo);
2195
2196 /* All the bits that am65_cpsw_nuss_mac_link_up() can possibly set */
2197 mac_control = CPSW_SL_CTL_GMII_EN | CPSW_SL_CTL_GIG | CPSW_SL_CTL_IFCTL_A |
2198 CPSW_SL_CTL_FULLDUPLEX | CPSW_SL_CTL_RX_FLOW_EN | CPSW_SL_CTL_TX_FLOW_EN;
2199 /* If interface mode is RGMII, CPSW_SL_CTL_EXT_EN might have been set for 10 Mbps */
2200 if (phy_interface_mode_is_rgmii(interface))
2201 mac_control |= CPSW_SL_CTL_EXT_EN;
2202 /* Only clear those bits that can be set by am65_cpsw_nuss_mac_link_up() */
2203 cpsw_sl_ctl_clr(port->slave.mac_sl, mac_control);
2204
2205 am65_cpsw_qos_link_down(ndev);
2206 netif_tx_stop_all_queues(ndev);
2207 }
2208
am65_cpsw_nuss_mac_link_up(struct phylink_config * config,struct phy_device * phy,unsigned int mode,phy_interface_t interface,int speed,int duplex,bool tx_pause,bool rx_pause)2209 static void am65_cpsw_nuss_mac_link_up(struct phylink_config *config, struct phy_device *phy,
2210 unsigned int mode, phy_interface_t interface, int speed,
2211 int duplex, bool tx_pause, bool rx_pause)
2212 {
2213 struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
2214 phylink_config);
2215 struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
2216 struct am65_cpsw_common *common = port->common;
2217 u32 mac_control = CPSW_SL_CTL_GMII_EN;
2218 struct net_device *ndev = port->ndev;
2219
2220 /* Bring the port out of idle state */
2221 cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
2222
2223 if (speed == SPEED_1000)
2224 mac_control |= CPSW_SL_CTL_GIG;
2225 /* TODO: Verify whether in-band is necessary for 10 Mbps RGMII */
2226 if (speed == SPEED_10 && phy_interface_mode_is_rgmii(interface))
2227 /* Can be used with in band mode only */
2228 mac_control |= CPSW_SL_CTL_EXT_EN;
2229 if (speed == SPEED_100 && interface == PHY_INTERFACE_MODE_RMII)
2230 mac_control |= CPSW_SL_CTL_IFCTL_A;
2231 if (duplex)
2232 mac_control |= CPSW_SL_CTL_FULLDUPLEX;
2233
2234 /* rx_pause/tx_pause */
2235 if (rx_pause)
2236 mac_control |= CPSW_SL_CTL_TX_FLOW_EN;
2237
2238 if (tx_pause)
2239 mac_control |= CPSW_SL_CTL_RX_FLOW_EN;
2240
2241 cpsw_sl_ctl_set(port->slave.mac_sl, mac_control);
2242
2243 /* enable forwarding */
2244 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
2245
2246 am65_cpsw_qos_link_up(ndev, speed);
2247 netif_tx_wake_all_queues(ndev);
2248 }
2249
2250 static const struct phylink_mac_ops am65_cpsw_phylink_mac_ops = {
2251 .mac_config = am65_cpsw_nuss_mac_config,
2252 .mac_link_down = am65_cpsw_nuss_mac_link_down,
2253 .mac_link_up = am65_cpsw_nuss_mac_link_up,
2254 };
2255
am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port * port)2256 static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port)
2257 {
2258 struct am65_cpsw_common *common = port->common;
2259
2260 if (!port->disabled)
2261 return;
2262
2263 cpsw_ale_control_set(common->ale, port->port_id,
2264 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
2265
2266 cpsw_sl_reset(port->slave.mac_sl, 100);
2267 cpsw_sl_ctl_reset(port->slave.mac_sl);
2268 }
2269
am65_cpsw_nuss_free_tx_chns(void * data)2270 static void am65_cpsw_nuss_free_tx_chns(void *data)
2271 {
2272 struct am65_cpsw_common *common = data;
2273 int i;
2274
2275 for (i = 0; i < common->tx_ch_num; i++) {
2276 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2277
2278 if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
2279 k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
2280
2281 if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
2282 k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
2283
2284 memset(tx_chn, 0, sizeof(*tx_chn));
2285 }
2286 }
2287
am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common * common)2288 static void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common)
2289 {
2290 struct device *dev = common->dev;
2291 int i;
2292
2293 common->tx_ch_rate_msk = 0;
2294 for (i = 0; i < common->tx_ch_num; i++) {
2295 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2296
2297 if (tx_chn->irq > 0)
2298 devm_free_irq(dev, tx_chn->irq, tx_chn);
2299
2300 netif_napi_del(&tx_chn->napi_tx);
2301 }
2302
2303 am65_cpsw_nuss_free_tx_chns(common);
2304 }
2305
am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common * common)2306 static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common)
2307 {
2308 struct device *dev = common->dev;
2309 int i, ret = 0;
2310
2311 for (i = 0; i < common->tx_ch_num; i++) {
2312 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2313
2314 hrtimer_init(&tx_chn->tx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
2315 tx_chn->tx_hrtimer.function = &am65_cpsw_nuss_tx_timer_callback;
2316
2317 ret = devm_request_irq(dev, tx_chn->irq,
2318 am65_cpsw_nuss_tx_irq,
2319 IRQF_TRIGGER_HIGH,
2320 tx_chn->tx_chn_name, tx_chn);
2321 if (ret) {
2322 dev_err(dev, "failure requesting tx%u irq %u, %d\n",
2323 tx_chn->id, tx_chn->irq, ret);
2324 goto err;
2325 }
2326
2327 netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx,
2328 am65_cpsw_nuss_tx_poll);
2329 }
2330
2331 return 0;
2332
2333 err:
2334 for (--i ; i >= 0 ; i--) {
2335 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2336
2337 netif_napi_del(&tx_chn->napi_tx);
2338 devm_free_irq(dev, tx_chn->irq, tx_chn);
2339 }
2340
2341 return ret;
2342 }
2343
am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common * common)2344 static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
2345 {
2346 u32 max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
2347 struct k3_udma_glue_tx_channel_cfg tx_cfg = { 0 };
2348 struct device *dev = common->dev;
2349 struct k3_ring_cfg ring_cfg = {
2350 .elm_size = K3_RINGACC_RING_ELSIZE_8,
2351 .mode = K3_RINGACC_RING_MODE_RING,
2352 .flags = 0
2353 };
2354 u32 hdesc_size, hdesc_size_out;
2355 int i, ret = 0;
2356
2357 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
2358 AM65_CPSW_NAV_SW_DATA_SIZE);
2359
2360 tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
2361 tx_cfg.tx_cfg = ring_cfg;
2362 tx_cfg.txcq_cfg = ring_cfg;
2363 tx_cfg.tx_cfg.size = max_desc_num;
2364 tx_cfg.txcq_cfg.size = max_desc_num;
2365
2366 for (i = 0; i < common->tx_ch_num; i++) {
2367 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2368
2369 snprintf(tx_chn->tx_chn_name,
2370 sizeof(tx_chn->tx_chn_name), "tx%d", i);
2371
2372 spin_lock_init(&tx_chn->lock);
2373 tx_chn->common = common;
2374 tx_chn->id = i;
2375 tx_chn->descs_num = max_desc_num;
2376
2377 tx_chn->tx_chn =
2378 k3_udma_glue_request_tx_chn(dev,
2379 tx_chn->tx_chn_name,
2380 &tx_cfg);
2381 if (IS_ERR(tx_chn->tx_chn)) {
2382 ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),
2383 "Failed to request tx dma channel\n");
2384 goto err;
2385 }
2386 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);
2387
2388 tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,
2389 tx_chn->descs_num,
2390 hdesc_size,
2391 tx_chn->tx_chn_name);
2392 if (IS_ERR(tx_chn->desc_pool)) {
2393 ret = PTR_ERR(tx_chn->desc_pool);
2394 dev_err(dev, "Failed to create poll %d\n", ret);
2395 goto err;
2396 }
2397
2398 hdesc_size_out = k3_cppi_desc_pool_desc_size(tx_chn->desc_pool);
2399 tx_chn->dsize_log2 = __fls(hdesc_size_out);
2400 WARN_ON(hdesc_size_out != (1 << tx_chn->dsize_log2));
2401
2402 tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
2403 if (tx_chn->irq < 0) {
2404 dev_err(dev, "Failed to get tx dma irq %d\n",
2405 tx_chn->irq);
2406 ret = tx_chn->irq;
2407 goto err;
2408 }
2409
2410 snprintf(tx_chn->tx_chn_name,
2411 sizeof(tx_chn->tx_chn_name), "%s-tx%d",
2412 dev_name(dev), tx_chn->id);
2413 }
2414
2415 ret = am65_cpsw_nuss_ndev_add_tx_napi(common);
2416 if (ret) {
2417 dev_err(dev, "Failed to add tx NAPI %d\n", ret);
2418 goto err;
2419 }
2420
2421 return 0;
2422
2423 err:
2424 am65_cpsw_nuss_free_tx_chns(common);
2425
2426 return ret;
2427 }
2428
am65_cpsw_nuss_free_rx_chns(void * data)2429 static void am65_cpsw_nuss_free_rx_chns(void *data)
2430 {
2431 struct am65_cpsw_common *common = data;
2432 struct am65_cpsw_rx_chn *rx_chn;
2433
2434 rx_chn = &common->rx_chns;
2435
2436 if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
2437 k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
2438
2439 if (!IS_ERR_OR_NULL(rx_chn->rx_chn))
2440 k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
2441 }
2442
am65_cpsw_nuss_remove_rx_chns(struct am65_cpsw_common * common)2443 static void am65_cpsw_nuss_remove_rx_chns(struct am65_cpsw_common *common)
2444 {
2445 struct device *dev = common->dev;
2446 struct am65_cpsw_rx_chn *rx_chn;
2447 struct am65_cpsw_rx_flow *flows;
2448 int i;
2449
2450 rx_chn = &common->rx_chns;
2451 flows = rx_chn->flows;
2452
2453 for (i = 0; i < common->rx_ch_num_flows; i++) {
2454 if (!(flows[i].irq < 0))
2455 devm_free_irq(dev, flows[i].irq, &flows[i]);
2456 netif_napi_del(&flows[i].napi_rx);
2457 }
2458
2459 am65_cpsw_nuss_free_rx_chns(common);
2460
2461 common->rx_flow_id_base = -1;
2462 }
2463
am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common * common)2464 static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
2465 {
2466 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
2467 struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 };
2468 u32 max_desc_num = AM65_CPSW_MAX_RX_DESC;
2469 struct device *dev = common->dev;
2470 struct am65_cpsw_rx_flow *flow;
2471 u32 hdesc_size, hdesc_size_out;
2472 u32 fdqring_id;
2473 int i, ret = 0;
2474
2475 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
2476 AM65_CPSW_NAV_SW_DATA_SIZE);
2477
2478 rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
2479 rx_cfg.flow_id_num = common->rx_ch_num_flows;
2480 rx_cfg.flow_id_base = common->rx_flow_id_base;
2481
2482 /* init all flows */
2483 rx_chn->dev = dev;
2484 rx_chn->descs_num = max_desc_num * rx_cfg.flow_id_num;
2485
2486 for (i = 0; i < common->rx_ch_num_flows; i++) {
2487 flow = &rx_chn->flows[i];
2488 flow->page_pool = NULL;
2489 }
2490
2491 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);
2492 if (IS_ERR(rx_chn->rx_chn)) {
2493 ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn),
2494 "Failed to request rx dma channel\n");
2495 goto err;
2496 }
2497 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);
2498
2499 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,
2500 rx_chn->descs_num,
2501 hdesc_size, "rx");
2502 if (IS_ERR(rx_chn->desc_pool)) {
2503 ret = PTR_ERR(rx_chn->desc_pool);
2504 dev_err(dev, "Failed to create rx poll %d\n", ret);
2505 goto err;
2506 }
2507
2508 hdesc_size_out = k3_cppi_desc_pool_desc_size(rx_chn->desc_pool);
2509 rx_chn->dsize_log2 = __fls(hdesc_size_out);
2510 WARN_ON(hdesc_size_out != (1 << rx_chn->dsize_log2));
2511
2512 common->rx_flow_id_base =
2513 k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
2514 dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
2515
2516 fdqring_id = K3_RINGACC_RING_ID_ANY;
2517 for (i = 0; i < rx_cfg.flow_id_num; i++) {
2518 struct k3_ring_cfg rxring_cfg = {
2519 .elm_size = K3_RINGACC_RING_ELSIZE_8,
2520 .mode = K3_RINGACC_RING_MODE_RING,
2521 .flags = 0,
2522 };
2523 struct k3_ring_cfg fdqring_cfg = {
2524 .elm_size = K3_RINGACC_RING_ELSIZE_8,
2525 .flags = K3_RINGACC_RING_SHARED,
2526 };
2527 struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
2528 .rx_cfg = rxring_cfg,
2529 .rxfdq_cfg = fdqring_cfg,
2530 .ring_rxq_id = K3_RINGACC_RING_ID_ANY,
2531 .src_tag_lo_sel =
2532 K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
2533 };
2534
2535 flow = &rx_chn->flows[i];
2536 flow->id = i;
2537 flow->common = common;
2538 flow->irq = -EINVAL;
2539
2540 rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
2541 rx_flow_cfg.rx_cfg.size = max_desc_num;
2542 /* share same FDQ for all flows */
2543 rx_flow_cfg.rxfdq_cfg.size = max_desc_num * rx_cfg.flow_id_num;
2544 rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode;
2545
2546 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,
2547 i, &rx_flow_cfg);
2548 if (ret) {
2549 dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
2550 goto err_flow;
2551 }
2552 if (!i)
2553 fdqring_id =
2554 k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,
2555 i);
2556
2557 flow->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);
2558 if (flow->irq <= 0) {
2559 dev_err(dev, "Failed to get rx dma irq %d\n",
2560 flow->irq);
2561 ret = flow->irq;
2562 goto err_flow;
2563 }
2564
2565 snprintf(flow->name,
2566 sizeof(flow->name), "%s-rx%d",
2567 dev_name(dev), i);
2568 hrtimer_init(&flow->rx_hrtimer, CLOCK_MONOTONIC,
2569 HRTIMER_MODE_REL_PINNED);
2570 flow->rx_hrtimer.function = &am65_cpsw_nuss_rx_timer_callback;
2571
2572 ret = devm_request_irq(dev, flow->irq,
2573 am65_cpsw_nuss_rx_irq,
2574 IRQF_TRIGGER_HIGH,
2575 flow->name, flow);
2576 if (ret) {
2577 dev_err(dev, "failure requesting rx %d irq %u, %d\n",
2578 i, flow->irq, ret);
2579 flow->irq = -EINVAL;
2580 goto err_flow;
2581 }
2582
2583 netif_napi_add(common->dma_ndev, &flow->napi_rx,
2584 am65_cpsw_nuss_rx_poll);
2585 }
2586
2587 /* setup classifier to route priorities to flows */
2588 cpsw_ale_classifier_setup_default(common->ale, common->rx_ch_num_flows);
2589
2590 return 0;
2591
2592 err_flow:
2593 for (--i; i >= 0 ; i--) {
2594 flow = &rx_chn->flows[i];
2595 netif_napi_del(&flow->napi_rx);
2596 devm_free_irq(dev, flow->irq, flow);
2597 }
2598
2599 err:
2600 am65_cpsw_nuss_free_rx_chns(common);
2601
2602 return ret;
2603 }
2604
am65_cpsw_nuss_init_host_p(struct am65_cpsw_common * common)2605 static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common)
2606 {
2607 struct am65_cpsw_host *host_p = am65_common_get_host(common);
2608
2609 host_p->common = common;
2610 host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;
2611 host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;
2612
2613 return 0;
2614 }
2615
am65_cpsw_am654_get_efuse_macid(struct device_node * of_node,int slave,u8 * mac_addr)2616 static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node,
2617 int slave, u8 *mac_addr)
2618 {
2619 u32 mac_lo, mac_hi, offset;
2620 struct regmap *syscon;
2621
2622 syscon = syscon_regmap_lookup_by_phandle_args(of_node, "ti,syscon-efuse",
2623 1, &offset);
2624 if (IS_ERR(syscon)) {
2625 if (PTR_ERR(syscon) == -ENODEV)
2626 return 0;
2627 return PTR_ERR(syscon);
2628 }
2629
2630 regmap_read(syscon, offset, &mac_lo);
2631 regmap_read(syscon, offset + 4, &mac_hi);
2632
2633 mac_addr[0] = (mac_hi >> 8) & 0xff;
2634 mac_addr[1] = mac_hi & 0xff;
2635 mac_addr[2] = (mac_lo >> 24) & 0xff;
2636 mac_addr[3] = (mac_lo >> 16) & 0xff;
2637 mac_addr[4] = (mac_lo >> 8) & 0xff;
2638 mac_addr[5] = mac_lo & 0xff;
2639
2640 return 0;
2641 }
2642
am65_cpsw_init_cpts(struct am65_cpsw_common * common)2643 static int am65_cpsw_init_cpts(struct am65_cpsw_common *common)
2644 {
2645 struct device *dev = common->dev;
2646 struct device_node *node;
2647 struct am65_cpts *cpts;
2648 void __iomem *reg_base;
2649
2650 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
2651 return 0;
2652
2653 node = of_get_child_by_name(dev->of_node, "cpts");
2654 if (!node) {
2655 dev_err(dev, "%s cpts not found\n", __func__);
2656 return -ENOENT;
2657 }
2658
2659 reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;
2660 cpts = am65_cpts_create(dev, reg_base, node);
2661 if (IS_ERR(cpts)) {
2662 int ret = PTR_ERR(cpts);
2663
2664 of_node_put(node);
2665 dev_err(dev, "cpts create err %d\n", ret);
2666 return ret;
2667 }
2668 common->cpts = cpts;
2669 /* Forbid PM runtime if CPTS is running.
2670 * K3 CPSWxG modules may completely lose context during ON->OFF
2671 * transitions depending on integration.
2672 * AM65x/J721E MCU CPSW2G: false
2673 * J721E MAIN_CPSW9G: true
2674 */
2675 pm_runtime_forbid(dev);
2676
2677 return 0;
2678 }
2679
am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common * common)2680 static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
2681 {
2682 struct device_node *node, *port_np;
2683 struct device *dev = common->dev;
2684 int ret;
2685
2686 node = of_get_child_by_name(dev->of_node, "ethernet-ports");
2687 if (!node)
2688 return -ENOENT;
2689
2690 for_each_child_of_node(node, port_np) {
2691 struct am65_cpsw_port *port;
2692 u32 port_id;
2693
2694 /* it is not a slave port node, continue */
2695 if (strcmp(port_np->name, "port"))
2696 continue;
2697
2698 ret = of_property_read_u32(port_np, "reg", &port_id);
2699 if (ret < 0) {
2700 dev_err(dev, "%pOF error reading port_id %d\n",
2701 port_np, ret);
2702 goto of_node_put;
2703 }
2704
2705 if (!port_id || port_id > common->port_num) {
2706 dev_err(dev, "%pOF has invalid port_id %u %s\n",
2707 port_np, port_id, port_np->name);
2708 ret = -EINVAL;
2709 goto of_node_put;
2710 }
2711
2712 port = am65_common_get_port(common, port_id);
2713 port->port_id = port_id;
2714 port->common = common;
2715 port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
2716 AM65_CPSW_NU_PORTS_OFFSET * (port_id);
2717 if (common->pdata.extra_modes)
2718 port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id);
2719 port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
2720 (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
2721 port->name = of_get_property(port_np, "label", NULL);
2722 port->fetch_ram_base =
2723 common->cpsw_base + AM65_CPSW_NU_FRAM_BASE +
2724 (AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1));
2725
2726 port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base);
2727 if (IS_ERR(port->slave.mac_sl)) {
2728 ret = PTR_ERR(port->slave.mac_sl);
2729 goto of_node_put;
2730 }
2731
2732 port->disabled = !of_device_is_available(port_np);
2733 if (port->disabled) {
2734 common->disabled_ports_mask |= BIT(port->port_id);
2735 continue;
2736 }
2737
2738 port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL);
2739 if (IS_ERR(port->slave.ifphy)) {
2740 ret = PTR_ERR(port->slave.ifphy);
2741 dev_err(dev, "%pOF error retrieving port phy: %d\n",
2742 port_np, ret);
2743 goto of_node_put;
2744 }
2745
2746 /* Initialize the Serdes PHY for the port */
2747 ret = am65_cpsw_init_serdes_phy(dev, port_np, port);
2748 if (ret)
2749 goto of_node_put;
2750
2751 port->slave.mac_only =
2752 of_property_read_bool(port_np, "ti,mac-only");
2753
2754 /* get phy/link info */
2755 port->slave.port_np = port_np;
2756 ret = of_get_phy_mode(port_np, &port->slave.phy_if);
2757 if (ret) {
2758 dev_err(dev, "%pOF read phy-mode err %d\n",
2759 port_np, ret);
2760 goto of_node_put;
2761 }
2762
2763 ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, port->slave.phy_if);
2764 if (ret)
2765 goto of_node_put;
2766
2767 ret = of_get_mac_address(port_np, port->slave.mac_addr);
2768 if (ret) {
2769 am65_cpsw_am654_get_efuse_macid(port_np,
2770 port->port_id,
2771 port->slave.mac_addr);
2772 if (!is_valid_ether_addr(port->slave.mac_addr)) {
2773 eth_random_addr(port->slave.mac_addr);
2774 dev_err(dev, "Use random MAC address\n");
2775 }
2776 }
2777
2778 /* Reset all Queue priorities to 0 */
2779 writel(0, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP);
2780 }
2781 of_node_put(node);
2782
2783 /* is there at least one ext.port */
2784 if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) {
2785 dev_err(dev, "No Ext. port are available\n");
2786 return -ENODEV;
2787 }
2788
2789 return 0;
2790
2791 of_node_put:
2792 of_node_put(port_np);
2793 of_node_put(node);
2794 return ret;
2795 }
2796
am65_cpsw_nuss_phylink_cleanup(struct am65_cpsw_common * common)2797 static void am65_cpsw_nuss_phylink_cleanup(struct am65_cpsw_common *common)
2798 {
2799 struct am65_cpsw_port *port;
2800 int i;
2801
2802 for (i = 0; i < common->port_num; i++) {
2803 port = &common->ports[i];
2804 if (port->slave.phylink)
2805 phylink_destroy(port->slave.phylink);
2806 }
2807 }
2808
2809 static int
am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common * common,u32 port_idx)2810 am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
2811 {
2812 struct am65_cpsw_ndev_priv *ndev_priv;
2813 struct device *dev = common->dev;
2814 struct am65_cpsw_port *port;
2815 struct phylink *phylink;
2816
2817 port = &common->ports[port_idx];
2818
2819 if (port->disabled)
2820 return 0;
2821
2822 /* alloc netdev */
2823 port->ndev = alloc_etherdev_mqs(sizeof(struct am65_cpsw_ndev_priv),
2824 AM65_CPSW_MAX_QUEUES,
2825 AM65_CPSW_MAX_QUEUES);
2826 if (!port->ndev) {
2827 dev_err(dev, "error allocating slave net_device %u\n",
2828 port->port_id);
2829 return -ENOMEM;
2830 }
2831
2832 ndev_priv = netdev_priv(port->ndev);
2833 ndev_priv->port = port;
2834 ndev_priv->msg_enable = AM65_CPSW_DEBUG;
2835 mutex_init(&ndev_priv->mm_lock);
2836 port->qos.link_speed = SPEED_UNKNOWN;
2837 SET_NETDEV_DEV(port->ndev, dev);
2838 port->ndev->dev.of_node = port->slave.port_np;
2839
2840 eth_hw_addr_set(port->ndev, port->slave.mac_addr);
2841
2842 port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;
2843 port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE -
2844 (VLAN_ETH_HLEN + ETH_FCS_LEN);
2845 port->ndev->hw_features = NETIF_F_SG |
2846 NETIF_F_RXCSUM |
2847 NETIF_F_HW_CSUM |
2848 NETIF_F_HW_TC;
2849 port->ndev->features = port->ndev->hw_features |
2850 NETIF_F_HW_VLAN_CTAG_FILTER;
2851 port->ndev->xdp_features = NETDEV_XDP_ACT_BASIC |
2852 NETDEV_XDP_ACT_REDIRECT |
2853 NETDEV_XDP_ACT_NDO_XMIT;
2854 port->ndev->vlan_features |= NETIF_F_SG;
2855 port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops;
2856 port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
2857
2858 /* Configuring Phylink */
2859 port->slave.phylink_config.dev = &port->ndev->dev;
2860 port->slave.phylink_config.type = PHYLINK_NETDEV;
2861 port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 |
2862 MAC_1000FD | MAC_5000FD;
2863 port->slave.phylink_config.mac_managed_pm = true; /* MAC does PM */
2864
2865 switch (port->slave.phy_if) {
2866 case PHY_INTERFACE_MODE_RGMII:
2867 case PHY_INTERFACE_MODE_RGMII_ID:
2868 case PHY_INTERFACE_MODE_RGMII_RXID:
2869 case PHY_INTERFACE_MODE_RGMII_TXID:
2870 phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces);
2871 break;
2872
2873 case PHY_INTERFACE_MODE_RMII:
2874 __set_bit(PHY_INTERFACE_MODE_RMII,
2875 port->slave.phylink_config.supported_interfaces);
2876 break;
2877
2878 case PHY_INTERFACE_MODE_QSGMII:
2879 case PHY_INTERFACE_MODE_SGMII:
2880 case PHY_INTERFACE_MODE_USXGMII:
2881 if (common->pdata.extra_modes & BIT(port->slave.phy_if)) {
2882 __set_bit(port->slave.phy_if,
2883 port->slave.phylink_config.supported_interfaces);
2884 } else {
2885 dev_err(dev, "selected phy-mode is not supported\n");
2886 return -EOPNOTSUPP;
2887 }
2888 break;
2889
2890 default:
2891 dev_err(dev, "selected phy-mode is not supported\n");
2892 return -EOPNOTSUPP;
2893 }
2894
2895 phylink = phylink_create(&port->slave.phylink_config,
2896 of_fwnode_handle(port->slave.port_np),
2897 port->slave.phy_if,
2898 &am65_cpsw_phylink_mac_ops);
2899 if (IS_ERR(phylink))
2900 return PTR_ERR(phylink);
2901
2902 port->slave.phylink = phylink;
2903
2904 /* Disable TX checksum offload by default due to HW bug */
2905 if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)
2906 port->ndev->features &= ~NETIF_F_HW_CSUM;
2907
2908 port->ndev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
2909 port->xdp_prog = NULL;
2910
2911 if (!common->dma_ndev)
2912 common->dma_ndev = port->ndev;
2913
2914 return 0;
2915 }
2916
am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common * common)2917 static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common)
2918 {
2919 int ret;
2920 int i;
2921
2922 for (i = 0; i < common->port_num; i++) {
2923 ret = am65_cpsw_nuss_init_port_ndev(common, i);
2924 if (ret)
2925 return ret;
2926 }
2927
2928 return ret;
2929 }
2930
am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common * common)2931 static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common)
2932 {
2933 struct am65_cpsw_port *port;
2934 int i;
2935
2936 for (i = 0; i < common->port_num; i++) {
2937 port = &common->ports[i];
2938 if (!port->ndev)
2939 continue;
2940 if (port->ndev->reg_state == NETREG_REGISTERED)
2941 unregister_netdev(port->ndev);
2942 free_netdev(port->ndev);
2943 port->ndev = NULL;
2944 }
2945 }
2946
am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common * common)2947 static void am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common *common)
2948 {
2949 int set_val = 0;
2950 int i;
2951
2952 if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask))
2953 set_val = 1;
2954
2955 dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val);
2956
2957 for (i = 1; i <= common->port_num; i++) {
2958 struct am65_cpsw_port *port = am65_common_get_port(common, i);
2959 struct am65_cpsw_ndev_priv *priv;
2960
2961 if (!port->ndev)
2962 continue;
2963
2964 priv = am65_ndev_to_priv(port->ndev);
2965 priv->offload_fwd_mark = set_val;
2966 }
2967 }
2968
am65_cpsw_port_dev_check(const struct net_device * ndev)2969 bool am65_cpsw_port_dev_check(const struct net_device *ndev)
2970 {
2971 if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) {
2972 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2973
2974 return !common->is_emac_mode;
2975 }
2976
2977 return false;
2978 }
2979
am65_cpsw_netdevice_port_link(struct net_device * ndev,struct net_device * br_ndev,struct netlink_ext_ack * extack)2980 static int am65_cpsw_netdevice_port_link(struct net_device *ndev,
2981 struct net_device *br_ndev,
2982 struct netlink_ext_ack *extack)
2983 {
2984 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2985 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2986 int err;
2987
2988 if (!common->br_members) {
2989 common->hw_bridge_dev = br_ndev;
2990 } else {
2991 /* This is adding the port to a second bridge, this is
2992 * unsupported
2993 */
2994 if (common->hw_bridge_dev != br_ndev)
2995 return -EOPNOTSUPP;
2996 }
2997
2998 err = switchdev_bridge_port_offload(ndev, ndev, NULL, NULL, NULL,
2999 false, extack);
3000 if (err)
3001 return err;
3002
3003 common->br_members |= BIT(priv->port->port_id);
3004
3005 am65_cpsw_port_offload_fwd_mark_update(common);
3006
3007 return NOTIFY_DONE;
3008 }
3009
am65_cpsw_netdevice_port_unlink(struct net_device * ndev)3010 static void am65_cpsw_netdevice_port_unlink(struct net_device *ndev)
3011 {
3012 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
3013 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
3014
3015 switchdev_bridge_port_unoffload(ndev, NULL, NULL, NULL);
3016
3017 common->br_members &= ~BIT(priv->port->port_id);
3018
3019 am65_cpsw_port_offload_fwd_mark_update(common);
3020
3021 if (!common->br_members)
3022 common->hw_bridge_dev = NULL;
3023 }
3024
3025 /* netdev notifier */
am65_cpsw_netdevice_event(struct notifier_block * unused,unsigned long event,void * ptr)3026 static int am65_cpsw_netdevice_event(struct notifier_block *unused,
3027 unsigned long event, void *ptr)
3028 {
3029 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
3030 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
3031 struct netdev_notifier_changeupper_info *info;
3032 int ret = NOTIFY_DONE;
3033
3034 if (!am65_cpsw_port_dev_check(ndev))
3035 return NOTIFY_DONE;
3036
3037 switch (event) {
3038 case NETDEV_CHANGEUPPER:
3039 info = ptr;
3040
3041 if (netif_is_bridge_master(info->upper_dev)) {
3042 if (info->linking)
3043 ret = am65_cpsw_netdevice_port_link(ndev,
3044 info->upper_dev,
3045 extack);
3046 else
3047 am65_cpsw_netdevice_port_unlink(ndev);
3048 }
3049 break;
3050 default:
3051 return NOTIFY_DONE;
3052 }
3053
3054 return notifier_from_errno(ret);
3055 }
3056
am65_cpsw_register_notifiers(struct am65_cpsw_common * cpsw)3057 static int am65_cpsw_register_notifiers(struct am65_cpsw_common *cpsw)
3058 {
3059 int ret = 0;
3060
3061 if (AM65_CPSW_IS_CPSW2G(cpsw) ||
3062 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
3063 return 0;
3064
3065 cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event;
3066 ret = register_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
3067 if (ret) {
3068 dev_err(cpsw->dev, "can't register netdevice notifier\n");
3069 return ret;
3070 }
3071
3072 ret = am65_cpsw_switchdev_register_notifiers(cpsw);
3073 if (ret)
3074 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
3075
3076 return ret;
3077 }
3078
am65_cpsw_unregister_notifiers(struct am65_cpsw_common * cpsw)3079 static void am65_cpsw_unregister_notifiers(struct am65_cpsw_common *cpsw)
3080 {
3081 if (AM65_CPSW_IS_CPSW2G(cpsw) ||
3082 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
3083 return;
3084
3085 am65_cpsw_switchdev_unregister_notifiers(cpsw);
3086 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
3087 }
3088
3089 static const struct devlink_ops am65_cpsw_devlink_ops = {};
3090
am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common * cpsw)3091 static void am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common *cpsw)
3092 {
3093 cpsw_ale_add_mcast(cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, 0,
3094 ALE_MCAST_BLOCK_LEARN_FWD);
3095 }
3096
am65_cpsw_init_host_port_switch(struct am65_cpsw_common * common)3097 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common)
3098 {
3099 struct am65_cpsw_host *host = am65_common_get_host(common);
3100
3101 writel(common->default_vlan, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3102
3103 am65_cpsw_init_stp_ale_entry(common);
3104
3105 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 1);
3106 dev_dbg(common->dev, "Set P0_UNI_FLOOD\n");
3107 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 0);
3108 }
3109
am65_cpsw_init_host_port_emac(struct am65_cpsw_common * common)3110 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common)
3111 {
3112 struct am65_cpsw_host *host = am65_common_get_host(common);
3113
3114 writel(0, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3115
3116 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 0);
3117 dev_dbg(common->dev, "unset P0_UNI_FLOOD\n");
3118
3119 /* learning make no sense in multi-mac mode */
3120 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 1);
3121 }
3122
am65_cpsw_dl_switch_mode_get(struct devlink * dl,u32 id,struct devlink_param_gset_ctx * ctx)3123 static int am65_cpsw_dl_switch_mode_get(struct devlink *dl, u32 id,
3124 struct devlink_param_gset_ctx *ctx)
3125 {
3126 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
3127 struct am65_cpsw_common *common = dl_priv->common;
3128
3129 dev_dbg(common->dev, "%s id:%u\n", __func__, id);
3130
3131 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
3132 return -EOPNOTSUPP;
3133
3134 ctx->val.vbool = !common->is_emac_mode;
3135
3136 return 0;
3137 }
3138
am65_cpsw_init_port_emac_ale(struct am65_cpsw_port * port)3139 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port)
3140 {
3141 struct am65_cpsw_slave_data *slave = &port->slave;
3142 struct am65_cpsw_common *common = port->common;
3143 u32 port_mask;
3144
3145 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3146
3147 if (slave->mac_only)
3148 /* enable mac-only mode on port */
3149 cpsw_ale_control_set(common->ale, port->port_id,
3150 ALE_PORT_MACONLY, 1);
3151
3152 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_NOLEARN, 1);
3153
3154 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
3155
3156 cpsw_ale_add_ucast(common->ale, port->ndev->dev_addr,
3157 HOST_PORT_NUM, ALE_SECURE, slave->port_vlan);
3158 cpsw_ale_add_mcast(common->ale, port->ndev->broadcast,
3159 port_mask, ALE_VLAN, slave->port_vlan, ALE_MCAST_FWD_2);
3160 }
3161
am65_cpsw_init_port_switch_ale(struct am65_cpsw_port * port)3162 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port)
3163 {
3164 struct am65_cpsw_slave_data *slave = &port->slave;
3165 struct am65_cpsw_common *cpsw = port->common;
3166 u32 port_mask;
3167
3168 cpsw_ale_control_set(cpsw->ale, port->port_id,
3169 ALE_PORT_NOLEARN, 0);
3170
3171 cpsw_ale_add_ucast(cpsw->ale, port->ndev->dev_addr,
3172 HOST_PORT_NUM, ALE_SECURE | ALE_BLOCKED | ALE_VLAN,
3173 slave->port_vlan);
3174
3175 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
3176
3177 cpsw_ale_add_mcast(cpsw->ale, port->ndev->broadcast,
3178 port_mask, ALE_VLAN, slave->port_vlan,
3179 ALE_MCAST_FWD_2);
3180
3181 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3182
3183 cpsw_ale_control_set(cpsw->ale, port->port_id,
3184 ALE_PORT_MACONLY, 0);
3185 }
3186
am65_cpsw_dl_switch_mode_set(struct devlink * dl,u32 id,struct devlink_param_gset_ctx * ctx,struct netlink_ext_ack * extack)3187 static int am65_cpsw_dl_switch_mode_set(struct devlink *dl, u32 id,
3188 struct devlink_param_gset_ctx *ctx,
3189 struct netlink_ext_ack *extack)
3190 {
3191 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
3192 struct am65_cpsw_common *cpsw = dl_priv->common;
3193 bool switch_en = ctx->val.vbool;
3194 bool if_running = false;
3195 int i;
3196
3197 dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id);
3198
3199 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
3200 return -EOPNOTSUPP;
3201
3202 if (switch_en == !cpsw->is_emac_mode)
3203 return 0;
3204
3205 if (!switch_en && cpsw->br_members) {
3206 dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n");
3207 return -EINVAL;
3208 }
3209
3210 rtnl_lock();
3211
3212 cpsw->is_emac_mode = !switch_en;
3213
3214 for (i = 0; i < cpsw->port_num; i++) {
3215 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3216
3217 if (!sl_ndev || !netif_running(sl_ndev))
3218 continue;
3219
3220 if_running = true;
3221 }
3222
3223 if (!if_running) {
3224 /* all ndevs are down */
3225 for (i = 0; i < cpsw->port_num; i++) {
3226 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3227 struct am65_cpsw_slave_data *slave;
3228
3229 if (!sl_ndev)
3230 continue;
3231
3232 slave = am65_ndev_to_slave(sl_ndev);
3233 if (switch_en)
3234 slave->port_vlan = cpsw->default_vlan;
3235 else
3236 slave->port_vlan = 0;
3237 }
3238
3239 goto exit;
3240 }
3241
3242 cpsw_ale_control_set(cpsw->ale, 0, ALE_BYPASS, 1);
3243 /* clean up ALE table */
3244 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_CLEAR, 1);
3245 cpsw_ale_control_get(cpsw->ale, HOST_PORT_NUM, ALE_AGEOUT);
3246
3247 if (switch_en) {
3248 dev_info(cpsw->dev, "Enable switch mode\n");
3249
3250 am65_cpsw_init_host_port_switch(cpsw);
3251
3252 for (i = 0; i < cpsw->port_num; i++) {
3253 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3254 struct am65_cpsw_slave_data *slave;
3255 struct am65_cpsw_port *port;
3256
3257 if (!sl_ndev)
3258 continue;
3259
3260 port = am65_ndev_to_port(sl_ndev);
3261 slave = am65_ndev_to_slave(sl_ndev);
3262 slave->port_vlan = cpsw->default_vlan;
3263
3264 if (netif_running(sl_ndev))
3265 am65_cpsw_init_port_switch_ale(port);
3266 }
3267
3268 } else {
3269 dev_info(cpsw->dev, "Disable switch mode\n");
3270
3271 am65_cpsw_init_host_port_emac(cpsw);
3272
3273 for (i = 0; i < cpsw->port_num; i++) {
3274 struct net_device *sl_ndev = cpsw->ports[i].ndev;
3275 struct am65_cpsw_port *port;
3276
3277 if (!sl_ndev)
3278 continue;
3279
3280 port = am65_ndev_to_port(sl_ndev);
3281 port->slave.port_vlan = 0;
3282 if (netif_running(sl_ndev))
3283 am65_cpsw_init_port_emac_ale(port);
3284 }
3285 }
3286 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_BYPASS, 0);
3287 exit:
3288 rtnl_unlock();
3289
3290 return 0;
3291 }
3292
3293 static const struct devlink_param am65_cpsw_devlink_params[] = {
3294 DEVLINK_PARAM_DRIVER(AM65_CPSW_DL_PARAM_SWITCH_MODE, "switch_mode",
3295 DEVLINK_PARAM_TYPE_BOOL,
3296 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
3297 am65_cpsw_dl_switch_mode_get,
3298 am65_cpsw_dl_switch_mode_set, NULL),
3299 };
3300
am65_cpsw_nuss_register_devlink(struct am65_cpsw_common * common)3301 static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common)
3302 {
3303 struct devlink_port_attrs attrs = {};
3304 struct am65_cpsw_devlink *dl_priv;
3305 struct device *dev = common->dev;
3306 struct devlink_port *dl_port;
3307 struct am65_cpsw_port *port;
3308 int ret = 0;
3309 int i;
3310
3311 common->devlink =
3312 devlink_alloc(&am65_cpsw_devlink_ops, sizeof(*dl_priv), dev);
3313 if (!common->devlink)
3314 return -ENOMEM;
3315
3316 dl_priv = devlink_priv(common->devlink);
3317 dl_priv->common = common;
3318
3319 /* Provide devlink hook to switch mode when multiple external ports
3320 * are present NUSS switchdev driver is enabled.
3321 */
3322 if (!AM65_CPSW_IS_CPSW2G(common) &&
3323 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
3324 ret = devlink_params_register(common->devlink,
3325 am65_cpsw_devlink_params,
3326 ARRAY_SIZE(am65_cpsw_devlink_params));
3327 if (ret) {
3328 dev_err(dev, "devlink params reg fail ret:%d\n", ret);
3329 goto dl_unreg;
3330 }
3331 }
3332
3333 for (i = 1; i <= common->port_num; i++) {
3334 port = am65_common_get_port(common, i);
3335 dl_port = &port->devlink_port;
3336
3337 if (port->ndev)
3338 attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
3339 else
3340 attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED;
3341 attrs.phys.port_number = port->port_id;
3342 attrs.switch_id.id_len = sizeof(resource_size_t);
3343 memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len);
3344 devlink_port_attrs_set(dl_port, &attrs);
3345
3346 ret = devlink_port_register(common->devlink, dl_port, port->port_id);
3347 if (ret) {
3348 dev_err(dev, "devlink_port reg fail for port %d, ret:%d\n",
3349 port->port_id, ret);
3350 goto dl_port_unreg;
3351 }
3352 }
3353 devlink_register(common->devlink);
3354 return ret;
3355
3356 dl_port_unreg:
3357 for (i = i - 1; i >= 1; i--) {
3358 port = am65_common_get_port(common, i);
3359 dl_port = &port->devlink_port;
3360
3361 devlink_port_unregister(dl_port);
3362 }
3363 dl_unreg:
3364 devlink_free(common->devlink);
3365 return ret;
3366 }
3367
am65_cpsw_unregister_devlink(struct am65_cpsw_common * common)3368 static void am65_cpsw_unregister_devlink(struct am65_cpsw_common *common)
3369 {
3370 struct devlink_port *dl_port;
3371 struct am65_cpsw_port *port;
3372 int i;
3373
3374 devlink_unregister(common->devlink);
3375
3376 for (i = 1; i <= common->port_num; i++) {
3377 port = am65_common_get_port(common, i);
3378 dl_port = &port->devlink_port;
3379
3380 devlink_port_unregister(dl_port);
3381 }
3382
3383 if (!AM65_CPSW_IS_CPSW2G(common) &&
3384 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
3385 devlink_params_unregister(common->devlink,
3386 am65_cpsw_devlink_params,
3387 ARRAY_SIZE(am65_cpsw_devlink_params));
3388
3389 devlink_free(common->devlink);
3390 }
3391
am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common * common)3392 static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common)
3393 {
3394 struct am65_cpsw_rx_chn *rx_chan = &common->rx_chns;
3395 struct am65_cpsw_tx_chn *tx_chan = common->tx_chns;
3396 struct device *dev = common->dev;
3397 struct am65_cpsw_port *port;
3398 int ret = 0, i;
3399
3400 /* init tx channels */
3401 ret = am65_cpsw_nuss_init_tx_chns(common);
3402 if (ret)
3403 return ret;
3404 ret = am65_cpsw_nuss_init_rx_chns(common);
3405 if (ret)
3406 goto err_remove_tx;
3407
3408 /* The DMA Channels are not guaranteed to be in a clean state.
3409 * Reset and disable them to ensure that they are back to the
3410 * clean state and ready to be used.
3411 */
3412 for (i = 0; i < common->tx_ch_num; i++) {
3413 k3_udma_glue_reset_tx_chn(tx_chan[i].tx_chn, &tx_chan[i],
3414 am65_cpsw_nuss_tx_cleanup);
3415 k3_udma_glue_disable_tx_chn(tx_chan[i].tx_chn);
3416 }
3417
3418 for (i = 0; i < common->rx_ch_num_flows; i++)
3419 k3_udma_glue_reset_rx_chn(rx_chan->rx_chn, i,
3420 rx_chan,
3421 am65_cpsw_nuss_rx_cleanup, !!i);
3422
3423 k3_udma_glue_disable_rx_chn(rx_chan->rx_chn);
3424
3425 ret = am65_cpsw_nuss_register_devlink(common);
3426 if (ret)
3427 goto err_remove_rx;
3428
3429 for (i = 0; i < common->port_num; i++) {
3430 port = &common->ports[i];
3431
3432 if (!port->ndev)
3433 continue;
3434
3435 SET_NETDEV_DEVLINK_PORT(port->ndev, &port->devlink_port);
3436
3437 ret = register_netdev(port->ndev);
3438 if (ret) {
3439 dev_err(dev, "error registering slave net device%i %d\n",
3440 i, ret);
3441 goto err_cleanup_ndev;
3442 }
3443 }
3444
3445 ret = am65_cpsw_register_notifiers(common);
3446 if (ret)
3447 goto err_cleanup_ndev;
3448
3449 /* can't auto unregister ndev using devm_add_action() due to
3450 * devres release sequence in DD core for DMA
3451 */
3452
3453 return 0;
3454
3455 err_cleanup_ndev:
3456 am65_cpsw_nuss_cleanup_ndev(common);
3457 am65_cpsw_unregister_devlink(common);
3458 err_remove_rx:
3459 am65_cpsw_nuss_remove_rx_chns(common);
3460 err_remove_tx:
3461 am65_cpsw_nuss_remove_tx_chns(common);
3462
3463 return ret;
3464 }
3465
am65_cpsw_nuss_update_tx_rx_chns(struct am65_cpsw_common * common,int num_tx,int num_rx)3466 int am65_cpsw_nuss_update_tx_rx_chns(struct am65_cpsw_common *common,
3467 int num_tx, int num_rx)
3468 {
3469 int ret;
3470
3471 am65_cpsw_nuss_remove_tx_chns(common);
3472 am65_cpsw_nuss_remove_rx_chns(common);
3473
3474 common->tx_ch_num = num_tx;
3475 common->rx_ch_num_flows = num_rx;
3476 ret = am65_cpsw_nuss_init_tx_chns(common);
3477 if (ret)
3478 return ret;
3479
3480 ret = am65_cpsw_nuss_init_rx_chns(common);
3481 if (ret)
3482 am65_cpsw_nuss_remove_tx_chns(common);
3483
3484 return ret;
3485 }
3486
3487 struct am65_cpsw_soc_pdata {
3488 u32 quirks_dis;
3489 };
3490
3491 static const struct am65_cpsw_soc_pdata am65x_soc_sr2_0 = {
3492 .quirks_dis = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
3493 };
3494
3495 static const struct soc_device_attribute am65_cpsw_socinfo[] = {
3496 { .family = "AM65X",
3497 .revision = "SR2.0",
3498 .data = &am65x_soc_sr2_0
3499 },
3500 {/* sentinel */}
3501 };
3502
3503 static const struct am65_cpsw_pdata am65x_sr1_0 = {
3504 .quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
3505 .ale_dev_id = "am65x-cpsw2g",
3506 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3507 };
3508
3509 static const struct am65_cpsw_pdata j721e_pdata = {
3510 .quirks = 0,
3511 .ale_dev_id = "am65x-cpsw2g",
3512 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3513 };
3514
3515 static const struct am65_cpsw_pdata am64x_cpswxg_pdata = {
3516 .quirks = AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ,
3517 .ale_dev_id = "am64-cpswxg",
3518 .fdqring_mode = K3_RINGACC_RING_MODE_RING,
3519 };
3520
3521 static const struct am65_cpsw_pdata j7200_cpswxg_pdata = {
3522 .quirks = 0,
3523 .ale_dev_id = "am64-cpswxg",
3524 .fdqring_mode = K3_RINGACC_RING_MODE_RING,
3525 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII) |
3526 BIT(PHY_INTERFACE_MODE_USXGMII),
3527 };
3528
3529 static const struct am65_cpsw_pdata j721e_cpswxg_pdata = {
3530 .quirks = 0,
3531 .ale_dev_id = "am64-cpswxg",
3532 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3533 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
3534 };
3535
3536 static const struct am65_cpsw_pdata j784s4_cpswxg_pdata = {
3537 .quirks = 0,
3538 .ale_dev_id = "am64-cpswxg",
3539 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
3540 .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII) |
3541 BIT(PHY_INTERFACE_MODE_USXGMII),
3542 };
3543
3544 static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
3545 { .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
3546 { .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
3547 { .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
3548 { .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
3549 { .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata},
3550 { .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata},
3551 { /* sentinel */ },
3552 };
3553 MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
3554
am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common * common)3555 static void am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common *common)
3556 {
3557 const struct soc_device_attribute *soc;
3558
3559 soc = soc_device_match(am65_cpsw_socinfo);
3560 if (soc && soc->data) {
3561 const struct am65_cpsw_soc_pdata *socdata = soc->data;
3562
3563 /* disable quirks */
3564 common->pdata.quirks &= ~socdata->quirks_dis;
3565 }
3566 }
3567
am65_cpsw_nuss_probe(struct platform_device * pdev)3568 static int am65_cpsw_nuss_probe(struct platform_device *pdev)
3569 {
3570 struct cpsw_ale_params ale_params = { 0 };
3571 const struct of_device_id *of_id;
3572 struct device *dev = &pdev->dev;
3573 struct am65_cpsw_common *common;
3574 struct device_node *node;
3575 struct resource *res;
3576 struct clk *clk;
3577 int ale_entries;
3578 __be64 id_temp;
3579 int ret, i;
3580
3581 common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
3582 if (!common)
3583 return -ENOMEM;
3584 common->dev = dev;
3585
3586 of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev);
3587 if (!of_id)
3588 return -EINVAL;
3589 common->pdata = *(const struct am65_cpsw_pdata *)of_id->data;
3590
3591 am65_cpsw_nuss_apply_socinfo(common);
3592
3593 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss");
3594 common->ss_base = devm_ioremap_resource(&pdev->dev, res);
3595 if (IS_ERR(common->ss_base))
3596 return PTR_ERR(common->ss_base);
3597 common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;
3598 /* Use device's physical base address as switch id */
3599 id_temp = cpu_to_be64(res->start);
3600 memcpy(common->switch_id, &id_temp, sizeof(res->start));
3601
3602 node = of_get_child_by_name(dev->of_node, "ethernet-ports");
3603 if (!node)
3604 return -ENOENT;
3605 common->port_num = of_get_child_count(node);
3606 of_node_put(node);
3607 if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS)
3608 return -ENOENT;
3609
3610 common->rx_flow_id_base = -1;
3611 init_completion(&common->tdown_complete);
3612 common->tx_ch_num = AM65_CPSW_DEFAULT_TX_CHNS;
3613 common->rx_ch_num_flows = AM65_CPSW_DEFAULT_RX_CHN_FLOWS;
3614 common->pf_p0_rx_ptype_rrobin = true;
3615 common->default_vlan = 1;
3616
3617 common->ports = devm_kcalloc(dev, common->port_num,
3618 sizeof(*common->ports),
3619 GFP_KERNEL);
3620 if (!common->ports)
3621 return -ENOMEM;
3622
3623 clk = devm_clk_get(dev, "fck");
3624 if (IS_ERR(clk))
3625 return dev_err_probe(dev, PTR_ERR(clk), "getting fck clock\n");
3626 common->bus_freq = clk_get_rate(clk);
3627
3628 pm_runtime_enable(dev);
3629 ret = pm_runtime_resume_and_get(dev);
3630 if (ret < 0) {
3631 pm_runtime_disable(dev);
3632 return ret;
3633 }
3634
3635 node = of_get_child_by_name(dev->of_node, "mdio");
3636 if (!node) {
3637 dev_warn(dev, "MDIO node not found\n");
3638 } else if (of_device_is_available(node)) {
3639 struct platform_device *mdio_pdev;
3640
3641 mdio_pdev = of_platform_device_create(node, NULL, dev);
3642 if (!mdio_pdev) {
3643 ret = -ENODEV;
3644 goto err_pm_clear;
3645 }
3646
3647 common->mdio_dev = &mdio_pdev->dev;
3648 }
3649 of_node_put(node);
3650
3651 am65_cpsw_nuss_get_ver(common);
3652
3653 ret = am65_cpsw_nuss_init_host_p(common);
3654 if (ret)
3655 goto err_of_clear;
3656
3657 ret = am65_cpsw_nuss_init_slave_ports(common);
3658 if (ret)
3659 goto err_of_clear;
3660
3661 /* init common data */
3662 ale_params.dev = dev;
3663 ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT;
3664 ale_params.ale_ports = common->port_num + 1;
3665 ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;
3666 ale_params.dev_id = common->pdata.ale_dev_id;
3667 ale_params.bus_freq = common->bus_freq;
3668
3669 common->ale = cpsw_ale_create(&ale_params);
3670 if (IS_ERR(common->ale)) {
3671 dev_err(dev, "error initializing ale engine\n");
3672 ret = PTR_ERR(common->ale);
3673 goto err_of_clear;
3674 }
3675
3676 ale_entries = common->ale->params.ale_entries;
3677 common->ale_context = devm_kzalloc(dev,
3678 ale_entries * ALE_ENTRY_WORDS * sizeof(u32),
3679 GFP_KERNEL);
3680 ret = am65_cpsw_init_cpts(common);
3681 if (ret)
3682 goto err_of_clear;
3683
3684 /* init ports */
3685 for (i = 0; i < common->port_num; i++)
3686 am65_cpsw_nuss_slave_disable_unused(&common->ports[i]);
3687
3688 dev_set_drvdata(dev, common);
3689
3690 common->is_emac_mode = true;
3691
3692 ret = am65_cpsw_nuss_init_ndevs(common);
3693 if (ret)
3694 goto err_ndevs_clear;
3695
3696 ret = am65_cpsw_nuss_register_ndevs(common);
3697 if (ret)
3698 goto err_ndevs_clear;
3699
3700 pm_runtime_put(dev);
3701 return 0;
3702
3703 err_ndevs_clear:
3704 am65_cpsw_nuss_cleanup_ndev(common);
3705 am65_cpsw_nuss_phylink_cleanup(common);
3706 am65_cpts_release(common->cpts);
3707 err_of_clear:
3708 if (common->mdio_dev)
3709 of_platform_device_destroy(common->mdio_dev, NULL);
3710 err_pm_clear:
3711 pm_runtime_put_sync(dev);
3712 pm_runtime_disable(dev);
3713 return ret;
3714 }
3715
am65_cpsw_nuss_remove(struct platform_device * pdev)3716 static void am65_cpsw_nuss_remove(struct platform_device *pdev)
3717 {
3718 struct device *dev = &pdev->dev;
3719 struct am65_cpsw_common *common;
3720 int ret;
3721
3722 common = dev_get_drvdata(dev);
3723
3724 ret = pm_runtime_resume_and_get(&pdev->dev);
3725 if (ret < 0) {
3726 /* Note, if this error path is taken, we're leaking some
3727 * resources.
3728 */
3729 dev_err(&pdev->dev, "Failed to resume device (%pe)\n",
3730 ERR_PTR(ret));
3731 return;
3732 }
3733
3734 am65_cpsw_unregister_notifiers(common);
3735
3736 /* must unregister ndevs here because DD release_driver routine calls
3737 * dma_deconfigure(dev) before devres_release_all(dev)
3738 */
3739 am65_cpsw_nuss_cleanup_ndev(common);
3740 am65_cpsw_unregister_devlink(common);
3741 am65_cpsw_nuss_remove_rx_chns(common);
3742 am65_cpsw_nuss_remove_tx_chns(common);
3743 am65_cpsw_nuss_phylink_cleanup(common);
3744 am65_cpts_release(common->cpts);
3745 am65_cpsw_disable_serdes_phy(common);
3746
3747 if (common->mdio_dev)
3748 of_platform_device_destroy(common->mdio_dev, NULL);
3749
3750 pm_runtime_put_sync(&pdev->dev);
3751 pm_runtime_disable(&pdev->dev);
3752 }
3753
am65_cpsw_nuss_suspend(struct device * dev)3754 static int am65_cpsw_nuss_suspend(struct device *dev)
3755 {
3756 struct am65_cpsw_common *common = dev_get_drvdata(dev);
3757 struct am65_cpsw_host *host_p = am65_common_get_host(common);
3758 struct am65_cpsw_port *port;
3759 struct net_device *ndev;
3760 int i, ret;
3761
3762 cpsw_ale_dump(common->ale, common->ale_context);
3763 host_p->vid_context = readl(host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3764 for (i = 0; i < common->port_num; i++) {
3765 port = &common->ports[i];
3766 ndev = port->ndev;
3767
3768 if (!ndev)
3769 continue;
3770
3771 port->vid_context = readl(port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3772 netif_device_detach(ndev);
3773 if (netif_running(ndev)) {
3774 rtnl_lock();
3775 ret = am65_cpsw_nuss_ndo_slave_stop(ndev);
3776 rtnl_unlock();
3777 if (ret < 0) {
3778 netdev_err(ndev, "failed to stop: %d", ret);
3779 return ret;
3780 }
3781 }
3782 }
3783
3784 am65_cpts_suspend(common->cpts);
3785
3786 am65_cpsw_nuss_remove_rx_chns(common);
3787 am65_cpsw_nuss_remove_tx_chns(common);
3788
3789 return 0;
3790 }
3791
am65_cpsw_nuss_resume(struct device * dev)3792 static int am65_cpsw_nuss_resume(struct device *dev)
3793 {
3794 struct am65_cpsw_common *common = dev_get_drvdata(dev);
3795 struct am65_cpsw_host *host_p = am65_common_get_host(common);
3796 struct am65_cpsw_port *port;
3797 struct net_device *ndev;
3798 int i, ret;
3799
3800 ret = am65_cpsw_nuss_init_tx_chns(common);
3801 if (ret)
3802 return ret;
3803 ret = am65_cpsw_nuss_init_rx_chns(common);
3804 if (ret) {
3805 am65_cpsw_nuss_remove_tx_chns(common);
3806 return ret;
3807 }
3808
3809 /* If RX IRQ was disabled before suspend, keep it disabled */
3810 for (i = 0; i < common->rx_ch_num_flows; i++) {
3811 if (common->rx_chns.flows[i].irq_disabled)
3812 disable_irq(common->rx_chns.flows[i].irq);
3813 }
3814
3815 am65_cpts_resume(common->cpts);
3816
3817 for (i = 0; i < common->port_num; i++) {
3818 port = &common->ports[i];
3819 ndev = port->ndev;
3820
3821 if (!ndev)
3822 continue;
3823
3824 if (netif_running(ndev)) {
3825 rtnl_lock();
3826 ret = am65_cpsw_nuss_ndo_slave_open(ndev);
3827 rtnl_unlock();
3828 if (ret < 0) {
3829 netdev_err(ndev, "failed to start: %d", ret);
3830 return ret;
3831 }
3832 }
3833
3834 netif_device_attach(ndev);
3835 writel(port->vid_context, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3836 }
3837
3838 writel(host_p->vid_context, host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3839 cpsw_ale_restore(common->ale, common->ale_context);
3840
3841 return 0;
3842 }
3843
3844 static const struct dev_pm_ops am65_cpsw_nuss_dev_pm_ops = {
3845 SYSTEM_SLEEP_PM_OPS(am65_cpsw_nuss_suspend, am65_cpsw_nuss_resume)
3846 };
3847
3848 static struct platform_driver am65_cpsw_nuss_driver = {
3849 .driver = {
3850 .name = AM65_CPSW_DRV_NAME,
3851 .of_match_table = am65_cpsw_nuss_of_mtable,
3852 .pm = &am65_cpsw_nuss_dev_pm_ops,
3853 },
3854 .probe = am65_cpsw_nuss_probe,
3855 .remove = am65_cpsw_nuss_remove,
3856 };
3857
3858 module_platform_driver(am65_cpsw_nuss_driver);
3859
3860 MODULE_LICENSE("GPL v2");
3861 MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
3862 MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");
3863