1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2024 AIROHA Inc
4 * Author: Lorenzo Bianconi <lorenzo@kernel.org>
5 */
6 #include <linux/of.h>
7 #include <linux/of_net.h>
8 #include <linux/of_reserved_mem.h>
9 #include <linux/platform_device.h>
10 #include <linux/tcp.h>
11 #include <linux/u64_stats_sync.h>
12 #include <net/dst_metadata.h>
13 #include <net/page_pool/helpers.h>
14 #include <net/pkt_cls.h>
15 #include <uapi/linux/ppp_defs.h>
16
17 #include "airoha_regs.h"
18 #include "airoha_eth.h"
19
airoha_rr(void __iomem * base,u32 offset)20 u32 airoha_rr(void __iomem *base, u32 offset)
21 {
22 return readl(base + offset);
23 }
24
airoha_wr(void __iomem * base,u32 offset,u32 val)25 void airoha_wr(void __iomem *base, u32 offset, u32 val)
26 {
27 writel(val, base + offset);
28 }
29
airoha_rmw(void __iomem * base,u32 offset,u32 mask,u32 val)30 u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val)
31 {
32 val |= (airoha_rr(base, offset) & ~mask);
33 airoha_wr(base, offset, val);
34
35 return val;
36 }
37
airoha_qdma_set_irqmask(struct airoha_irq_bank * irq_bank,int index,u32 clear,u32 set)38 static void airoha_qdma_set_irqmask(struct airoha_irq_bank *irq_bank,
39 int index, u32 clear, u32 set)
40 {
41 struct airoha_qdma *qdma = irq_bank->qdma;
42 int bank = irq_bank - &qdma->irq_banks[0];
43 unsigned long flags;
44
45 if (WARN_ON_ONCE(index >= ARRAY_SIZE(irq_bank->irqmask)))
46 return;
47
48 spin_lock_irqsave(&irq_bank->irq_lock, flags);
49
50 irq_bank->irqmask[index] &= ~clear;
51 irq_bank->irqmask[index] |= set;
52 airoha_qdma_wr(qdma, REG_INT_ENABLE(bank, index),
53 irq_bank->irqmask[index]);
54 /* Read irq_enable register in order to guarantee the update above
55 * completes in the spinlock critical section.
56 */
57 airoha_qdma_rr(qdma, REG_INT_ENABLE(bank, index));
58
59 spin_unlock_irqrestore(&irq_bank->irq_lock, flags);
60 }
61
airoha_qdma_irq_enable(struct airoha_irq_bank * irq_bank,int index,u32 mask)62 static void airoha_qdma_irq_enable(struct airoha_irq_bank *irq_bank,
63 int index, u32 mask)
64 {
65 airoha_qdma_set_irqmask(irq_bank, index, 0, mask);
66 }
67
airoha_qdma_irq_disable(struct airoha_irq_bank * irq_bank,int index,u32 mask)68 static void airoha_qdma_irq_disable(struct airoha_irq_bank *irq_bank,
69 int index, u32 mask)
70 {
71 airoha_qdma_set_irqmask(irq_bank, index, mask, 0);
72 }
73
airoha_set_macaddr(struct airoha_gdm_dev * dev,const u8 * addr)74 static int airoha_set_macaddr(struct airoha_gdm_dev *dev, const u8 *addr)
75 {
76 u8 ref_addr[ETH_ALEN] __aligned(2);
77 struct airoha_eth *eth = dev->eth;
78 u32 reg, val, lmin, lmax;
79 int i;
80
81 eth_zero_addr(ref_addr);
82 lmin = (addr[3] << 16) | (addr[4] << 8) | addr[5];
83 lmax = lmin;
84
85 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
86 struct airoha_gdm_port *port = eth->ports[i];
87 int j;
88
89 if (!port)
90 continue;
91
92 for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
93 struct airoha_gdm_dev *iter_dev;
94 struct net_device *netdev;
95
96 iter_dev = port->devs[j];
97 if (!iter_dev || iter_dev == dev)
98 continue;
99
100 if (airoha_is_lan_gdm_dev(iter_dev) !=
101 airoha_is_lan_gdm_dev(dev))
102 continue;
103
104 netdev = netdev_from_priv(iter_dev);
105 if (netdev->reg_state != NETREG_REGISTERED)
106 continue;
107
108 ether_addr_copy(ref_addr, netdev->dev_addr);
109 val = (netdev->dev_addr[3] << 16) |
110 (netdev->dev_addr[4] << 8) | netdev->dev_addr[5];
111 if (val < lmin)
112 lmin = val;
113 if (val > lmax)
114 lmax = val;
115 }
116 }
117
118 if (!is_zero_ether_addr(ref_addr) && memcmp(ref_addr, addr, 3)) {
119 /* According to the HW design, hw mac address MSBs must be
120 * the same for each net_device with the same LAN/WAN
121 * configuration.
122 */
123 struct net_device *netdev = netdev_from_priv(dev);
124
125 dev_warn(eth->dev,
126 "%s: wrong mac addr, MSBs must be %02x:%02x:%02x\n",
127 netdev->name, ref_addr[0], ref_addr[1],
128 ref_addr[2]);
129 dev_warn(eth->dev, "FE hw forwarding won't work properly\n");
130
131 return -EINVAL;
132 }
133
134 reg = airoha_is_lan_gdm_dev(dev) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H;
135 val = (addr[0] << 16) | (addr[1] << 8) | addr[2];
136 airoha_fe_wr(eth, reg, val);
137
138 airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), lmin);
139 airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), lmax);
140
141 airoha_ppe_init_upd_mem(dev, addr);
142
143 return 0;
144 }
145
airoha_set_gdm_port_fwd_cfg(struct airoha_eth * eth,u32 addr,u32 val)146 static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr,
147 u32 val)
148 {
149 airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK,
150 FIELD_PREP(GDM_OCFQ_MASK, val));
151 airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK,
152 FIELD_PREP(GDM_MCFQ_MASK, val));
153 airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK,
154 FIELD_PREP(GDM_BCFQ_MASK, val));
155 airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK,
156 FIELD_PREP(GDM_UCFQ_MASK, val));
157 }
158
airoha_set_vip_for_gdm_port(struct airoha_gdm_dev * dev,bool enable)159 static int airoha_set_vip_for_gdm_port(struct airoha_gdm_dev *dev, bool enable)
160 {
161 struct airoha_gdm_port *port = dev->port;
162 struct airoha_eth *eth = dev->eth;
163 u32 vip_port;
164
165 vip_port = eth->soc->ops.get_vip_port(port, dev->nbq);
166 if (enable) {
167 airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port);
168 airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port);
169 } else {
170 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port);
171 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port);
172 }
173
174 return 0;
175 }
176
airoha_fe_maccr_init(struct airoha_eth * eth)177 static void airoha_fe_maccr_init(struct airoha_eth *eth)
178 {
179 int p;
180
181 for (p = 1; p <= ARRAY_SIZE(eth->ports); p++) {
182 airoha_fe_set(eth, REG_GDM_FWD_CFG(p),
183 GDM_TCP_CKSUM_MASK | GDM_UDP_CKSUM_MASK |
184 GDM_IP4_CKSUM_MASK | GDM_DROP_CRC_ERR_MASK);
185 airoha_fe_rmw(eth, REG_GDM_LEN_CFG(p),
186 GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK,
187 FIELD_PREP(GDM_SHORT_LEN_MASK, 60) |
188 FIELD_PREP(GDM_LONG_LEN_MASK, AIROHA_MAX_RX_SIZE));
189 }
190
191 airoha_fe_rmw(eth, REG_CDM_VLAN_CTRL(1), CDM_VLAN_MASK,
192 FIELD_PREP(CDM_VLAN_MASK, 0x8100));
193
194 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD);
195 }
196
airoha_fe_vip_setup(struct airoha_eth * eth)197 static void airoha_fe_vip_setup(struct airoha_eth *eth)
198 {
199 airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC);
200 airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK);
201
202 airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP);
203 airoha_fe_wr(eth, REG_FE_VIP_EN(4),
204 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
205 PATN_EN_MASK);
206
207 airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP);
208 airoha_fe_wr(eth, REG_FE_VIP_EN(6),
209 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
210 PATN_EN_MASK);
211
212 airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP);
213 airoha_fe_wr(eth, REG_FE_VIP_EN(7),
214 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
215 PATN_EN_MASK);
216
217 /* BOOTP (0x43) */
218 airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43);
219 airoha_fe_wr(eth, REG_FE_VIP_EN(8),
220 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
221 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
222
223 /* BOOTP (0x44) */
224 airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44);
225 airoha_fe_wr(eth, REG_FE_VIP_EN(9),
226 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
227 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
228
229 /* ISAKMP */
230 airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4);
231 airoha_fe_wr(eth, REG_FE_VIP_EN(10),
232 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
233 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
234
235 airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP);
236 airoha_fe_wr(eth, REG_FE_VIP_EN(11),
237 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
238 PATN_EN_MASK);
239
240 /* DHCPv6 */
241 airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223);
242 airoha_fe_wr(eth, REG_FE_VIP_EN(12),
243 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
244 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
245
246 airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP);
247 airoha_fe_wr(eth, REG_FE_VIP_EN(19),
248 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
249 PATN_EN_MASK);
250
251 /* ETH->ETH_P_1905 (0x893a) */
252 airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a);
253 airoha_fe_wr(eth, REG_FE_VIP_EN(20),
254 PATN_FCPU_EN_MASK | PATN_EN_MASK);
255
256 airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP);
257 airoha_fe_wr(eth, REG_FE_VIP_EN(21),
258 PATN_FCPU_EN_MASK | PATN_EN_MASK);
259 }
260
airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth * eth,u32 port,u32 queue)261 static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth,
262 u32 port, u32 queue)
263 {
264 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
265 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK,
266 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
267 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue));
268
269 return airoha_fe_get(eth, REG_FE_PSE_QUEUE_CFG_VAL,
270 PSE_CFG_OQ_RSV_MASK);
271 }
272
airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth * eth,u32 port,u32 queue,u32 val)273 static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth,
274 u32 port, u32 queue, u32 val)
275 {
276 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK,
277 FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val));
278 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
279 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK |
280 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK,
281 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
282 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) |
283 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK);
284 }
285
airoha_fe_get_pse_all_rsv(struct airoha_eth * eth)286 static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth)
287 {
288 return airoha_fe_get(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK);
289 }
290
airoha_fe_set_pse_oq_rsv(struct airoha_eth * eth,u32 port,u32 queue,u32 val)291 static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth,
292 u32 port, u32 queue, u32 val)
293 {
294 u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue);
295 u32 tmp, all_rsv, fq_limit;
296
297 airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val);
298
299 /* modify all rsv */
300 all_rsv = airoha_fe_get_pse_all_rsv(eth);
301 all_rsv += (val - orig_val);
302 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
303 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
304
305 /* modify hthd */
306 fq_limit = airoha_fe_get(eth, PSE_FQ_CFG, PSE_FQ_LIMIT_MASK);
307 tmp = fq_limit - all_rsv - 0x20;
308 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
309 PSE_SHARE_USED_HTHD_MASK,
310 FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp));
311
312 tmp = fq_limit - all_rsv - 0x100;
313 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
314 PSE_SHARE_USED_MTHD_MASK,
315 FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp));
316 tmp = (3 * tmp) >> 2;
317 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET,
318 PSE_SHARE_USED_LTHD_MASK,
319 FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp));
320
321 return 0;
322 }
323
airoha_fe_pse_ports_init(struct airoha_eth * eth)324 static void airoha_fe_pse_ports_init(struct airoha_eth *eth)
325 {
326 const u32 pse_port_num_queues[] = {
327 [FE_PSE_PORT_CDM1] = 6,
328 [FE_PSE_PORT_GDM1] = 6,
329 [FE_PSE_PORT_GDM2] = 32,
330 [FE_PSE_PORT_GDM3] = 6,
331 [FE_PSE_PORT_PPE1] = 4,
332 [FE_PSE_PORT_CDM2] = 6,
333 [FE_PSE_PORT_CDM3] = 8,
334 [FE_PSE_PORT_CDM4] = 10,
335 [FE_PSE_PORT_PPE2] = 4,
336 [FE_PSE_PORT_GDM4] = 2,
337 [FE_PSE_PORT_CDM5] = 2,
338 };
339 int q;
340
341 if (airoha_ppe_is_enabled(eth, 1)) {
342 u32 all_rsv;
343
344 /* hw misses PPE2 oq rsv */
345 all_rsv = airoha_fe_get_pse_all_rsv(eth);
346 all_rsv += PSE_RSV_PAGES *
347 pse_port_num_queues[FE_PSE_PORT_PPE2];
348 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
349 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
350 }
351
352 /* CDM1 */
353 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++)
354 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q,
355 PSE_QUEUE_RSV_PAGES);
356 /* GDM1 */
357 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++)
358 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q,
359 PSE_QUEUE_RSV_PAGES);
360 /* GDM2 */
361 for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++)
362 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0);
363 /* GDM3 */
364 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++)
365 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q,
366 PSE_QUEUE_RSV_PAGES);
367 /* PPE1 */
368 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) {
369 if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2)
370 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q,
371 PSE_QUEUE_RSV_PAGES);
372 else
373 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0);
374 }
375 /* CDM2 */
376 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++)
377 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q,
378 PSE_QUEUE_RSV_PAGES);
379 /* CDM3 */
380 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++)
381 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0);
382 /* CDM4 */
383 for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++)
384 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q,
385 PSE_QUEUE_RSV_PAGES);
386 if (airoha_ppe_is_enabled(eth, 1)) {
387 /* PPE2 */
388 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) {
389 if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2)
390 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2,
391 q,
392 PSE_QUEUE_RSV_PAGES);
393 else
394 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2,
395 q, 0);
396 }
397 }
398 /* GDM4 */
399 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++)
400 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q,
401 PSE_QUEUE_RSV_PAGES);
402 /* CDM5 */
403 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++)
404 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q,
405 PSE_QUEUE_RSV_PAGES);
406 }
407
airoha_fe_mc_vlan_clear(struct airoha_eth * eth)408 static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth)
409 {
410 int i;
411
412 for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) {
413 int err, j;
414 u32 val;
415
416 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
417
418 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
419 MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK;
420 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
421 err = read_poll_timeout(airoha_fe_rr, val,
422 val & MC_VLAN_CFG_CMD_DONE_MASK,
423 USEC_PER_MSEC, 5 * USEC_PER_MSEC,
424 false, eth, REG_MC_VLAN_CFG);
425 if (err)
426 return err;
427
428 for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) {
429 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
430
431 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
432 FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) |
433 MC_VLAN_CFG_RW_MASK;
434 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
435 err = read_poll_timeout(airoha_fe_rr, val,
436 val & MC_VLAN_CFG_CMD_DONE_MASK,
437 USEC_PER_MSEC,
438 5 * USEC_PER_MSEC, false, eth,
439 REG_MC_VLAN_CFG);
440 if (err)
441 return err;
442 }
443 }
444
445 return 0;
446 }
447
airoha_fe_crsn_qsel_init(struct airoha_eth * eth)448 static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth)
449 {
450 /* CDM1_CRSN_QSEL */
451 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_22 >> 2),
452 CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
453 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
454 CDM_CRSN_QSEL_Q1));
455 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_08 >> 2),
456 CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
457 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
458 CDM_CRSN_QSEL_Q1));
459 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_21 >> 2),
460 CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
461 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
462 CDM_CRSN_QSEL_Q1));
463 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_24 >> 2),
464 CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
465 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
466 CDM_CRSN_QSEL_Q6));
467 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_25 >> 2),
468 CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
469 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
470 CDM_CRSN_QSEL_Q1));
471 /* CDM2_CRSN_QSEL */
472 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_08 >> 2),
473 CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
474 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
475 CDM_CRSN_QSEL_Q1));
476 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_21 >> 2),
477 CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
478 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
479 CDM_CRSN_QSEL_Q1));
480 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_22 >> 2),
481 CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
482 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
483 CDM_CRSN_QSEL_Q1));
484 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_24 >> 2),
485 CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
486 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
487 CDM_CRSN_QSEL_Q6));
488 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_25 >> 2),
489 CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
490 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
491 CDM_CRSN_QSEL_Q1));
492 }
493
airoha_fe_init(struct airoha_eth * eth)494 static int airoha_fe_init(struct airoha_eth *eth)
495 {
496 airoha_fe_maccr_init(eth);
497
498 /* PSE IQ reserve */
499 airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK,
500 FIELD_PREP(PSE_IQ_RES1_P2_MASK, 0x10));
501 airoha_fe_rmw(eth, REG_PSE_IQ_REV2,
502 PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK,
503 FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) |
504 FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34));
505
506 /* enable FE copy engine for KA/DPI */
507 airoha_fe_wr(eth, REG_FE_PCE_CFG, PCE_DPI_EN_MASK | PCE_KA_EN_MASK);
508 /* set vip queue selection to ring 1 */
509 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(1), CDM_VIP_QSEL_MASK,
510 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4));
511 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_VIP_QSEL_MASK,
512 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4));
513 /* set GDM4 source interface offset to 8 */
514 airoha_fe_rmw(eth, REG_GDM_SRC_PORT_SET(4),
515 GDM_SPORT_OFF2_MASK |
516 GDM_SPORT_OFF1_MASK |
517 GDM_SPORT_OFF0_MASK,
518 FIELD_PREP(GDM_SPORT_OFF2_MASK, 8) |
519 FIELD_PREP(GDM_SPORT_OFF1_MASK, 8) |
520 FIELD_PREP(GDM_SPORT_OFF0_MASK, 8));
521
522 /* set PSE Page as 128B */
523 airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG,
524 FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK,
525 FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, 2) |
526 FE_DMA_GLO_PG_SZ_MASK);
527 airoha_fe_wr(eth, REG_FE_RST_GLO_CFG,
528 FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK |
529 FE_RST_GDM4_MBI_ARB_MASK);
530 usleep_range(1000, 2000);
531
532 /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1
533 * connect other rings to PSE Port0 OQ-0
534 */
535 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4));
536 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28));
537 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4));
538 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28));
539
540 airoha_fe_vip_setup(eth);
541 airoha_fe_pse_ports_init(eth);
542
543 airoha_fe_set(eth, REG_GDM_MISC_CFG,
544 GDM2_RDM_ACK_WAIT_PREF_MASK |
545 GDM2_CHN_VLD_MODE_MASK);
546 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_OAM_QSEL_MASK,
547 FIELD_PREP(CDM_OAM_QSEL_MASK, 15));
548
549 /* init fragment and assemble Force Port */
550 /* NPU Core-3, NPU Bridge Channel-3 */
551 airoha_fe_rmw(eth, REG_IP_FRAG_FP,
552 IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK,
553 FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) |
554 FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3));
555 /* QDMA LAN, RX Ring-22 */
556 airoha_fe_rmw(eth, REG_IP_FRAG_FP,
557 IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK,
558 FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) |
559 FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22));
560
561 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM3_IDX), GDM_PAD_EN_MASK);
562 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM4_IDX), GDM_PAD_EN_MASK);
563
564 /* Enable split for MIB counters for GDM3 and GDM4 */
565 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM3_IDX),
566 FE_GDM_TX_MIB_SPLIT_EN_MASK |
567 FE_GDM_RX_MIB_SPLIT_EN_MASK);
568 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM4_IDX),
569 FE_GDM_TX_MIB_SPLIT_EN_MASK |
570 FE_GDM_RX_MIB_SPLIT_EN_MASK);
571
572 airoha_fe_crsn_qsel_init(eth);
573
574 airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK);
575 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK);
576
577 /* default aging mode for mbi unlock issue */
578 airoha_fe_rmw(eth, REG_GDM_CHN_RLS(2),
579 MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK,
580 FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) |
581 FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3));
582
583 /* disable IFC by default */
584 airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK);
585
586 /* enable 1:N vlan action, init vlan table */
587 airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK);
588
589 return airoha_fe_mc_vlan_clear(eth);
590 }
591
airoha_qdma_fill_rx_queue(struct airoha_queue * q)592 static int airoha_qdma_fill_rx_queue(struct airoha_queue *q)
593 {
594 struct airoha_qdma *qdma = q->qdma;
595 int qid = q - &qdma->q_rx[0];
596 int nframes = 0;
597
598 while (q->queued < q->ndesc - 1) {
599 struct airoha_queue_entry *e = &q->entry[q->head];
600 struct airoha_qdma_desc *desc = &q->desc[q->head];
601 struct page *page;
602 int offset;
603 u32 val;
604
605 page = page_pool_dev_alloc_frag(q->page_pool, &offset,
606 q->buf_size);
607 if (!page)
608 break;
609
610 q->head = (q->head + 1) % q->ndesc;
611 q->queued++;
612 nframes++;
613
614 offset += AIROHA_RX_HEADROOM;
615 e->buf = page_address(page) + offset;
616 e->dma_addr = page_pool_get_dma_addr(page) + offset;
617 e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size));
618
619 val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len);
620 WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
621 WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr));
622 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head);
623 WRITE_ONCE(desc->data, cpu_to_le32(val));
624 WRITE_ONCE(desc->msg0, 0);
625 WRITE_ONCE(desc->msg1, 0);
626 WRITE_ONCE(desc->msg2, 0);
627 WRITE_ONCE(desc->msg3, 0);
628 }
629
630 if (nframes)
631 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid),
632 RX_RING_CPU_IDX_MASK,
633 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
634
635 return nframes;
636 }
637
638 static struct airoha_gdm_dev *
airoha_qdma_get_gdm_dev(struct airoha_eth * eth,struct airoha_qdma_desc * desc)639 airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc *desc)
640 {
641 struct airoha_gdm_port *port;
642 u16 p, d;
643
644 if (eth->soc->ops.get_dev_from_sport(desc, &p, &d))
645 return ERR_PTR(-ENODEV);
646
647 if (p >= ARRAY_SIZE(eth->ports))
648 return ERR_PTR(-ENODEV);
649
650 port = eth->ports[p];
651 if (!port)
652 return ERR_PTR(-ENODEV);
653
654 if (d >= ARRAY_SIZE(port->devs))
655 return ERR_PTR(-ENODEV);
656
657 return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV);
658 }
659
airoha_qdma_rx_process(struct airoha_queue * q,int budget)660 static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
661 {
662 enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
663 struct airoha_qdma *qdma = q->qdma;
664 struct airoha_eth *eth = qdma->eth;
665 int qid = q - &qdma->q_rx[0];
666 int done = 0;
667
668 while (done < budget) {
669 struct airoha_queue_entry *e = &q->entry[q->tail];
670 struct airoha_qdma_desc *desc = &q->desc[q->tail];
671 u32 hash, reason, msg1, desc_ctrl;
672 struct airoha_gdm_dev *dev;
673 struct net_device *netdev;
674 int data_len, len;
675 struct page *page;
676
677 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
678 if (!(desc_ctrl & QDMA_DESC_DONE_MASK))
679 break;
680
681 dma_rmb();
682
683 q->tail = (q->tail + 1) % q->ndesc;
684 q->queued--;
685
686 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
687 dir);
688
689 page = virt_to_head_page(e->buf);
690 len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
691 data_len = q->skb ? AIROHA_RX_LEN(q->buf_size) : e->dma_len;
692 if (!len || data_len < len)
693 goto free_frag;
694
695 dev = airoha_qdma_get_gdm_dev(eth, desc);
696 if (IS_ERR(dev))
697 goto free_frag;
698
699 netdev = netdev_from_priv(dev);
700 if (!q->skb) { /* first buffer */
701 q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM,
702 q->buf_size);
703 if (!q->skb)
704 goto free_frag;
705
706 skb_reserve(q->skb, AIROHA_RX_HEADROOM);
707 __skb_put(q->skb, len);
708 skb_mark_for_recycle(q->skb);
709 q->skb->dev = netdev;
710 q->skb->protocol = eth_type_trans(q->skb, netdev);
711 q->skb->ip_summed = CHECKSUM_UNNECESSARY;
712 skb_record_rx_queue(q->skb, qid);
713 } else { /* scattered frame */
714 struct skb_shared_info *shinfo = skb_shinfo(q->skb);
715 int nr_frags = shinfo->nr_frags;
716
717 if (nr_frags >= ARRAY_SIZE(shinfo->frags))
718 goto free_frag;
719
720 skb_add_rx_frag(q->skb, nr_frags, page,
721 e->buf - page_address(page), len,
722 q->buf_size);
723 }
724
725 if (FIELD_GET(QDMA_DESC_MORE_MASK, desc_ctrl))
726 continue;
727
728 if (netdev_uses_dsa(netdev)) {
729 struct airoha_gdm_port *port = dev->port;
730
731 /* PPE module requires untagged packets to work
732 * properly and it provides DSA port index via the
733 * DMA descriptor. Report DSA tag to the DSA stack
734 * via skb dst info.
735 */
736 u32 msg0 = le32_to_cpu(READ_ONCE(desc->msg0));
737 u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0);
738
739 if (sptag < ARRAY_SIZE(port->dsa_meta) &&
740 port->dsa_meta[sptag])
741 skb_dst_set_noref(q->skb,
742 &port->dsa_meta[sptag]->dst);
743 }
744
745 msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
746 hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1);
747 if (hash != AIROHA_RXD4_FOE_ENTRY)
748 skb_set_hash(q->skb, jhash_1word(hash, 0),
749 PKT_HASH_TYPE_L4);
750
751 reason = FIELD_GET(AIROHA_RXD4_PPE_CPU_REASON, msg1);
752 if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
753 airoha_ppe_check_skb(ð->ppe->dev, q->skb, hash,
754 false);
755
756 done++;
757 napi_gro_receive(&q->napi, q->skb);
758 q->skb = NULL;
759 continue;
760 free_frag:
761 if (q->skb) {
762 dev_kfree_skb(q->skb);
763 q->skb = NULL;
764 }
765 page_pool_put_full_page(q->page_pool, page, true);
766 }
767 airoha_qdma_fill_rx_queue(q);
768
769 return done;
770 }
771
airoha_qdma_rx_napi_poll(struct napi_struct * napi,int budget)772 static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
773 {
774 struct airoha_queue *q = container_of(napi, struct airoha_queue, napi);
775 int cur, done = 0;
776
777 do {
778 cur = airoha_qdma_rx_process(q, budget - done);
779 done += cur;
780 } while (cur && done < budget);
781
782 if (done < budget && napi_complete(napi)) {
783 struct airoha_qdma *qdma = q->qdma;
784 int i, qid = q - &qdma->q_rx[0];
785 int intr_reg = qid < RX_DONE_HIGH_OFFSET ? QDMA_INT_REG_IDX1
786 : QDMA_INT_REG_IDX2;
787
788 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) {
789 if (!(BIT(qid) & RX_IRQ_BANK_PIN_MASK(i)))
790 continue;
791
792 airoha_qdma_irq_enable(&qdma->irq_banks[i], intr_reg,
793 BIT(qid % RX_DONE_HIGH_OFFSET));
794 }
795 }
796
797 return done;
798 }
799
airoha_qdma_init_rx_queue(struct airoha_queue * q,struct airoha_qdma * qdma,int ndesc)800 static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
801 struct airoha_qdma *qdma, int ndesc)
802 {
803 const struct page_pool_params pp_params = {
804 .order = 0,
805 .pool_size = 256,
806 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
807 .dma_dir = DMA_FROM_DEVICE,
808 .max_len = PAGE_SIZE,
809 .nid = NUMA_NO_NODE,
810 .dev = qdma->eth->dev,
811 .napi = &q->napi,
812 };
813 struct airoha_eth *eth = qdma->eth;
814 int qid = q - &qdma->q_rx[0], thr;
815 dma_addr_t dma_addr;
816
817 q->buf_size = PAGE_SIZE / 2;
818 q->qdma = qdma;
819
820 q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
821 GFP_KERNEL);
822 if (!q->entry)
823 return -ENOMEM;
824
825 q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc),
826 &dma_addr, GFP_KERNEL);
827 if (!q->desc)
828 return -ENOMEM;
829
830 q->page_pool = page_pool_create(&pp_params);
831 if (IS_ERR(q->page_pool)) {
832 int err = PTR_ERR(q->page_pool);
833
834 q->page_pool = NULL;
835 return err;
836 }
837
838 q->ndesc = ndesc;
839 netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
840
841 airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);
842 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),
843 RX_RING_SIZE_MASK,
844 FIELD_PREP(RX_RING_SIZE_MASK, ndesc));
845
846 thr = clamp(ndesc >> 3, 1, 32);
847 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK,
848 FIELD_PREP(RX_RING_THR_MASK, thr));
849 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
850 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head));
851 airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid), RX_RING_SG_EN_MASK);
852
853 airoha_qdma_fill_rx_queue(q);
854
855 return 0;
856 }
857
airoha_qdma_cleanup_rx_queue(struct airoha_queue * q)858 static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
859 {
860 struct airoha_qdma *qdma = q->qdma;
861 struct airoha_eth *eth = qdma->eth;
862 int qid = q - &qdma->q_rx[0];
863
864 while (q->queued) {
865 struct airoha_queue_entry *e = &q->entry[q->tail];
866 struct airoha_qdma_desc *desc = &q->desc[q->tail];
867 struct page *page = virt_to_head_page(e->buf);
868
869 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
870 page_pool_get_dma_dir(q->page_pool));
871 page_pool_put_full_page(q->page_pool, page, false);
872 /* Reset DMA descriptor */
873 WRITE_ONCE(desc->ctrl, 0);
874 WRITE_ONCE(desc->addr, 0);
875 WRITE_ONCE(desc->data, 0);
876 WRITE_ONCE(desc->msg0, 0);
877 WRITE_ONCE(desc->msg1, 0);
878 WRITE_ONCE(desc->msg2, 0);
879 WRITE_ONCE(desc->msg3, 0);
880
881 q->tail = (q->tail + 1) % q->ndesc;
882 q->queued--;
883 }
884
885 q->head = q->tail;
886 /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is
887 * empty.
888 */
889 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK,
890 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
891 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
892 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail));
893 }
894
airoha_qdma_init_rx(struct airoha_qdma * qdma)895 static int airoha_qdma_init_rx(struct airoha_qdma *qdma)
896 {
897 int i;
898
899 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
900 int err;
901
902 if (!(RX_DONE_INT_MASK & BIT(i))) {
903 /* rx-queue not binded to irq */
904 continue;
905 }
906
907 err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma,
908 RX_DSCP_NUM(i));
909 if (err)
910 return err;
911 }
912
913 return 0;
914 }
915
airoha_qdma_wake_netdev_txqs(struct airoha_queue * q)916 static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q)
917 {
918 struct airoha_qdma *qdma = q->qdma;
919 struct airoha_eth *eth = qdma->eth;
920 int i, qid = q - &qdma->q_tx[0];
921
922 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
923 struct airoha_gdm_port *port = eth->ports[i];
924 int d;
925
926 if (!port)
927 continue;
928
929 for (d = 0; d < ARRAY_SIZE(port->devs); d++) {
930 struct airoha_gdm_dev *dev = port->devs[d];
931 struct net_device *netdev;
932 int j;
933
934 if (!dev)
935 continue;
936
937 if (dev->qdma != qdma)
938 continue;
939
940 netdev = netdev_from_priv(dev);
941 for (j = 0; j < netdev->num_tx_queues; j++) {
942 if (airoha_qdma_get_txq(qdma, j) != qid)
943 continue;
944
945 netif_wake_subqueue(netdev, j);
946 }
947 }
948 }
949 q->txq_stopped = false;
950 }
951
airoha_unmap_xmit_buf(struct airoha_eth * eth,struct airoha_queue_entry * e)952 static void airoha_unmap_xmit_buf(struct airoha_eth *eth,
953 struct airoha_queue_entry *e)
954 {
955 switch (e->dma_type) {
956 case AIROHA_DMA_MAP_PAGE:
957 dma_unmap_page(eth->dev, e->dma_addr, e->dma_len,
958 DMA_TO_DEVICE);
959 break;
960 case AIROHA_DMA_MAP_SINGLE:
961 dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
962 DMA_TO_DEVICE);
963 break;
964 case AIROHA_DMA_UNMAPPED:
965 default:
966 break;
967 }
968 e->dma_type = AIROHA_DMA_UNMAPPED;
969 }
970
airoha_qdma_tx_napi_poll(struct napi_struct * napi,int budget)971 static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
972 {
973 struct airoha_tx_irq_queue *irq_q;
974 int id, done = 0, irq_queued;
975 struct airoha_qdma *qdma;
976 struct airoha_eth *eth;
977 u32 status, head;
978
979 irq_q = container_of(napi, struct airoha_tx_irq_queue, napi);
980 qdma = irq_q->qdma;
981 id = irq_q - &qdma->q_tx_irq[0];
982 eth = qdma->eth;
983
984 status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id));
985 head = FIELD_GET(IRQ_HEAD_IDX_MASK, status);
986 head = head % irq_q->size;
987 irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status);
988
989 while (irq_queued > 0 && done < budget) {
990 u32 qid, val = irq_q->q[head];
991 struct airoha_qdma_desc *desc;
992 struct airoha_queue_entry *e;
993 struct airoha_queue *q;
994 u32 index, desc_ctrl;
995 struct sk_buff *skb;
996
997 if (val == 0xff)
998 break;
999
1000 irq_q->q[head] = 0xff; /* mark as done */
1001 head = (head + 1) % irq_q->size;
1002 irq_queued--;
1003 done++;
1004
1005 qid = FIELD_GET(IRQ_RING_IDX_MASK, val);
1006 if (qid >= ARRAY_SIZE(qdma->q_tx))
1007 continue;
1008
1009 q = &qdma->q_tx[qid];
1010 if (!q->ndesc)
1011 continue;
1012
1013 index = FIELD_GET(IRQ_DESC_IDX_MASK, val);
1014 if (index >= q->ndesc)
1015 continue;
1016
1017 spin_lock_bh(&q->lock);
1018
1019 if (!q->queued)
1020 goto unlock;
1021
1022 desc = &q->desc[index];
1023 desc_ctrl = le32_to_cpu(desc->ctrl);
1024
1025 if (!(desc_ctrl & QDMA_DESC_DONE_MASK) &&
1026 !(desc_ctrl & QDMA_DESC_DROP_MASK))
1027 goto unlock;
1028
1029 e = &q->entry[index];
1030 skb = e->skb;
1031 e->skb = NULL;
1032
1033 airoha_unmap_xmit_buf(eth, e);
1034 list_add_tail(&e->list, &q->tx_list);
1035
1036 WRITE_ONCE(desc->msg0, 0);
1037 WRITE_ONCE(desc->msg1, 0);
1038 q->queued--;
1039
1040 if (skb) {
1041 struct netdev_queue *txq;
1042
1043 txq = skb_get_tx_queue(skb->dev, skb);
1044 netdev_tx_completed_queue(txq, 1, skb->len);
1045 dev_kfree_skb_any(skb);
1046 }
1047
1048 if (q->txq_stopped && q->ndesc - q->queued >= q->free_thr) {
1049 /* Since multiple net_device TX queues can share the
1050 * same hw QDMA TX queue, there is no guarantee we have
1051 * inflight packets queued in hw belonging to a
1052 * net_device TX queue stopped in the xmit path.
1053 * In order to avoid any potential net_device TX queue
1054 * stall, we need to wake all the net_device TX queues
1055 * feeding the same hw QDMA TX queue.
1056 */
1057 airoha_qdma_wake_netdev_txqs(q);
1058 }
1059
1060 unlock:
1061 spin_unlock_bh(&q->lock);
1062 }
1063
1064 if (done) {
1065 int i, len = done >> 7;
1066
1067 for (i = 0; i < len; i++)
1068 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
1069 IRQ_CLEAR_LEN_MASK, 0x80);
1070 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
1071 IRQ_CLEAR_LEN_MASK, (done & 0x7f));
1072 }
1073
1074 if (done < budget && napi_complete(napi))
1075 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0,
1076 TX_DONE_INT_MASK(id));
1077
1078 return done;
1079 }
1080
airoha_qdma_init_tx_queue(struct airoha_queue * q,struct airoha_qdma * qdma,int size)1081 static int airoha_qdma_init_tx_queue(struct airoha_queue *q,
1082 struct airoha_qdma *qdma, int size)
1083 {
1084 struct airoha_eth *eth = qdma->eth;
1085 int i, qid = q - &qdma->q_tx[0];
1086 dma_addr_t dma_addr;
1087
1088 spin_lock_init(&q->lock);
1089 q->qdma = qdma;
1090 q->free_thr = 1 + MAX_SKB_FRAGS;
1091 INIT_LIST_HEAD(&q->tx_list);
1092
1093 q->entry = devm_kzalloc(eth->dev, size * sizeof(*q->entry),
1094 GFP_KERNEL);
1095 if (!q->entry)
1096 return -ENOMEM;
1097
1098 q->desc = dmam_alloc_coherent(eth->dev, size * sizeof(*q->desc),
1099 &dma_addr, GFP_KERNEL);
1100 if (!q->desc)
1101 return -ENOMEM;
1102
1103 for (i = 0; i < size; i++) {
1104 u32 val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1);
1105
1106 list_add_tail(&q->entry[i].list, &q->tx_list);
1107 WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val));
1108 }
1109 q->ndesc = size;
1110
1111 /* xmit ring drop default setting */
1112 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(qid),
1113 TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK);
1114
1115 airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr);
1116 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,
1117 FIELD_PREP(TX_RING_CPU_IDX_MASK, 0));
1118 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,
1119 FIELD_PREP(TX_RING_DMA_IDX_MASK, 0));
1120
1121 return 0;
1122 }
1123
airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue * irq_q,struct airoha_qdma * qdma,int size)1124 static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q,
1125 struct airoha_qdma *qdma, int size)
1126 {
1127 int id = irq_q - &qdma->q_tx_irq[0];
1128 struct airoha_eth *eth = qdma->eth;
1129 dma_addr_t dma_addr;
1130
1131 irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32),
1132 &dma_addr, GFP_KERNEL);
1133 if (!irq_q->q)
1134 return -ENOMEM;
1135
1136 memset(irq_q->q, 0xff, size * sizeof(u32));
1137 irq_q->size = size;
1138 irq_q->qdma = qdma;
1139
1140 netif_napi_add_tx(eth->napi_dev, &irq_q->napi,
1141 airoha_qdma_tx_napi_poll);
1142
1143 airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr);
1144 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK,
1145 FIELD_PREP(TX_IRQ_DEPTH_MASK, size));
1146 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK,
1147 FIELD_PREP(TX_IRQ_THR_MASK, 1));
1148
1149 return 0;
1150 }
1151
airoha_qdma_init_tx(struct airoha_qdma * qdma)1152 static int airoha_qdma_init_tx(struct airoha_qdma *qdma)
1153 {
1154 int i, err;
1155
1156 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
1157 err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma,
1158 IRQ_QUEUE_LEN(i));
1159 if (err)
1160 return err;
1161 }
1162
1163 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
1164 err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma,
1165 TX_DSCP_NUM);
1166 if (err)
1167 return err;
1168 }
1169
1170 return 0;
1171 }
1172
airoha_qdma_tx_cleanup(struct airoha_qdma * qdma)1173 static void airoha_qdma_tx_cleanup(struct airoha_qdma *qdma)
1174 {
1175 u32 status;
1176 int i;
1177
1178 airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,
1179 GLOBAL_CFG_TX_DMA_EN_MASK);
1180 if (read_poll_timeout(airoha_qdma_rr, status,
1181 !(status & GLOBAL_CFG_TX_DMA_BUSY_MASK),
1182 USEC_PER_MSEC, 50 * USEC_PER_MSEC, true,
1183 qdma, REG_QDMA_GLOBAL_CFG))
1184 dev_warn(qdma->eth->dev, "QDMA TX DMA busy timeout\n");
1185
1186 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
1187 struct airoha_queue *q = &qdma->q_tx[i];
1188 u16 index = 0;
1189 int j;
1190
1191 if (!q->ndesc)
1192 continue;
1193
1194 spin_lock_bh(&q->lock);
1195
1196 q->flushing = true;
1197 for (j = 0; j < q->ndesc; j++) {
1198 struct airoha_queue_entry *e = &q->entry[j];
1199 struct airoha_qdma_desc *desc = &q->desc[j];
1200 struct sk_buff *skb = e->skb;
1201
1202 if (e->dma_type == AIROHA_DMA_UNMAPPED)
1203 continue;
1204
1205 airoha_unmap_xmit_buf(qdma->eth, e);
1206 list_add_tail(&e->list, &q->tx_list);
1207
1208 WRITE_ONCE(desc->ctrl, 0);
1209 WRITE_ONCE(desc->addr, 0);
1210 WRITE_ONCE(desc->data, 0);
1211 WRITE_ONCE(desc->msg0, 0);
1212 WRITE_ONCE(desc->msg1, 0);
1213 WRITE_ONCE(desc->msg2, 0);
1214
1215 if (skb) {
1216 struct netdev_queue *txq;
1217
1218 txq = skb_get_tx_queue(skb->dev, skb);
1219 netdev_tx_completed_queue(txq, 1, skb->len);
1220 dev_kfree_skb_any(skb);
1221 e->skb = NULL;
1222 }
1223
1224 q->queued--;
1225 }
1226
1227 if (!list_empty(&q->tx_list)) {
1228 struct airoha_queue_entry *e;
1229
1230 e = list_first_entry(&q->tx_list,
1231 struct airoha_queue_entry, list);
1232 index = e - q->entry;
1233 }
1234 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(i), TX_RING_CPU_IDX_MASK,
1235 FIELD_PREP(TX_RING_CPU_IDX_MASK, index));
1236 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(i), TX_RING_DMA_IDX_MASK,
1237 FIELD_PREP(TX_RING_DMA_IDX_MASK, index));
1238
1239 spin_unlock_bh(&q->lock);
1240 }
1241 }
1242
airoha_qdma_init_hfwd_queues(struct airoha_qdma * qdma)1243 static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma)
1244 {
1245 int size, index, num_desc = HW_DSCP_NUM;
1246 struct airoha_eth *eth = qdma->eth;
1247 int id = qdma - ð->qdma[0];
1248 u32 status, buf_size;
1249 dma_addr_t dma_addr;
1250 const char *name;
1251
1252 name = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d-buf", id);
1253 if (!name)
1254 return -ENOMEM;
1255
1256 buf_size = id ? AIROHA_MAX_PACKET_SIZE / 2 : AIROHA_MAX_PACKET_SIZE;
1257 index = of_property_match_string(eth->dev->of_node,
1258 "memory-region-names", name);
1259 if (index >= 0) {
1260 struct reserved_mem *rmem;
1261 struct device_node *np;
1262
1263 /* Consume reserved memory for hw forwarding buffers queue if
1264 * available in the DTS
1265 */
1266 np = of_parse_phandle(eth->dev->of_node, "memory-region",
1267 index);
1268 if (!np)
1269 return -ENODEV;
1270
1271 rmem = of_reserved_mem_lookup(np);
1272 of_node_put(np);
1273 if (!rmem)
1274 return -ENODEV;
1275
1276 dma_addr = rmem->base;
1277 /* Compute the number of hw descriptors according to the
1278 * reserved memory size and the payload buffer size
1279 */
1280 num_desc = div_u64(rmem->size, buf_size);
1281 } else {
1282 size = buf_size * num_desc;
1283 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr,
1284 GFP_KERNEL))
1285 return -ENOMEM;
1286 }
1287
1288 airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr);
1289
1290 size = num_desc * sizeof(struct airoha_qdma_fwd_desc);
1291 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, GFP_KERNEL))
1292 return -ENOMEM;
1293
1294 airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr);
1295 /* QDMA0: 2KB. QDMA1: 1KB */
1296 airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG,
1297 HW_FWD_DSCP_PAYLOAD_SIZE_MASK,
1298 FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK, !!id));
1299 airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK,
1300 FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128));
1301 airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG,
1302 LMGR_INIT_START | LMGR_SRAM_MODE_MASK |
1303 HW_FWD_DESC_NUM_MASK,
1304 FIELD_PREP(HW_FWD_DESC_NUM_MASK, num_desc) |
1305 LMGR_INIT_START | LMGR_SRAM_MODE_MASK);
1306
1307 return read_poll_timeout(airoha_qdma_rr, status,
1308 !(status & LMGR_INIT_START), USEC_PER_MSEC,
1309 30 * USEC_PER_MSEC, true, qdma,
1310 REG_LMGR_INIT_CFG);
1311 }
1312
airoha_qdma_init_qos(struct airoha_qdma * qdma)1313 static void airoha_qdma_init_qos(struct airoha_qdma *qdma)
1314 {
1315 airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK);
1316 airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK);
1317
1318 airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG,
1319 PSE_BUF_ESTIMATE_EN_MASK);
1320
1321 airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG,
1322 EGRESS_RATE_METER_EN_MASK |
1323 EGRESS_RATE_METER_EQ_RATE_EN_MASK);
1324 /* 2047us x 31 = 63.457ms */
1325 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
1326 EGRESS_RATE_METER_WINDOW_SZ_MASK,
1327 FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK, 0x1f));
1328 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
1329 EGRESS_RATE_METER_TIMESLICE_MASK,
1330 FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK, 0x7ff));
1331
1332 /* ratelimit init */
1333 airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK);
1334 /* fast-tick 25us */
1335 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK,
1336 FIELD_PREP(GLB_FAST_TICK_MASK, 25));
1337 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK,
1338 FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40));
1339
1340 airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK);
1341 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK,
1342 FIELD_PREP(EGRESS_FAST_TICK_MASK, 25));
1343 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG,
1344 EGRESS_SLOW_TICK_RATIO_MASK,
1345 FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40));
1346
1347 airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK);
1348 airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG,
1349 INGRESS_TRTCM_MODE_MASK);
1350 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK,
1351 FIELD_PREP(INGRESS_FAST_TICK_MASK, 125));
1352 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG,
1353 INGRESS_SLOW_TICK_RATIO_MASK,
1354 FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8));
1355
1356 airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK);
1357 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK,
1358 FIELD_PREP(SLA_FAST_TICK_MASK, 25));
1359 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK,
1360 FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40));
1361 }
1362
airoha_qdma_init_qos_stats(struct airoha_qdma * qdma)1363 static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma)
1364 {
1365 int i;
1366
1367 for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) {
1368 /* Tx-cpu transferred count */
1369 airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0);
1370 airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1),
1371 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
1372 CNTR_ALL_DSCP_RING_EN_MASK |
1373 FIELD_PREP(CNTR_CHAN_MASK, i));
1374 /* Tx-fwd transferred count */
1375 airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0);
1376 airoha_qdma_wr(qdma, REG_CNTR_CFG((i << 1) + 1),
1377 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
1378 CNTR_ALL_DSCP_RING_EN_MASK |
1379 FIELD_PREP(CNTR_SRC_MASK, 1) |
1380 FIELD_PREP(CNTR_CHAN_MASK, i));
1381 }
1382 }
1383
airoha_qdma_hw_init(struct airoha_qdma * qdma)1384 static int airoha_qdma_hw_init(struct airoha_qdma *qdma)
1385 {
1386 int i;
1387
1388 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) {
1389 /* clear pending irqs */
1390 airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff);
1391 /* setup rx irqs */
1392 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX0,
1393 INT_RX0_MASK(RX_IRQ_BANK_PIN_MASK(i)));
1394 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX1,
1395 INT_RX1_MASK(RX_IRQ_BANK_PIN_MASK(i)));
1396 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX2,
1397 INT_RX2_MASK(RX_IRQ_BANK_PIN_MASK(i)));
1398 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX3,
1399 INT_RX3_MASK(RX_IRQ_BANK_PIN_MASK(i)));
1400 }
1401 /* setup tx irqs */
1402 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0,
1403 TX_COHERENT_LOW_INT_MASK | INT_TX_MASK);
1404 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX4,
1405 TX_COHERENT_HIGH_INT_MASK);
1406
1407 /* setup irq binding */
1408 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
1409 if (!qdma->q_tx[i].ndesc)
1410 continue;
1411
1412 if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i))
1413 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i),
1414 TX_RING_IRQ_BLOCKING_CFG_MASK);
1415 else
1416 airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i),
1417 TX_RING_IRQ_BLOCKING_CFG_MASK);
1418 }
1419
1420 airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG,
1421 FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) |
1422 GLOBAL_CFG_CPU_TXR_RR_MASK |
1423 GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK |
1424 GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK |
1425 GLOBAL_CFG_MULTICAST_EN_MASK |
1426 GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK |
1427 GLOBAL_CFG_TX_WB_DONE_MASK |
1428 FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2));
1429
1430 airoha_qdma_init_qos(qdma);
1431
1432 /* disable qdma rx delay interrupt */
1433 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
1434 if (!qdma->q_rx[i].ndesc)
1435 continue;
1436
1437 airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i),
1438 RX_DELAY_INT_MASK);
1439 }
1440
1441 airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG,
1442 TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN);
1443 airoha_qdma_init_qos_stats(qdma);
1444
1445 return 0;
1446 }
1447
airoha_irq_handler(int irq,void * dev_instance)1448 static irqreturn_t airoha_irq_handler(int irq, void *dev_instance)
1449 {
1450 struct airoha_irq_bank *irq_bank = dev_instance;
1451 struct airoha_qdma *qdma = irq_bank->qdma;
1452 u32 rx_intr_mask = 0, rx_intr1, rx_intr2;
1453 u32 intr[ARRAY_SIZE(irq_bank->irqmask)];
1454 int i;
1455
1456 for (i = 0; i < ARRAY_SIZE(intr); i++) {
1457 intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i));
1458 intr[i] &= irq_bank->irqmask[i];
1459 airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]);
1460 }
1461
1462 if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state))
1463 return IRQ_NONE;
1464
1465 rx_intr1 = intr[1] & RX_DONE_LOW_INT_MASK;
1466 if (rx_intr1) {
1467 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX1, rx_intr1);
1468 rx_intr_mask |= rx_intr1;
1469 }
1470
1471 rx_intr2 = intr[2] & RX_DONE_HIGH_INT_MASK;
1472 if (rx_intr2) {
1473 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX2, rx_intr2);
1474 rx_intr_mask |= (rx_intr2 << 16);
1475 }
1476
1477 for (i = 0; rx_intr_mask && i < ARRAY_SIZE(qdma->q_rx); i++) {
1478 if (!qdma->q_rx[i].ndesc)
1479 continue;
1480
1481 if (rx_intr_mask & BIT(i))
1482 napi_schedule(&qdma->q_rx[i].napi);
1483 }
1484
1485 if (intr[0] & INT_TX_MASK) {
1486 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
1487 if (!(intr[0] & TX_DONE_INT_MASK(i)))
1488 continue;
1489
1490 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX0,
1491 TX_DONE_INT_MASK(i));
1492 napi_schedule(&qdma->q_tx_irq[i].napi);
1493 }
1494 }
1495
1496 return IRQ_HANDLED;
1497 }
1498
airoha_qdma_init_irq_banks(struct platform_device * pdev,struct airoha_qdma * qdma)1499 static int airoha_qdma_init_irq_banks(struct platform_device *pdev,
1500 struct airoha_qdma *qdma)
1501 {
1502 struct airoha_eth *eth = qdma->eth;
1503 int i, id = qdma - ð->qdma[0];
1504
1505 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) {
1506 struct airoha_irq_bank *irq_bank = &qdma->irq_banks[i];
1507 int err, irq_index = 4 * id + i;
1508 const char *name;
1509
1510 spin_lock_init(&irq_bank->irq_lock);
1511 irq_bank->qdma = qdma;
1512
1513 irq_bank->irq = platform_get_irq(pdev, irq_index);
1514 if (irq_bank->irq < 0)
1515 return irq_bank->irq;
1516
1517 name = devm_kasprintf(eth->dev, GFP_KERNEL,
1518 KBUILD_MODNAME ".%d", irq_index);
1519 if (!name)
1520 return -ENOMEM;
1521
1522 err = devm_request_irq(eth->dev, irq_bank->irq,
1523 airoha_irq_handler, IRQF_SHARED, name,
1524 irq_bank);
1525 if (err)
1526 return err;
1527 }
1528
1529 return 0;
1530 }
1531
airoha_qdma_init(struct platform_device * pdev,struct airoha_eth * eth,struct airoha_qdma * qdma)1532 static int airoha_qdma_init(struct platform_device *pdev,
1533 struct airoha_eth *eth,
1534 struct airoha_qdma *qdma)
1535 {
1536 int err, id = qdma - ð->qdma[0];
1537 const char *res;
1538
1539 qdma->eth = eth;
1540 res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id);
1541 if (!res)
1542 return -ENOMEM;
1543
1544 qdma->regs = devm_platform_ioremap_resource_byname(pdev, res);
1545 if (IS_ERR(qdma->regs))
1546 return dev_err_probe(eth->dev, PTR_ERR(qdma->regs),
1547 "failed to iomap qdma%d regs\n", id);
1548
1549 err = airoha_qdma_init_irq_banks(pdev, qdma);
1550 if (err)
1551 return err;
1552
1553 err = airoha_qdma_init_rx(qdma);
1554 if (err)
1555 return err;
1556
1557 err = airoha_qdma_init_tx(qdma);
1558 if (err)
1559 return err;
1560
1561 err = airoha_qdma_init_hfwd_queues(qdma);
1562 if (err)
1563 return err;
1564
1565 return airoha_qdma_hw_init(qdma);
1566 }
1567
airoha_qdma_cleanup(struct airoha_eth * eth,struct airoha_qdma * qdma)1568 static void airoha_qdma_cleanup(struct airoha_eth *eth,
1569 struct airoha_qdma *qdma)
1570 {
1571 int i;
1572
1573 if (test_bit(DEV_STATE_INITIALIZED, ð->state)) {
1574 u32 status;
1575
1576 airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,
1577 GLOBAL_CFG_RX_DMA_EN_MASK);
1578 if (read_poll_timeout(airoha_qdma_rr, status,
1579 !(status & GLOBAL_CFG_RX_DMA_BUSY_MASK),
1580 USEC_PER_MSEC, 50 * USEC_PER_MSEC, true,
1581 qdma, REG_QDMA_GLOBAL_CFG))
1582 dev_warn(eth->dev, "QDMA RX DMA busy timeout\n");
1583 }
1584
1585 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
1586 if (!qdma->q_rx[i].ndesc)
1587 continue;
1588
1589 netif_napi_del(&qdma->q_rx[i].napi);
1590 airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]);
1591 if (qdma->q_rx[i].page_pool) {
1592 page_pool_destroy(qdma->q_rx[i].page_pool);
1593 qdma->q_rx[i].page_pool = NULL;
1594 }
1595 }
1596
1597 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
1598 if (!qdma->q_tx_irq[i].size)
1599 continue;
1600
1601 netif_napi_del(&qdma->q_tx_irq[i].napi);
1602 }
1603
1604 }
1605
airoha_hw_init(struct platform_device * pdev,struct airoha_eth * eth)1606 static int airoha_hw_init(struct platform_device *pdev,
1607 struct airoha_eth *eth)
1608 {
1609 int err, i;
1610
1611 /* disable xsi */
1612 err = reset_control_bulk_assert(eth->soc->num_xsi_rsts, eth->xsi_rsts);
1613 if (err)
1614 return err;
1615
1616 err = reset_control_bulk_assert(ARRAY_SIZE(eth->rsts), eth->rsts);
1617 if (err)
1618 return err;
1619
1620 msleep(20);
1621 err = reset_control_bulk_deassert(ARRAY_SIZE(eth->rsts), eth->rsts);
1622 if (err)
1623 return err;
1624
1625 msleep(20);
1626 err = airoha_fe_init(eth);
1627 if (err)
1628 return err;
1629
1630 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
1631 err = airoha_qdma_init(pdev, eth, ð->qdma[i]);
1632 if (err)
1633 goto error;
1634 }
1635
1636 err = airoha_ppe_init(eth);
1637 if (err)
1638 goto error;
1639
1640 set_bit(DEV_STATE_INITIALIZED, ð->state);
1641
1642 return 0;
1643 error:
1644 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
1645 airoha_qdma_cleanup(eth, ð->qdma[i]);
1646
1647 return err;
1648 }
1649
airoha_hw_cleanup(struct airoha_eth * eth)1650 static void airoha_hw_cleanup(struct airoha_eth *eth)
1651 {
1652 int i;
1653
1654 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
1655 airoha_qdma_cleanup(eth, ð->qdma[i]);
1656 airoha_ppe_deinit(eth);
1657 }
1658
airoha_qdma_start_napi(struct airoha_qdma * qdma)1659 static void airoha_qdma_start_napi(struct airoha_qdma *qdma)
1660 {
1661 int i;
1662
1663 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
1664 napi_enable(&qdma->q_tx_irq[i].napi);
1665
1666 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
1667 if (!qdma->q_rx[i].ndesc)
1668 continue;
1669
1670 napi_enable(&qdma->q_rx[i].napi);
1671 }
1672 }
1673
airoha_qdma_stop_napi(struct airoha_qdma * qdma)1674 static void airoha_qdma_stop_napi(struct airoha_qdma *qdma)
1675 {
1676 int i;
1677
1678 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
1679 napi_disable(&qdma->q_tx_irq[i].napi);
1680
1681 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
1682 if (!qdma->q_rx[i].ndesc)
1683 continue;
1684
1685 napi_disable(&qdma->q_rx[i].napi);
1686 }
1687 }
1688
airoha_dev_get_hw_stats(struct airoha_gdm_dev * dev)1689 static void airoha_dev_get_hw_stats(struct airoha_gdm_dev *dev)
1690 {
1691 struct airoha_gdm_port *port = dev->port;
1692 struct airoha_eth *eth = dev->eth;
1693 u32 val, i = 0;
1694
1695 /* Read relevant MIB for GDM with multiple port attached */
1696 if (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX)
1697 airoha_fe_rmw(eth, REG_FE_GDM_MIB_CFG(port->id),
1698 FE_TX_MIB_ID_MASK | FE_RX_MIB_ID_MASK,
1699 FIELD_PREP(FE_TX_MIB_ID_MASK, dev->nbq) |
1700 FIELD_PREP(FE_RX_MIB_ID_MASK, dev->nbq));
1701
1702 u64_stats_update_begin(&dev->stats.syncp);
1703
1704 /* TX */
1705 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_H(port->id));
1706 dev->stats.tx_ok_pkts += ((u64)val << 32);
1707 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_L(port->id));
1708 dev->stats.tx_ok_pkts += val;
1709
1710 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_H(port->id));
1711 dev->stats.tx_ok_bytes += ((u64)val << 32);
1712 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_L(port->id));
1713 dev->stats.tx_ok_bytes += val;
1714
1715 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_DROP_CNT(port->id));
1716 dev->stats.tx_drops += val;
1717
1718 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_BC_CNT(port->id));
1719 dev->stats.tx_broadcast += val;
1720
1721 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_MC_CNT(port->id));
1722 dev->stats.tx_multicast += val;
1723
1724 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_RUNT_CNT(port->id));
1725 dev->stats.tx_len[i] += val;
1726
1727 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_H(port->id));
1728 dev->stats.tx_len[i] += ((u64)val << 32);
1729 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_L(port->id));
1730 dev->stats.tx_len[i++] += val;
1731
1732 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_H(port->id));
1733 dev->stats.tx_len[i] += ((u64)val << 32);
1734 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_L(port->id));
1735 dev->stats.tx_len[i++] += val;
1736
1737 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_H(port->id));
1738 dev->stats.tx_len[i] += ((u64)val << 32);
1739 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_L(port->id));
1740 dev->stats.tx_len[i++] += val;
1741
1742 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_H(port->id));
1743 dev->stats.tx_len[i] += ((u64)val << 32);
1744 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_L(port->id));
1745 dev->stats.tx_len[i++] += val;
1746
1747 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_H(port->id));
1748 dev->stats.tx_len[i] += ((u64)val << 32);
1749 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_L(port->id));
1750 dev->stats.tx_len[i++] += val;
1751
1752 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_H(port->id));
1753 dev->stats.tx_len[i] += ((u64)val << 32);
1754 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_L(port->id));
1755 dev->stats.tx_len[i++] += val;
1756
1757 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_LONG_CNT(port->id));
1758 dev->stats.tx_len[i++] += val;
1759
1760 /* RX */
1761 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_H(port->id));
1762 dev->stats.rx_ok_pkts += ((u64)val << 32);
1763 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_L(port->id));
1764 dev->stats.rx_ok_pkts += val;
1765
1766 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_H(port->id));
1767 dev->stats.rx_ok_bytes += ((u64)val << 32);
1768 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_L(port->id));
1769 dev->stats.rx_ok_bytes += val;
1770
1771 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_DROP_CNT(port->id));
1772 dev->stats.rx_drops += val;
1773
1774 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_BC_CNT(port->id));
1775 dev->stats.rx_broadcast += val;
1776
1777 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_MC_CNT(port->id));
1778 dev->stats.rx_multicast += val;
1779
1780 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ERROR_DROP_CNT(port->id));
1781 dev->stats.rx_errors += val;
1782
1783 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_CRC_ERR_CNT(port->id));
1784 dev->stats.rx_crc_error += val;
1785
1786 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OVERFLOW_DROP_CNT(port->id));
1787 dev->stats.rx_over_errors += val;
1788
1789 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_FRAG_CNT(port->id));
1790 dev->stats.rx_fragment += val;
1791
1792 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_JABBER_CNT(port->id));
1793 dev->stats.rx_jabber += val;
1794
1795 i = 0;
1796 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_RUNT_CNT(port->id));
1797 dev->stats.rx_len[i] += val;
1798
1799 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_H(port->id));
1800 dev->stats.rx_len[i] += ((u64)val << 32);
1801 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_L(port->id));
1802 dev->stats.rx_len[i++] += val;
1803
1804 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_H(port->id));
1805 dev->stats.rx_len[i] += ((u64)val << 32);
1806 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_L(port->id));
1807 dev->stats.rx_len[i++] += val;
1808
1809 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_H(port->id));
1810 dev->stats.rx_len[i] += ((u64)val << 32);
1811 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_L(port->id));
1812 dev->stats.rx_len[i++] += val;
1813
1814 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_H(port->id));
1815 dev->stats.rx_len[i] += ((u64)val << 32);
1816 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_L(port->id));
1817 dev->stats.rx_len[i++] += val;
1818
1819 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_H(port->id));
1820 dev->stats.rx_len[i] += ((u64)val << 32);
1821 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_L(port->id));
1822 dev->stats.rx_len[i++] += val;
1823
1824 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_H(port->id));
1825 dev->stats.rx_len[i] += ((u64)val << 32);
1826 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_L(port->id));
1827 dev->stats.rx_len[i++] += val;
1828
1829 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_LONG_CNT(port->id));
1830 dev->stats.rx_len[i++] += val;
1831
1832 u64_stats_update_end(&dev->stats.syncp);
1833 }
1834
airoha_update_hw_stats(struct airoha_gdm_dev * dev)1835 static void airoha_update_hw_stats(struct airoha_gdm_dev *dev)
1836 {
1837 struct airoha_gdm_port *port = dev->port;
1838 int i;
1839
1840 spin_lock(&port->stats_lock);
1841
1842 for (i = 0; i < ARRAY_SIZE(port->devs); i++) {
1843 if (port->devs[i])
1844 airoha_dev_get_hw_stats(port->devs[i]);
1845 }
1846
1847 /* Reset MIB counters */
1848 airoha_fe_set(dev->eth, REG_FE_GDM_MIB_CLEAR(port->id),
1849 FE_GDM_MIB_RX_CLEAR_MASK | FE_GDM_MIB_TX_CLEAR_MASK);
1850
1851 spin_unlock(&port->stats_lock);
1852 }
1853
airoha_dev_set_xmit_frame_size(struct net_device * netdev)1854 static void airoha_dev_set_xmit_frame_size(struct net_device *netdev)
1855 {
1856 struct airoha_gdm_dev *dev = netdev_priv(netdev);
1857
1858 airoha_ppe_set_xmit_frame_size(dev);
1859 if (!airoha_is_lan_gdm_dev(dev))
1860 airoha_fe_rmw(dev->eth, REG_WAN_MTU0, WAN_MTU0_MASK,
1861 FIELD_PREP(WAN_MTU0_MASK,
1862 VLAN_ETH_HLEN + netdev->mtu));
1863 }
1864
airoha_dev_open(struct net_device * netdev)1865 static int airoha_dev_open(struct net_device *netdev)
1866 {
1867 struct airoha_gdm_dev *dev = netdev_priv(netdev);
1868 struct airoha_gdm_port *port = dev->port;
1869 struct airoha_qdma *qdma = dev->qdma;
1870 u32 pse_port = FE_PSE_PORT_PPE1;
1871 int err;
1872
1873 netif_tx_start_all_queues(netdev);
1874 err = airoha_set_vip_for_gdm_port(dev, true);
1875 if (err)
1876 return err;
1877
1878 if (netdev_uses_dsa(netdev))
1879 airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id),
1880 GDM_STAG_EN_MASK);
1881 else
1882 airoha_fe_clear(qdma->eth, REG_GDM_INGRESS_CFG(port->id),
1883 GDM_STAG_EN_MASK);
1884
1885 airoha_dev_set_xmit_frame_size(netdev);
1886 port->users++;
1887
1888 if (!airoha_is_lan_gdm_dev(dev) &&
1889 airoha_ppe_is_enabled(qdma->eth, 1))
1890 pse_port = FE_PSE_PORT_PPE2;
1891 airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id),
1892 pse_port);
1893
1894 return 0;
1895 }
1896
airoha_dev_stop(struct net_device * netdev)1897 static int airoha_dev_stop(struct net_device *netdev)
1898 {
1899 struct airoha_gdm_dev *dev = netdev_priv(netdev);
1900 struct airoha_gdm_port *port = dev->port;
1901 struct airoha_qdma *qdma = dev->qdma;
1902
1903 netif_tx_disable(netdev);
1904 airoha_set_vip_for_gdm_port(dev, false);
1905
1906 if (--port->users)
1907 airoha_ppe_set_xmit_frame_size(dev);
1908 else
1909 airoha_set_gdm_port_fwd_cfg(qdma->eth,
1910 REG_GDM_FWD_CFG(port->id),
1911 FE_PSE_PORT_DROP);
1912 return 0;
1913 }
1914
airoha_dev_set_macaddr(struct net_device * netdev,void * p)1915 static int airoha_dev_set_macaddr(struct net_device *netdev, void *p)
1916 {
1917 struct airoha_gdm_dev *dev = netdev_priv(netdev);
1918 struct sockaddr *addr = p;
1919 int err;
1920
1921 err = eth_prepare_mac_addr_change(netdev, p);
1922 if (err)
1923 return err;
1924
1925 err = airoha_set_macaddr(dev, addr->sa_data);
1926 if (err)
1927 return err;
1928
1929 eth_commit_mac_addr_change(netdev, p);
1930
1931 return 0;
1932 }
1933
airoha_enable_gdm2_loopback(struct airoha_gdm_dev * dev)1934 static int airoha_enable_gdm2_loopback(struct airoha_gdm_dev *dev)
1935 {
1936 struct airoha_gdm_port *port = dev->port;
1937 struct airoha_eth *eth = dev->eth;
1938 u32 val, pse_port, chan;
1939 int i, src_port;
1940
1941 src_port = eth->soc->ops.get_sport(port, dev->nbq);
1942 if (src_port < 0)
1943 return src_port;
1944
1945 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
1946 FE_PSE_PORT_DROP);
1947 airoha_fe_clear(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
1948 GDM_STRIP_CRC_MASK);
1949
1950 /* Enable GDM2 loopback */
1951 airoha_fe_wr(eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), 0xffffffff);
1952 airoha_fe_wr(eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX), 0xffff);
1953
1954 chan = port->id == AIROHA_GDM3_IDX ? airoha_is_7581(eth) ? 4 : 3 : 0;
1955 airoha_fe_rmw(eth, REG_GDM_LPBK_CFG(AIROHA_GDM2_IDX),
1956 LPBK_CHAN_MASK | LPBK_MODE_MASK | LPBK_EN_MASK,
1957 FIELD_PREP(LPBK_CHAN_MASK, chan) |
1958 LBK_GAP_MODE_MASK | LBK_LEN_MODE_MASK |
1959 LBK_CHAN_MODE_MASK | LPBK_EN_MASK);
1960 /* Forward the traffic to the proper GDM port */
1961 pse_port = port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3
1962 : FE_PSE_PORT_GDM4;
1963 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
1964 pse_port);
1965
1966 /* Disable VIP and IFC for GDM2 */
1967 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX));
1968 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX));
1969
1970 airoha_fe_rmw(eth, REG_FE_WAN_PORT,
1971 WAN1_EN_MASK | WAN1_MASK | WAN0_MASK,
1972 FIELD_PREP(WAN0_MASK, src_port));
1973 val = src_port & SP_CPORT_DFT_MASK;
1974 airoha_fe_rmw(eth,
1975 REG_SP_DFT_CPORT(src_port >> fls(SP_CPORT_DFT_MASK)),
1976 SP_CPORT_MASK(val),
1977 __field_prep(SP_CPORT_MASK(val), FE_PSE_PORT_CDM2));
1978
1979 for (i = 0; i < eth->soc->num_ppe; i++)
1980 airoha_ppe_set_cpu_port(dev, i, AIROHA_GDM2_IDX);
1981
1982 if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) {
1983 u32 mask = FC_ID_OF_SRC_PORT_MASK(dev->nbq);
1984
1985 airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask,
1986 __field_prep(mask, AIROHA_GDM2_IDX));
1987 }
1988
1989 return 0;
1990 }
1991
1992 static struct airoha_gdm_dev *
airoha_get_wan_gdm_dev(struct airoha_eth * eth)1993 airoha_get_wan_gdm_dev(struct airoha_eth *eth)
1994 {
1995 int i;
1996
1997 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
1998 struct airoha_gdm_port *port = eth->ports[i];
1999 int j;
2000
2001 if (!port)
2002 continue;
2003
2004 for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
2005 struct airoha_gdm_dev *dev = port->devs[j];
2006
2007 if (dev && !airoha_is_lan_gdm_dev(dev))
2008 return dev;
2009 }
2010 }
2011
2012 return NULL;
2013 }
2014
airoha_dev_set_qdma(struct airoha_gdm_dev * dev)2015 static void airoha_dev_set_qdma(struct airoha_gdm_dev *dev)
2016 {
2017 struct net_device *netdev = netdev_from_priv(dev);
2018 struct airoha_eth *eth = dev->eth;
2019 int ppe_id;
2020
2021 /* QDMA0 is used for lan ports while QDMA1 is used for WAN ports */
2022 dev->qdma = ð->qdma[!airoha_is_lan_gdm_dev(dev)];
2023 netdev->irq = dev->qdma->irq_banks[0].irq;
2024
2025 ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1);
2026 airoha_ppe_set_cpu_port(dev, ppe_id, airoha_get_fe_port(dev));
2027 }
2028
airoha_dev_init(struct net_device * netdev)2029 static int airoha_dev_init(struct net_device *netdev)
2030 {
2031 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2032 struct airoha_gdm_port *port = dev->port;
2033
2034 switch (port->id) {
2035 case AIROHA_GDM3_IDX:
2036 case AIROHA_GDM4_IDX:
2037 if (airoha_get_wan_gdm_dev(dev->eth))
2038 break;
2039 fallthrough;
2040 case AIROHA_GDM2_IDX:
2041 /* GDM2 is always used as wan */
2042 dev->flags |= AIROHA_PRIV_F_WAN;
2043 break;
2044 default:
2045 break;
2046 }
2047
2048 airoha_dev_set_qdma(dev);
2049 airoha_set_macaddr(dev, netdev->dev_addr);
2050
2051 if (!airoha_is_lan_gdm_dev(dev) &&
2052 (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX)) {
2053 int err;
2054
2055 err = airoha_enable_gdm2_loopback(dev);
2056 if (err)
2057 return err;
2058 }
2059
2060 return 0;
2061 }
2062
airoha_dev_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * storage)2063 static void airoha_dev_get_stats64(struct net_device *netdev,
2064 struct rtnl_link_stats64 *storage)
2065 {
2066 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2067 unsigned int start;
2068
2069 airoha_update_hw_stats(dev);
2070 do {
2071 start = u64_stats_fetch_begin(&dev->stats.syncp);
2072 storage->rx_packets = dev->stats.rx_ok_pkts;
2073 storage->tx_packets = dev->stats.tx_ok_pkts;
2074 storage->rx_bytes = dev->stats.rx_ok_bytes;
2075 storage->tx_bytes = dev->stats.tx_ok_bytes;
2076 storage->multicast = dev->stats.rx_multicast;
2077 storage->rx_errors = dev->stats.rx_errors;
2078 storage->rx_dropped = dev->stats.rx_drops;
2079 storage->tx_dropped = dev->stats.tx_drops;
2080 storage->rx_crc_errors = dev->stats.rx_crc_error;
2081 storage->rx_over_errors = dev->stats.rx_over_errors;
2082 } while (u64_stats_fetch_retry(&dev->stats.syncp, start));
2083 }
2084
airoha_dev_change_mtu(struct net_device * netdev,int mtu)2085 static int airoha_dev_change_mtu(struct net_device *netdev, int mtu)
2086 {
2087 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2088 struct airoha_gdm_port *port = dev->port;
2089
2090 WRITE_ONCE(netdev->mtu, mtu);
2091 if (port->users)
2092 airoha_dev_set_xmit_frame_size(netdev);
2093
2094 return 0;
2095 }
2096
airoha_dev_select_queue(struct net_device * netdev,struct sk_buff * skb,struct net_device * sb_dev)2097 static u16 airoha_dev_select_queue(struct net_device *netdev,
2098 struct sk_buff *skb,
2099 struct net_device *sb_dev)
2100 {
2101 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2102 struct airoha_gdm_port *port = dev->port;
2103 int queue, channel;
2104
2105 /* For dsa device select QoS channel according to the dsa user port
2106 * index, rely on port id otherwise. Select QoS queue based on the
2107 * skb priority.
2108 */
2109 channel = netdev_uses_dsa(netdev) ? skb_get_queue_mapping(skb) : port->id;
2110 channel = channel % AIROHA_NUM_QOS_CHANNELS;
2111 queue = skb->priority % AIROHA_NUM_QOS_QUEUES;
2112 queue = channel * AIROHA_NUM_QOS_QUEUES + queue;
2113
2114 return queue < netdev->num_tx_queues ? queue : 0;
2115 }
2116
airoha_get_dsa_tag(struct sk_buff * skb,struct net_device * dev)2117 static u32 airoha_get_dsa_tag(struct sk_buff *skb, struct net_device *dev)
2118 {
2119 #if IS_ENABLED(CONFIG_NET_DSA)
2120 struct ethhdr *ehdr;
2121 u8 xmit_tpid;
2122 u16 tag;
2123
2124 if (!netdev_uses_dsa(dev))
2125 return 0;
2126
2127 if (dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK)
2128 return 0;
2129
2130 if (skb_cow_head(skb, 0))
2131 return 0;
2132
2133 ehdr = (struct ethhdr *)skb->data;
2134 tag = be16_to_cpu(ehdr->h_proto);
2135 xmit_tpid = tag >> 8;
2136
2137 switch (xmit_tpid) {
2138 case MTK_HDR_XMIT_TAGGED_TPID_8100:
2139 ehdr->h_proto = cpu_to_be16(ETH_P_8021Q);
2140 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_8100 << 8);
2141 break;
2142 case MTK_HDR_XMIT_TAGGED_TPID_88A8:
2143 ehdr->h_proto = cpu_to_be16(ETH_P_8021AD);
2144 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_88A8 << 8);
2145 break;
2146 default:
2147 /* PPE module requires untagged DSA packets to work properly,
2148 * so move DSA tag to DMA descriptor.
2149 */
2150 memmove(skb->data + MTK_HDR_LEN, skb->data, 2 * ETH_ALEN);
2151 __skb_pull(skb, MTK_HDR_LEN);
2152 break;
2153 }
2154
2155 return tag;
2156 #else
2157 return 0;
2158 #endif
2159 }
2160
airoha_get_fe_port(struct airoha_gdm_dev * dev)2161 int airoha_get_fe_port(struct airoha_gdm_dev *dev)
2162 {
2163 struct airoha_gdm_port *port = dev->port;
2164 struct airoha_eth *eth = dev->eth;
2165
2166 switch (eth->soc->version) {
2167 case 0x7583:
2168 return port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3
2169 : port->id;
2170 case 0x7581:
2171 default:
2172 return port->id == AIROHA_GDM4_IDX ? FE_PSE_PORT_GDM4
2173 : port->id;
2174 }
2175 }
2176
airoha_dev_xmit(struct sk_buff * skb,struct net_device * netdev)2177 static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
2178 struct net_device *netdev)
2179 {
2180 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2181 struct airoha_qdma *qdma = dev->qdma;
2182 u32 nr_frags, tag, msg0, msg1, len;
2183 struct airoha_queue_entry *e;
2184 struct netdev_queue *txq;
2185 struct airoha_queue *q;
2186 LIST_HEAD(tx_list);
2187 dma_addr_t addr;
2188 int i = 0, qid;
2189 u16 index;
2190 u8 fport;
2191
2192 qid = airoha_qdma_get_txq(qdma, skb_get_queue_mapping(skb));
2193 tag = airoha_get_dsa_tag(skb, netdev);
2194
2195 msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
2196 qid / AIROHA_NUM_QOS_QUEUES) |
2197 FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
2198 qid % AIROHA_NUM_QOS_QUEUES) |
2199 FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK, tag);
2200 if (skb->ip_summed == CHECKSUM_PARTIAL)
2201 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TCO_MASK, 1) |
2202 FIELD_PREP(QDMA_ETH_TXMSG_UCO_MASK, 1) |
2203 FIELD_PREP(QDMA_ETH_TXMSG_ICO_MASK, 1);
2204
2205 /* TSO: fill MSS info in tcp checksum field */
2206 if (skb_is_gso(skb)) {
2207 if (skb_cow_head(skb, 0))
2208 goto error;
2209
2210 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 |
2211 SKB_GSO_TCPV6)) {
2212 __be16 csum = cpu_to_be16(skb_shinfo(skb)->gso_size);
2213
2214 tcp_hdr(skb)->check = (__force __sum16)csum;
2215 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TSO_MASK, 1);
2216 }
2217 }
2218
2219 fport = airoha_get_fe_port(dev);
2220 msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) |
2221 FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
2222 FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
2223
2224 q = &qdma->q_tx[qid];
2225 if (WARN_ON_ONCE(!q->ndesc))
2226 goto error;
2227
2228 spin_lock_bh(&q->lock);
2229
2230 if (q->flushing)
2231 goto error_unlock;
2232
2233 txq = skb_get_tx_queue(netdev, skb);
2234 nr_frags = 1 + skb_shinfo(skb)->nr_frags;
2235
2236 if (q->queued + nr_frags >= q->ndesc) {
2237 /* not enough space in the queue */
2238 netif_tx_stop_queue(txq);
2239 q->txq_stopped = true;
2240 spin_unlock_bh(&q->lock);
2241 return NETDEV_TX_BUSY;
2242 }
2243
2244 e = list_first_entry(&q->tx_list, struct airoha_queue_entry,
2245 list);
2246 len = skb_headlen(skb);
2247 addr = dma_map_single(netdev->dev.parent, skb->data, len,
2248 DMA_TO_DEVICE);
2249 if (unlikely(dma_mapping_error(netdev->dev.parent, addr)))
2250 goto error_unlock;
2251
2252 e->dma_type = AIROHA_DMA_MAP_SINGLE;
2253 index = e - q->entry;
2254
2255 while (true) {
2256 struct airoha_qdma_desc *desc = &q->desc[index];
2257 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2258 u32 val;
2259
2260 list_move_tail(&e->list, &tx_list);
2261 e->skb = i == nr_frags - 1 ? skb : NULL;
2262 e->dma_addr = addr;
2263 e->dma_len = len;
2264
2265 e = list_first_entry(&q->tx_list, struct airoha_queue_entry,
2266 list);
2267 index = e - q->entry;
2268
2269 val = FIELD_PREP(QDMA_DESC_LEN_MASK, len);
2270 if (i < nr_frags - 1)
2271 val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1);
2272 WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
2273 WRITE_ONCE(desc->addr, cpu_to_le32(addr));
2274 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, index);
2275 WRITE_ONCE(desc->data, cpu_to_le32(val));
2276 WRITE_ONCE(desc->msg0, cpu_to_le32(msg0));
2277 WRITE_ONCE(desc->msg1, cpu_to_le32(msg1));
2278 WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
2279
2280 if (++i == nr_frags)
2281 break;
2282
2283 len = skb_frag_size(frag);
2284 addr = skb_frag_dma_map(netdev->dev.parent, frag, 0, len,
2285 DMA_TO_DEVICE);
2286 if (unlikely(dma_mapping_error(netdev->dev.parent, addr)))
2287 goto error_unmap;
2288
2289 e->dma_type = AIROHA_DMA_MAP_PAGE;
2290 }
2291 q->queued += i;
2292
2293 skb_tx_timestamp(skb);
2294 netdev_tx_sent_queue(txq, skb->len);
2295 if (q->ndesc - q->queued < q->free_thr) {
2296 netif_tx_stop_queue(txq);
2297 q->txq_stopped = true;
2298 }
2299
2300 if (netif_xmit_stopped(txq) || !netdev_xmit_more())
2301 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
2302 TX_RING_CPU_IDX_MASK,
2303 FIELD_PREP(TX_RING_CPU_IDX_MASK, index));
2304
2305 spin_unlock_bh(&q->lock);
2306
2307 return NETDEV_TX_OK;
2308
2309 error_unmap:
2310 list_for_each_entry(e, &tx_list, list)
2311 airoha_unmap_xmit_buf(dev->eth, e);
2312 list_splice(&tx_list, &q->tx_list);
2313 error_unlock:
2314 spin_unlock_bh(&q->lock);
2315 error:
2316 dev_kfree_skb_any(skb);
2317 netdev->stats.tx_dropped++;
2318
2319 return NETDEV_TX_OK;
2320 }
2321
airoha_ethtool_get_drvinfo(struct net_device * netdev,struct ethtool_drvinfo * info)2322 static void airoha_ethtool_get_drvinfo(struct net_device *netdev,
2323 struct ethtool_drvinfo *info)
2324 {
2325 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2326 struct airoha_eth *eth = dev->eth;
2327
2328 strscpy(info->driver, eth->dev->driver->name, sizeof(info->driver));
2329 strscpy(info->bus_info, dev_name(eth->dev), sizeof(info->bus_info));
2330 }
2331
airoha_ethtool_get_mac_stats(struct net_device * netdev,struct ethtool_eth_mac_stats * stats)2332 static void airoha_ethtool_get_mac_stats(struct net_device *netdev,
2333 struct ethtool_eth_mac_stats *stats)
2334 {
2335 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2336 unsigned int start;
2337
2338 airoha_update_hw_stats(dev);
2339 do {
2340 start = u64_stats_fetch_begin(&dev->stats.syncp);
2341 stats->FramesTransmittedOK = dev->stats.tx_ok_pkts;
2342 stats->OctetsTransmittedOK = dev->stats.tx_ok_bytes;
2343 stats->MulticastFramesXmittedOK = dev->stats.tx_multicast;
2344 stats->BroadcastFramesXmittedOK = dev->stats.tx_broadcast;
2345 stats->FramesReceivedOK = dev->stats.rx_ok_pkts;
2346 stats->OctetsReceivedOK = dev->stats.rx_ok_bytes;
2347 stats->BroadcastFramesReceivedOK = dev->stats.rx_broadcast;
2348 } while (u64_stats_fetch_retry(&dev->stats.syncp, start));
2349 }
2350
2351 static const struct ethtool_rmon_hist_range airoha_ethtool_rmon_ranges[] = {
2352 { 0, 64 },
2353 { 65, 127 },
2354 { 128, 255 },
2355 { 256, 511 },
2356 { 512, 1023 },
2357 { 1024, 1518 },
2358 { 1519, 10239 },
2359 {},
2360 };
2361
2362 static void
airoha_ethtool_get_rmon_stats(struct net_device * netdev,struct ethtool_rmon_stats * stats,const struct ethtool_rmon_hist_range ** ranges)2363 airoha_ethtool_get_rmon_stats(struct net_device *netdev,
2364 struct ethtool_rmon_stats *stats,
2365 const struct ethtool_rmon_hist_range **ranges)
2366 {
2367 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2368 struct airoha_hw_stats *hw_stats = &dev->stats;
2369 unsigned int start;
2370
2371 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) !=
2372 ARRAY_SIZE(hw_stats->tx_len) + 1);
2373 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) !=
2374 ARRAY_SIZE(hw_stats->rx_len) + 1);
2375
2376 *ranges = airoha_ethtool_rmon_ranges;
2377 airoha_update_hw_stats(dev);
2378 do {
2379 int i;
2380
2381 start = u64_stats_fetch_begin(&dev->stats.syncp);
2382 stats->fragments = hw_stats->rx_fragment;
2383 stats->jabbers = hw_stats->rx_jabber;
2384 for (i = 0; i < ARRAY_SIZE(airoha_ethtool_rmon_ranges) - 1;
2385 i++) {
2386 stats->hist[i] = hw_stats->rx_len[i];
2387 stats->hist_tx[i] = hw_stats->tx_len[i];
2388 }
2389 } while (u64_stats_fetch_retry(&dev->stats.syncp, start));
2390 }
2391
airoha_qdma_set_chan_tx_sched(struct net_device * netdev,int channel,enum tx_sched_mode mode,const u16 * weights,u8 n_weights)2392 static int airoha_qdma_set_chan_tx_sched(struct net_device *netdev,
2393 int channel, enum tx_sched_mode mode,
2394 const u16 *weights, u8 n_weights)
2395 {
2396 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2397 int i;
2398
2399 for (i = 0; i < AIROHA_NUM_QOS_QUEUES; i++)
2400 airoha_qdma_clear(dev->qdma, REG_QUEUE_CLOSE_CFG(channel),
2401 TXQ_DISABLE_CHAN_QUEUE_MASK(channel, i));
2402
2403 for (i = 0; i < n_weights; i++) {
2404 u32 status;
2405 int err;
2406
2407 airoha_qdma_wr(dev->qdma, REG_TXWRR_WEIGHT_CFG,
2408 TWRR_RW_CMD_MASK |
2409 FIELD_PREP(TWRR_CHAN_IDX_MASK, channel) |
2410 FIELD_PREP(TWRR_QUEUE_IDX_MASK, i) |
2411 FIELD_PREP(TWRR_VALUE_MASK, weights[i]));
2412 err = read_poll_timeout(airoha_qdma_rr, status,
2413 status & TWRR_RW_CMD_DONE,
2414 USEC_PER_MSEC, 10 * USEC_PER_MSEC,
2415 true, dev->qdma, REG_TXWRR_WEIGHT_CFG);
2416 if (err)
2417 return err;
2418 }
2419
2420 airoha_qdma_rmw(dev->qdma, REG_CHAN_QOS_MODE(channel >> 3),
2421 CHAN_QOS_MODE_MASK(channel),
2422 __field_prep(CHAN_QOS_MODE_MASK(channel), mode));
2423
2424 return 0;
2425 }
2426
airoha_qdma_set_tx_prio_sched(struct net_device * dev,int channel)2427 static int airoha_qdma_set_tx_prio_sched(struct net_device *dev, int channel)
2428 {
2429 static const u16 w[AIROHA_NUM_QOS_QUEUES] = {};
2430
2431 return airoha_qdma_set_chan_tx_sched(dev, channel, TC_SCH_SP, w,
2432 ARRAY_SIZE(w));
2433 }
2434
airoha_qdma_set_tx_ets_sched(struct net_device * dev,int channel,struct tc_ets_qopt_offload * opt)2435 static int airoha_qdma_set_tx_ets_sched(struct net_device *dev, int channel,
2436 struct tc_ets_qopt_offload *opt)
2437 {
2438 struct tc_ets_qopt_offload_replace_params *p = &opt->replace_params;
2439 enum tx_sched_mode mode = TC_SCH_SP;
2440 u16 w[AIROHA_NUM_QOS_QUEUES] = {};
2441 int i, nstrict = 0;
2442
2443 if (p->bands > AIROHA_NUM_QOS_QUEUES)
2444 return -EINVAL;
2445
2446 for (i = 0; i < p->bands; i++) {
2447 if (!p->quanta[i])
2448 nstrict++;
2449 }
2450
2451 /* this configuration is not supported by the hw */
2452 if (nstrict == AIROHA_NUM_QOS_QUEUES - 1)
2453 return -EINVAL;
2454
2455 /* EN7581 SoC supports fixed QoS band priority where WRR queues have
2456 * lowest priorities with respect to SP ones.
2457 * e.g: WRR0, WRR1, .., WRRm, SP0, SP1, .., SPn
2458 */
2459 for (i = 0; i < nstrict; i++) {
2460 if (p->priomap[p->bands - i - 1] != i)
2461 return -EINVAL;
2462 }
2463
2464 for (i = 0; i < p->bands - nstrict; i++) {
2465 if (p->priomap[i] != nstrict + i)
2466 return -EINVAL;
2467
2468 w[i] = p->weights[nstrict + i];
2469 }
2470
2471 if (!nstrict)
2472 mode = TC_SCH_WRR8;
2473 else if (nstrict < AIROHA_NUM_QOS_QUEUES - 1)
2474 mode = nstrict + 1;
2475
2476 return airoha_qdma_set_chan_tx_sched(dev, channel, mode, w,
2477 ARRAY_SIZE(w));
2478 }
2479
airoha_qdma_get_tx_ets_stats(struct net_device * netdev,int channel,struct tc_ets_qopt_offload * opt)2480 static int airoha_qdma_get_tx_ets_stats(struct net_device *netdev, int channel,
2481 struct tc_ets_qopt_offload *opt)
2482 {
2483 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2484 struct airoha_qdma *qdma = dev->qdma;
2485
2486 u64 cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1));
2487 u64 fwd_tx_packets = airoha_qdma_rr(qdma,
2488 REG_CNTR_VAL((channel << 1) + 1));
2489 u64 tx_packets = (cpu_tx_packets - dev->cpu_tx_packets) +
2490 (fwd_tx_packets - dev->fwd_tx_packets);
2491
2492 _bstats_update(opt->stats.bstats, 0, tx_packets);
2493 dev->cpu_tx_packets = cpu_tx_packets;
2494 dev->fwd_tx_packets = fwd_tx_packets;
2495
2496 return 0;
2497 }
2498
airoha_tc_setup_qdisc_ets(struct net_device * dev,struct tc_ets_qopt_offload * opt)2499 static int airoha_tc_setup_qdisc_ets(struct net_device *dev,
2500 struct tc_ets_qopt_offload *opt)
2501 {
2502 int channel;
2503
2504 if (opt->parent == TC_H_ROOT)
2505 return -EINVAL;
2506
2507 channel = TC_H_MAJ(opt->handle) >> 16;
2508 channel = channel % AIROHA_NUM_QOS_CHANNELS;
2509
2510 switch (opt->command) {
2511 case TC_ETS_REPLACE:
2512 return airoha_qdma_set_tx_ets_sched(dev, channel, opt);
2513 case TC_ETS_DESTROY:
2514 /* PRIO is default qdisc scheduler */
2515 return airoha_qdma_set_tx_prio_sched(dev, channel);
2516 case TC_ETS_STATS:
2517 return airoha_qdma_get_tx_ets_stats(dev, channel, opt);
2518 default:
2519 return -EOPNOTSUPP;
2520 }
2521 }
2522
airoha_qdma_get_rl_param(struct airoha_qdma * qdma,int queue_id,u32 addr,enum trtcm_param_type param,u32 * val_low,u32 * val_high)2523 static int airoha_qdma_get_rl_param(struct airoha_qdma *qdma, int queue_id,
2524 u32 addr, enum trtcm_param_type param,
2525 u32 *val_low, u32 *val_high)
2526 {
2527 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id);
2528 u32 val, config = FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) |
2529 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) |
2530 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx);
2531
2532 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config);
2533 if (read_poll_timeout(airoha_qdma_rr, val,
2534 val & RATE_LIMIT_PARAM_RW_DONE_MASK,
2535 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, qdma,
2536 REG_TRTCM_CFG_PARAM(addr)))
2537 return -ETIMEDOUT;
2538
2539 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr));
2540 if (val_high)
2541 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr));
2542
2543 return 0;
2544 }
2545
airoha_qdma_set_rl_param(struct airoha_qdma * qdma,int queue_id,u32 addr,enum trtcm_param_type param,u32 val)2546 static int airoha_qdma_set_rl_param(struct airoha_qdma *qdma, int queue_id,
2547 u32 addr, enum trtcm_param_type param,
2548 u32 val)
2549 {
2550 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id);
2551 u32 config = RATE_LIMIT_PARAM_RW_MASK |
2552 FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) |
2553 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) |
2554 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx);
2555
2556 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val);
2557 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config);
2558
2559 return read_poll_timeout(airoha_qdma_rr, val,
2560 val & RATE_LIMIT_PARAM_RW_DONE_MASK,
2561 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true,
2562 qdma, REG_TRTCM_CFG_PARAM(addr));
2563 }
2564
airoha_qdma_set_rl_config(struct airoha_qdma * qdma,int queue_id,u32 addr,bool enable,u32 enable_mask)2565 static int airoha_qdma_set_rl_config(struct airoha_qdma *qdma, int queue_id,
2566 u32 addr, bool enable, u32 enable_mask)
2567 {
2568 u32 val;
2569 int err;
2570
2571 err = airoha_qdma_get_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE,
2572 &val, NULL);
2573 if (err)
2574 return err;
2575
2576 val = enable ? val | enable_mask : val & ~enable_mask;
2577
2578 return airoha_qdma_set_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE,
2579 val);
2580 }
2581
airoha_qdma_set_rl_token_bucket(struct airoha_qdma * qdma,int queue_id,u32 rate_val,u32 bucket_size)2582 static int airoha_qdma_set_rl_token_bucket(struct airoha_qdma *qdma,
2583 int queue_id, u32 rate_val,
2584 u32 bucket_size)
2585 {
2586 u32 val, config, tick, unit, rate, rate_frac;
2587 int err;
2588
2589 err = airoha_qdma_get_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG,
2590 TRTCM_MISC_MODE, &config, NULL);
2591 if (err)
2592 return err;
2593
2594 val = airoha_qdma_rr(qdma, REG_INGRESS_TRTCM_CFG);
2595 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val);
2596 if (config & TRTCM_TICK_SEL)
2597 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val);
2598 if (!tick)
2599 return -EINVAL;
2600
2601 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick;
2602 if (!unit)
2603 return -EINVAL;
2604
2605 rate = rate_val / unit;
2606 rate_frac = rate_val % unit;
2607 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit;
2608 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) |
2609 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac);
2610
2611 err = airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG,
2612 TRTCM_TOKEN_RATE_MODE, rate);
2613 if (err)
2614 return err;
2615
2616 val = bucket_size;
2617 if (!(config & TRTCM_PKT_MODE))
2618 val = max_t(u32, val, MIN_TOKEN_SIZE);
2619 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET);
2620
2621 return airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG,
2622 TRTCM_BUCKETSIZE_SHIFT_MODE, val);
2623 }
2624
airoha_qdma_init_rl_config(struct airoha_qdma * qdma,int queue_id,bool enable,enum trtcm_unit_type unit)2625 static int airoha_qdma_init_rl_config(struct airoha_qdma *qdma, int queue_id,
2626 bool enable, enum trtcm_unit_type unit)
2627 {
2628 bool tick_sel = queue_id == 0 || queue_id == 2 || queue_id == 8;
2629 enum trtcm_param mode = TRTCM_METER_MODE;
2630 int err;
2631
2632 mode |= unit == TRTCM_PACKET_UNIT ? TRTCM_PKT_MODE : 0;
2633 err = airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG,
2634 enable, mode);
2635 if (err)
2636 return err;
2637
2638 return airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG,
2639 tick_sel, TRTCM_TICK_SEL);
2640 }
2641
airoha_qdma_get_trtcm_param(struct airoha_qdma * qdma,int channel,u32 addr,enum trtcm_param_type param,enum trtcm_mode_type mode,u32 * val_low,u32 * val_high)2642 static int airoha_qdma_get_trtcm_param(struct airoha_qdma *qdma, int channel,
2643 u32 addr, enum trtcm_param_type param,
2644 enum trtcm_mode_type mode,
2645 u32 *val_low, u32 *val_high)
2646 {
2647 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel);
2648 u32 val, config = FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) |
2649 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) |
2650 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) |
2651 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode);
2652
2653 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config);
2654 if (read_poll_timeout(airoha_qdma_rr, val,
2655 val & TRTCM_PARAM_RW_DONE_MASK,
2656 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true,
2657 qdma, REG_TRTCM_CFG_PARAM(addr)))
2658 return -ETIMEDOUT;
2659
2660 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr));
2661 if (val_high)
2662 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr));
2663
2664 return 0;
2665 }
2666
airoha_qdma_set_trtcm_param(struct airoha_qdma * qdma,int channel,u32 addr,enum trtcm_param_type param,enum trtcm_mode_type mode,u32 val)2667 static int airoha_qdma_set_trtcm_param(struct airoha_qdma *qdma, int channel,
2668 u32 addr, enum trtcm_param_type param,
2669 enum trtcm_mode_type mode, u32 val)
2670 {
2671 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel);
2672 u32 config = TRTCM_PARAM_RW_MASK |
2673 FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) |
2674 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) |
2675 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) |
2676 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode);
2677
2678 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val);
2679 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config);
2680
2681 return read_poll_timeout(airoha_qdma_rr, val,
2682 val & TRTCM_PARAM_RW_DONE_MASK,
2683 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true,
2684 qdma, REG_TRTCM_CFG_PARAM(addr));
2685 }
2686
airoha_qdma_set_trtcm_config(struct airoha_qdma * qdma,int channel,u32 addr,enum trtcm_mode_type mode,bool enable,u32 enable_mask)2687 static int airoha_qdma_set_trtcm_config(struct airoha_qdma *qdma, int channel,
2688 u32 addr, enum trtcm_mode_type mode,
2689 bool enable, u32 enable_mask)
2690 {
2691 u32 val;
2692
2693 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE,
2694 mode, &val, NULL))
2695 return -EINVAL;
2696
2697 val = enable ? val | enable_mask : val & ~enable_mask;
2698
2699 return airoha_qdma_set_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE,
2700 mode, val);
2701 }
2702
airoha_qdma_set_trtcm_token_bucket(struct airoha_qdma * qdma,int channel,u32 addr,enum trtcm_mode_type mode,u32 rate_val,u32 bucket_size)2703 static int airoha_qdma_set_trtcm_token_bucket(struct airoha_qdma *qdma,
2704 int channel, u32 addr,
2705 enum trtcm_mode_type mode,
2706 u32 rate_val, u32 bucket_size)
2707 {
2708 u32 val, config, tick, unit, rate, rate_frac;
2709 int err;
2710
2711 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE,
2712 mode, &config, NULL))
2713 return -EINVAL;
2714
2715 val = airoha_qdma_rr(qdma, addr);
2716 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val);
2717 if (config & TRTCM_TICK_SEL)
2718 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val);
2719 if (!tick)
2720 return -EINVAL;
2721
2722 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick;
2723 if (!unit)
2724 return -EINVAL;
2725
2726 rate = rate_val / unit;
2727 rate_frac = rate_val % unit;
2728 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit;
2729 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) |
2730 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac);
2731
2732 err = airoha_qdma_set_trtcm_param(qdma, channel, addr,
2733 TRTCM_TOKEN_RATE_MODE, mode, rate);
2734 if (err)
2735 return err;
2736
2737 val = max_t(u32, bucket_size, MIN_TOKEN_SIZE);
2738 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET);
2739
2740 return airoha_qdma_set_trtcm_param(qdma, channel, addr,
2741 TRTCM_BUCKETSIZE_SHIFT_MODE,
2742 mode, val);
2743 }
2744
airoha_qdma_set_tx_rate_limit(struct net_device * netdev,int channel,u32 rate,u32 bucket_size)2745 static int airoha_qdma_set_tx_rate_limit(struct net_device *netdev,
2746 int channel, u32 rate,
2747 u32 bucket_size)
2748 {
2749 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2750 int i, err;
2751
2752 for (i = 0; i <= TRTCM_PEAK_MODE; i++) {
2753 err = airoha_qdma_set_trtcm_config(dev->qdma, channel,
2754 REG_EGRESS_TRTCM_CFG, i,
2755 !!rate, TRTCM_METER_MODE);
2756 if (err)
2757 return err;
2758
2759 err = airoha_qdma_set_trtcm_token_bucket(dev->qdma, channel,
2760 REG_EGRESS_TRTCM_CFG,
2761 i, rate, bucket_size);
2762 if (err)
2763 return err;
2764 }
2765
2766 return 0;
2767 }
2768
airoha_tc_htb_modify_queue(struct net_device * dev,struct tc_htb_qopt_offload * opt)2769 static int airoha_tc_htb_modify_queue(struct net_device *dev,
2770 struct tc_htb_qopt_offload *opt)
2771 {
2772 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS;
2773 u32 rate = div_u64(opt->rate, 1000) << 3; /* kbps */
2774 int err;
2775
2776 if (opt->parent_classid != TC_HTB_CLASSID_ROOT) {
2777 NL_SET_ERR_MSG_MOD(opt->extack, "invalid parent classid");
2778 return -EINVAL;
2779 }
2780
2781 err = airoha_qdma_set_tx_rate_limit(dev, channel, rate, opt->quantum);
2782 if (err)
2783 NL_SET_ERR_MSG_MOD(opt->extack,
2784 "failed configuring htb offload");
2785
2786 return err;
2787 }
2788
airoha_tc_htb_alloc_leaf_queue(struct net_device * netdev,struct tc_htb_qopt_offload * opt)2789 static int airoha_tc_htb_alloc_leaf_queue(struct net_device *netdev,
2790 struct tc_htb_qopt_offload *opt)
2791 {
2792 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS;
2793 int err, num_tx_queues = AIROHA_NUM_TX_RING + channel + 1;
2794 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2795 struct airoha_qdma *qdma = dev->qdma;
2796
2797 /* Here we need to check the requested QDMA channel is not already
2798 * in use by another net_device running on the same QDMA block.
2799 */
2800 if (test_and_set_bit(channel, qdma->qos_channel_map)) {
2801 NL_SET_ERR_MSG_MOD(opt->extack,
2802 "qdma qos channel already in use");
2803 return -EBUSY;
2804 }
2805
2806 err = airoha_tc_htb_modify_queue(netdev, opt);
2807 if (err)
2808 goto error;
2809
2810 if (num_tx_queues > netdev->real_num_tx_queues) {
2811 err = netif_set_real_num_tx_queues(netdev, num_tx_queues);
2812 if (err) {
2813 airoha_qdma_set_tx_rate_limit(netdev, channel, 0,
2814 opt->quantum);
2815 NL_SET_ERR_MSG_MOD(opt->extack,
2816 "failed setting real_num_tx_queues");
2817 goto error;
2818 }
2819 }
2820
2821 set_bit(channel, dev->qos_sq_bmap);
2822 opt->qid = AIROHA_NUM_TX_RING + channel;
2823
2824 return 0;
2825 error:
2826 clear_bit(channel, qdma->qos_channel_map);
2827
2828 return err;
2829 }
2830
airoha_qdma_set_rx_meter(struct airoha_gdm_dev * dev,u32 rate,u32 bucket_size,enum trtcm_unit_type unit_type)2831 static int airoha_qdma_set_rx_meter(struct airoha_gdm_dev *dev,
2832 u32 rate, u32 bucket_size,
2833 enum trtcm_unit_type unit_type)
2834 {
2835 struct airoha_qdma *qdma = dev->qdma;
2836 int i;
2837
2838 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
2839 int err;
2840
2841 if (!qdma->q_rx[i].ndesc)
2842 continue;
2843
2844 err = airoha_qdma_init_rl_config(qdma, i, !!rate, unit_type);
2845 if (err)
2846 return err;
2847
2848 err = airoha_qdma_set_rl_token_bucket(qdma, i, rate,
2849 bucket_size);
2850 if (err)
2851 return err;
2852 }
2853
2854 return 0;
2855 }
2856
airoha_tc_matchall_act_validate(struct tc_cls_matchall_offload * f)2857 static int airoha_tc_matchall_act_validate(struct tc_cls_matchall_offload *f)
2858 {
2859 const struct flow_action *actions = &f->rule->action;
2860 const struct flow_action_entry *act;
2861
2862 if (!flow_action_has_entries(actions)) {
2863 NL_SET_ERR_MSG_MOD(f->common.extack,
2864 "filter run with no actions");
2865 return -EINVAL;
2866 }
2867
2868 if (!flow_offload_has_one_action(actions)) {
2869 NL_SET_ERR_MSG_MOD(f->common.extack,
2870 "only once action per filter is supported");
2871 return -EOPNOTSUPP;
2872 }
2873
2874 act = &actions->entries[0];
2875 if (act->id != FLOW_ACTION_POLICE) {
2876 NL_SET_ERR_MSG_MOD(f->common.extack, "unsupported action");
2877 return -EOPNOTSUPP;
2878 }
2879
2880 if (act->police.exceed.act_id != FLOW_ACTION_DROP) {
2881 NL_SET_ERR_MSG_MOD(f->common.extack,
2882 "invalid exceed action id");
2883 return -EOPNOTSUPP;
2884 }
2885
2886 if (act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
2887 NL_SET_ERR_MSG_MOD(f->common.extack,
2888 "invalid notexceed action id");
2889 return -EOPNOTSUPP;
2890 }
2891
2892 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT &&
2893 !flow_action_is_last_entry(actions, act)) {
2894 NL_SET_ERR_MSG_MOD(f->common.extack,
2895 "action accept must be last");
2896 return -EOPNOTSUPP;
2897 }
2898
2899 if (act->police.peakrate_bytes_ps || act->police.avrate ||
2900 act->police.overhead || act->police.mtu) {
2901 NL_SET_ERR_MSG_MOD(f->common.extack,
2902 "peakrate/avrate/overhead/mtu unsupported");
2903 return -EOPNOTSUPP;
2904 }
2905
2906 return 0;
2907 }
2908
airoha_dev_tc_matchall(struct net_device * netdev,struct tc_cls_matchall_offload * f)2909 static int airoha_dev_tc_matchall(struct net_device *netdev,
2910 struct tc_cls_matchall_offload *f)
2911 {
2912 enum trtcm_unit_type unit_type = TRTCM_BYTE_UNIT;
2913 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2914 u32 rate = 0, bucket_size = 0;
2915
2916 switch (f->command) {
2917 case TC_CLSMATCHALL_REPLACE: {
2918 const struct flow_action_entry *act;
2919 int err;
2920
2921 err = airoha_tc_matchall_act_validate(f);
2922 if (err)
2923 return err;
2924
2925 act = &f->rule->action.entries[0];
2926 if (act->police.rate_pkt_ps) {
2927 rate = act->police.rate_pkt_ps;
2928 bucket_size = act->police.burst_pkt;
2929 unit_type = TRTCM_PACKET_UNIT;
2930 } else {
2931 rate = div_u64(act->police.rate_bytes_ps, 1000);
2932 rate = rate << 3; /* Kbps */
2933 bucket_size = act->police.burst;
2934 }
2935 fallthrough;
2936 }
2937 case TC_CLSMATCHALL_DESTROY:
2938 return airoha_qdma_set_rx_meter(dev, rate, bucket_size,
2939 unit_type);
2940 default:
2941 return -EOPNOTSUPP;
2942 }
2943 }
2944
airoha_dev_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)2945 static int airoha_dev_setup_tc_block_cb(enum tc_setup_type type,
2946 void *type_data, void *cb_priv)
2947 {
2948 struct net_device *netdev = cb_priv;
2949 struct airoha_gdm_dev *dev = netdev_priv(netdev);
2950 struct airoha_eth *eth = dev->eth;
2951
2952 if (!tc_can_offload(netdev))
2953 return -EOPNOTSUPP;
2954
2955 switch (type) {
2956 case TC_SETUP_CLSFLOWER:
2957 return airoha_ppe_setup_tc_block_cb(ð->ppe->dev, type_data);
2958 case TC_SETUP_CLSMATCHALL:
2959 return airoha_dev_tc_matchall(netdev, type_data);
2960 default:
2961 return -EOPNOTSUPP;
2962 }
2963 }
2964
airoha_dev_setup_tc_block(struct net_device * dev,struct flow_block_offload * f)2965 static int airoha_dev_setup_tc_block(struct net_device *dev,
2966 struct flow_block_offload *f)
2967 {
2968 flow_setup_cb_t *cb = airoha_dev_setup_tc_block_cb;
2969 static LIST_HEAD(block_cb_list);
2970 struct flow_block_cb *block_cb;
2971
2972 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
2973 return -EOPNOTSUPP;
2974
2975 f->driver_block_list = &block_cb_list;
2976 switch (f->command) {
2977 case FLOW_BLOCK_BIND:
2978 block_cb = flow_block_cb_lookup(f->block, cb, dev);
2979 if (block_cb) {
2980 flow_block_cb_incref(block_cb);
2981 return 0;
2982 }
2983 block_cb = flow_block_cb_alloc(cb, dev, dev, NULL);
2984 if (IS_ERR(block_cb))
2985 return PTR_ERR(block_cb);
2986
2987 flow_block_cb_incref(block_cb);
2988 flow_block_cb_add(block_cb, f);
2989 list_add_tail(&block_cb->driver_list, &block_cb_list);
2990 return 0;
2991 case FLOW_BLOCK_UNBIND:
2992 block_cb = flow_block_cb_lookup(f->block, cb, dev);
2993 if (!block_cb)
2994 return -ENOENT;
2995
2996 if (!flow_block_cb_decref(block_cb)) {
2997 flow_block_cb_remove(block_cb, f);
2998 list_del(&block_cb->driver_list);
2999 }
3000 return 0;
3001 default:
3002 return -EOPNOTSUPP;
3003 }
3004 }
3005
airoha_tc_remove_htb_queue(struct net_device * netdev,int queue)3006 static void airoha_tc_remove_htb_queue(struct net_device *netdev, int queue)
3007 {
3008 struct airoha_gdm_dev *dev = netdev_priv(netdev);
3009 int num_tx_queues = AIROHA_NUM_TX_RING;
3010 struct airoha_qdma *qdma = dev->qdma;
3011
3012 airoha_qdma_set_tx_rate_limit(netdev, queue, 0, 0);
3013
3014 clear_bit(queue, qdma->qos_channel_map);
3015 clear_bit(queue, dev->qos_sq_bmap);
3016
3017 if (!bitmap_empty(dev->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS))
3018 num_tx_queues += find_last_bit(dev->qos_sq_bmap,
3019 AIROHA_NUM_QOS_CHANNELS) + 1;
3020 netif_set_real_num_tx_queues(netdev, num_tx_queues);
3021 }
3022
airoha_tc_htb_delete_leaf_queue(struct net_device * netdev,struct tc_htb_qopt_offload * opt)3023 static int airoha_tc_htb_delete_leaf_queue(struct net_device *netdev,
3024 struct tc_htb_qopt_offload *opt)
3025 {
3026 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS;
3027 struct airoha_gdm_dev *dev = netdev_priv(netdev);
3028
3029 if (!test_bit(channel, dev->qos_sq_bmap)) {
3030 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id");
3031 return -EINVAL;
3032 }
3033
3034 airoha_tc_remove_htb_queue(netdev, channel);
3035
3036 return 0;
3037 }
3038
airoha_tc_htb_destroy(struct net_device * netdev)3039 static int airoha_tc_htb_destroy(struct net_device *netdev)
3040 {
3041 struct airoha_gdm_dev *dev = netdev_priv(netdev);
3042 int q;
3043
3044 for_each_set_bit(q, dev->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS)
3045 airoha_tc_remove_htb_queue(netdev, q);
3046
3047 return 0;
3048 }
3049
airoha_tc_get_htb_get_leaf_queue(struct net_device * netdev,struct tc_htb_qopt_offload * opt)3050 static int airoha_tc_get_htb_get_leaf_queue(struct net_device *netdev,
3051 struct tc_htb_qopt_offload *opt)
3052 {
3053 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS;
3054 struct airoha_gdm_dev *dev = netdev_priv(netdev);
3055
3056 if (!test_bit(channel, dev->qos_sq_bmap)) {
3057 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id");
3058 return -EINVAL;
3059 }
3060
3061 opt->qid = AIROHA_NUM_TX_RING + channel;
3062
3063 return 0;
3064 }
3065
airoha_tc_setup_qdisc_htb(struct net_device * dev,struct tc_htb_qopt_offload * opt)3066 static int airoha_tc_setup_qdisc_htb(struct net_device *dev,
3067 struct tc_htb_qopt_offload *opt)
3068 {
3069 switch (opt->command) {
3070 case TC_HTB_CREATE:
3071 break;
3072 case TC_HTB_DESTROY:
3073 return airoha_tc_htb_destroy(dev);
3074 case TC_HTB_NODE_MODIFY:
3075 return airoha_tc_htb_modify_queue(dev, opt);
3076 case TC_HTB_LEAF_ALLOC_QUEUE:
3077 return airoha_tc_htb_alloc_leaf_queue(dev, opt);
3078 case TC_HTB_LEAF_DEL:
3079 case TC_HTB_LEAF_DEL_LAST:
3080 case TC_HTB_LEAF_DEL_LAST_FORCE:
3081 return airoha_tc_htb_delete_leaf_queue(dev, opt);
3082 case TC_HTB_LEAF_QUERY_QUEUE:
3083 return airoha_tc_get_htb_get_leaf_queue(dev, opt);
3084 default:
3085 return -EOPNOTSUPP;
3086 }
3087
3088 return 0;
3089 }
3090
airoha_dev_tc_setup(struct net_device * dev,enum tc_setup_type type,void * type_data)3091 static int airoha_dev_tc_setup(struct net_device *dev,
3092 enum tc_setup_type type, void *type_data)
3093 {
3094 switch (type) {
3095 case TC_SETUP_QDISC_ETS:
3096 return airoha_tc_setup_qdisc_ets(dev, type_data);
3097 case TC_SETUP_QDISC_HTB:
3098 return airoha_tc_setup_qdisc_htb(dev, type_data);
3099 case TC_SETUP_BLOCK:
3100 case TC_SETUP_FT:
3101 return airoha_dev_setup_tc_block(dev, type_data);
3102 default:
3103 return -EOPNOTSUPP;
3104 }
3105 }
3106
3107 static const struct net_device_ops airoha_netdev_ops = {
3108 .ndo_init = airoha_dev_init,
3109 .ndo_open = airoha_dev_open,
3110 .ndo_stop = airoha_dev_stop,
3111 .ndo_change_mtu = airoha_dev_change_mtu,
3112 .ndo_select_queue = airoha_dev_select_queue,
3113 .ndo_start_xmit = airoha_dev_xmit,
3114 .ndo_get_stats64 = airoha_dev_get_stats64,
3115 .ndo_set_mac_address = airoha_dev_set_macaddr,
3116 .ndo_setup_tc = airoha_dev_tc_setup,
3117 };
3118
3119 static const struct ethtool_ops airoha_ethtool_ops = {
3120 .get_drvinfo = airoha_ethtool_get_drvinfo,
3121 .get_eth_mac_stats = airoha_ethtool_get_mac_stats,
3122 .get_rmon_stats = airoha_ethtool_get_rmon_stats,
3123 .get_link_ksettings = phy_ethtool_get_link_ksettings,
3124 .get_link = ethtool_op_get_link,
3125 };
3126
airoha_metadata_dst_alloc(struct airoha_gdm_port * port)3127 static int airoha_metadata_dst_alloc(struct airoha_gdm_port *port)
3128 {
3129 int i;
3130
3131 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) {
3132 struct metadata_dst *md_dst;
3133
3134 md_dst = metadata_dst_alloc(0, METADATA_HW_PORT_MUX,
3135 GFP_KERNEL);
3136 if (!md_dst)
3137 return -ENOMEM;
3138
3139 md_dst->u.port_info.port_id = i;
3140 port->dsa_meta[i] = md_dst;
3141 }
3142
3143 return 0;
3144 }
3145
airoha_metadata_dst_free(struct airoha_gdm_port * port)3146 static void airoha_metadata_dst_free(struct airoha_gdm_port *port)
3147 {
3148 int i;
3149
3150 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) {
3151 if (!port->dsa_meta[i])
3152 continue;
3153
3154 dst_release(&port->dsa_meta[i]->dst);
3155 }
3156 }
3157
airoha_is_valid_gdm_dev(struct airoha_eth * eth,struct airoha_gdm_dev * dev)3158 bool airoha_is_valid_gdm_dev(struct airoha_eth *eth,
3159 struct airoha_gdm_dev *dev)
3160 {
3161 int i;
3162
3163 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
3164 struct airoha_gdm_port *port = eth->ports[i];
3165 int j;
3166
3167 if (!port)
3168 continue;
3169
3170 for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
3171 if (port->devs[j] == dev)
3172 return true;
3173 }
3174 }
3175
3176 return false;
3177 }
3178
airoha_alloc_gdm_device(struct airoha_eth * eth,struct airoha_gdm_port * port,int nbq,struct device_node * np)3179 static int airoha_alloc_gdm_device(struct airoha_eth *eth,
3180 struct airoha_gdm_port *port,
3181 int nbq, struct device_node *np)
3182 {
3183 struct net_device *netdev;
3184 struct airoha_gdm_dev *dev;
3185 u8 index;
3186 int err;
3187
3188 netdev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*dev),
3189 AIROHA_NUM_NETDEV_TX_RINGS,
3190 AIROHA_NUM_RX_RING);
3191 if (!netdev) {
3192 dev_err(eth->dev, "alloc_etherdev failed\n");
3193 return -ENOMEM;
3194 }
3195
3196 netdev->netdev_ops = &airoha_netdev_ops;
3197 netdev->ethtool_ops = &airoha_ethtool_ops;
3198 netdev->max_mtu = AIROHA_MAX_MTU;
3199 netdev->watchdog_timeo = 5 * HZ;
3200 netdev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_TSO6 |
3201 NETIF_F_IPV6_CSUM | NETIF_F_SG | NETIF_F_TSO |
3202 NETIF_F_HW_TC;
3203 netdev->features |= netdev->hw_features;
3204 netdev->vlan_features = netdev->hw_features;
3205 SET_NETDEV_DEV(netdev, eth->dev);
3206
3207 /* reserve hw queues for HTB offloading */
3208 err = netif_set_real_num_tx_queues(netdev, AIROHA_NUM_TX_RING);
3209 if (err)
3210 return err;
3211
3212 err = of_get_ethdev_address(np, netdev);
3213 if (err) {
3214 if (err == -EPROBE_DEFER)
3215 return err;
3216
3217 eth_hw_addr_random(netdev);
3218 dev_info(eth->dev, "generated random MAC address %pM\n",
3219 netdev->dev_addr);
3220 }
3221
3222 /* Allowed nbq for EN7581 on GDM3 port are 4 and 5 for PCIE0
3223 * and PCIE1 respectively.
3224 */
3225 index = nbq;
3226 if (index && airoha_is_7581(eth) && port->id == AIROHA_GDM3_IDX)
3227 index -= 4;
3228
3229 if (index >= ARRAY_SIZE(port->devs) || port->devs[index]) {
3230 dev_err(eth->dev, "invalid nbq id: %d\n", nbq);
3231 return -EINVAL;
3232 }
3233
3234 netdev->dev.of_node = of_node_get(np);
3235 dev = netdev_priv(netdev);
3236 u64_stats_init(&dev->stats.syncp);
3237 dev->port = port;
3238 dev->eth = eth;
3239 dev->nbq = nbq;
3240 port->devs[index] = dev;
3241
3242 return 0;
3243 }
3244
airoha_alloc_gdm_port(struct airoha_eth * eth,struct device_node * np)3245 static int airoha_alloc_gdm_port(struct airoha_eth *eth,
3246 struct device_node *np)
3247 {
3248 const __be32 *id_ptr = of_get_property(np, "reg", NULL);
3249 struct airoha_gdm_port *port;
3250 struct device_node *node;
3251 int err, nbq, p, d = 0;
3252 u32 id;
3253
3254 if (!id_ptr) {
3255 dev_err(eth->dev, "missing gdm port id\n");
3256 return -EINVAL;
3257 }
3258
3259 id = be32_to_cpup(id_ptr);
3260 p = id - 1;
3261
3262 if (!id || id > ARRAY_SIZE(eth->ports)) {
3263 dev_err(eth->dev, "invalid gdm port id: %d\n", id);
3264 return -EINVAL;
3265 }
3266
3267 if (eth->ports[p]) {
3268 dev_err(eth->dev, "duplicate gdm port id: %d\n", id);
3269 return -EINVAL;
3270 }
3271
3272 port = devm_kzalloc(eth->dev, sizeof(*port), GFP_KERNEL);
3273 if (!port)
3274 return -ENOMEM;
3275
3276 port->id = id;
3277 spin_lock_init(&port->stats_lock);
3278 eth->ports[p] = port;
3279
3280 err = airoha_metadata_dst_alloc(port);
3281 if (err)
3282 return err;
3283
3284 /* Default nbq value to ensure backward compatibility */
3285 nbq = id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0;
3286
3287 for_each_child_of_node(np, node) {
3288 /* Multiple external serdes connected to the FE GDM port via an
3289 * external arbiter.
3290 */
3291 const __be32 *nbq_ptr;
3292
3293 if (!of_device_is_compatible(node, "airoha,eth-port"))
3294 continue;
3295
3296 d++;
3297 if (!of_device_is_available(node))
3298 continue;
3299
3300 nbq_ptr = of_get_property(node, "reg", NULL);
3301 if (!nbq_ptr) {
3302 dev_err(eth->dev, "missing nbq id\n");
3303 of_node_put(node);
3304 return -EINVAL;
3305 }
3306
3307 /* Verify the provided nbq parameter is valid */
3308 nbq = be32_to_cpup(nbq_ptr);
3309 err = eth->soc->ops.get_sport(port, nbq);
3310 if (err < 0) {
3311 of_node_put(node);
3312 return err;
3313 }
3314
3315 err = airoha_alloc_gdm_device(eth, port, nbq, node);
3316 if (err) {
3317 of_node_put(node);
3318 return err;
3319 }
3320 }
3321
3322 return !d ? airoha_alloc_gdm_device(eth, port, nbq, np) : 0;
3323 }
3324
airoha_register_gdm_devices(struct airoha_eth * eth)3325 static int airoha_register_gdm_devices(struct airoha_eth *eth)
3326 {
3327 int i;
3328
3329 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
3330 struct airoha_gdm_port *port = eth->ports[i];
3331 int j;
3332
3333 if (!port)
3334 continue;
3335
3336 for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
3337 struct airoha_gdm_dev *dev = port->devs[j];
3338 int err;
3339
3340 if (!dev)
3341 continue;
3342
3343 err = register_netdev(netdev_from_priv(dev));
3344 if (err)
3345 return err;
3346 }
3347 }
3348
3349 set_bit(DEV_STATE_REGISTERED, ð->state);
3350
3351 return 0;
3352 }
3353
airoha_probe(struct platform_device * pdev)3354 static int airoha_probe(struct platform_device *pdev)
3355 {
3356 struct reset_control_bulk_data *xsi_rsts;
3357 struct device_node *np;
3358 struct airoha_eth *eth;
3359 int i, err;
3360
3361 eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL);
3362 if (!eth)
3363 return -ENOMEM;
3364
3365 eth->soc = of_device_get_match_data(&pdev->dev);
3366 if (!eth->soc)
3367 return -EINVAL;
3368
3369 eth->dev = &pdev->dev;
3370
3371 err = dma_set_mask_and_coherent(eth->dev, DMA_BIT_MASK(32));
3372 if (err) {
3373 dev_err(eth->dev, "failed configuring DMA mask\n");
3374 return err;
3375 }
3376
3377 eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe");
3378 if (IS_ERR(eth->fe_regs))
3379 return dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs),
3380 "failed to iomap fe regs\n");
3381
3382 eth->rsts[0].id = "fe";
3383 eth->rsts[1].id = "pdma";
3384 eth->rsts[2].id = "qdma";
3385 err = devm_reset_control_bulk_get_exclusive(eth->dev,
3386 ARRAY_SIZE(eth->rsts),
3387 eth->rsts);
3388 if (err) {
3389 dev_err(eth->dev, "failed to get bulk reset lines\n");
3390 return err;
3391 }
3392
3393 xsi_rsts = devm_kcalloc(eth->dev,
3394 eth->soc->num_xsi_rsts, sizeof(*xsi_rsts),
3395 GFP_KERNEL);
3396 if (!xsi_rsts)
3397 return -ENOMEM;
3398
3399 eth->xsi_rsts = xsi_rsts;
3400 for (i = 0; i < eth->soc->num_xsi_rsts; i++)
3401 eth->xsi_rsts[i].id = eth->soc->xsi_rsts_names[i];
3402
3403 err = devm_reset_control_bulk_get_exclusive(eth->dev,
3404 eth->soc->num_xsi_rsts,
3405 eth->xsi_rsts);
3406 if (err) {
3407 dev_err(eth->dev, "failed to get bulk xsi reset lines\n");
3408 return err;
3409 }
3410
3411 eth->napi_dev = alloc_netdev_dummy(0);
3412 if (!eth->napi_dev)
3413 return -ENOMEM;
3414
3415 /* Enable threaded NAPI by default */
3416 eth->napi_dev->threaded = true;
3417 strscpy(eth->napi_dev->name, "qdma_eth", sizeof(eth->napi_dev->name));
3418 platform_set_drvdata(pdev, eth);
3419
3420 err = airoha_hw_init(pdev, eth);
3421 if (err)
3422 goto error_netdev_free;
3423
3424 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
3425 airoha_qdma_start_napi(ð->qdma[i]);
3426 airoha_qdma_set(ð->qdma[i], REG_QDMA_GLOBAL_CFG,
3427 GLOBAL_CFG_TX_DMA_EN_MASK |
3428 GLOBAL_CFG_RX_DMA_EN_MASK);
3429 }
3430
3431 for_each_child_of_node(pdev->dev.of_node, np) {
3432 if (!of_device_is_compatible(np, "airoha,eth-mac"))
3433 continue;
3434
3435 if (!of_device_is_available(np))
3436 continue;
3437
3438 err = airoha_alloc_gdm_port(eth, np);
3439 if (err) {
3440 of_node_put(np);
3441 goto error_napi_stop;
3442 }
3443 }
3444
3445 err = airoha_register_gdm_devices(eth);
3446 if (err)
3447 goto error_napi_stop;
3448
3449 return 0;
3450
3451 error_napi_stop:
3452 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
3453 airoha_qdma_stop_napi(ð->qdma[i]);
3454 airoha_qdma_tx_cleanup(ð->qdma[i]);
3455 }
3456
3457 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
3458 struct airoha_gdm_port *port = eth->ports[i];
3459 int j;
3460
3461 if (!port)
3462 continue;
3463
3464 for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
3465 struct airoha_gdm_dev *dev = port->devs[j];
3466 struct net_device *netdev;
3467
3468 if (!dev)
3469 continue;
3470
3471 netdev = netdev_from_priv(dev);
3472 if (netdev->reg_state == NETREG_REGISTERED)
3473 unregister_netdev(netdev);
3474 of_node_put(netdev->dev.of_node);
3475 }
3476 airoha_metadata_dst_free(port);
3477 }
3478 airoha_hw_cleanup(eth);
3479 error_netdev_free:
3480 free_netdev(eth->napi_dev);
3481 platform_set_drvdata(pdev, NULL);
3482
3483 return err;
3484 }
3485
airoha_remove(struct platform_device * pdev)3486 static void airoha_remove(struct platform_device *pdev)
3487 {
3488 struct airoha_eth *eth = platform_get_drvdata(pdev);
3489 int i;
3490
3491 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
3492 airoha_qdma_stop_napi(ð->qdma[i]);
3493 airoha_qdma_tx_cleanup(ð->qdma[i]);
3494 }
3495
3496 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
3497 struct airoha_gdm_port *port = eth->ports[i];
3498 int j;
3499
3500 if (!port)
3501 continue;
3502
3503 for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
3504 struct airoha_gdm_dev *dev = port->devs[j];
3505 struct net_device *netdev;
3506
3507 if (!dev)
3508 continue;
3509
3510 netdev = netdev_from_priv(dev);
3511 unregister_netdev(netdev);
3512 of_node_put(netdev->dev.of_node);
3513 }
3514 airoha_metadata_dst_free(port);
3515 }
3516 airoha_hw_cleanup(eth);
3517
3518 free_netdev(eth->napi_dev);
3519 platform_set_drvdata(pdev, NULL);
3520 }
3521
3522 static const char * const en7581_xsi_rsts_names[] = {
3523 "xsi-mac",
3524 "hsi0-mac",
3525 "hsi1-mac",
3526 "hsi-mac",
3527 "xfp-mac",
3528 };
3529
airoha_en7581_get_sport(struct airoha_gdm_port * port,int nbq)3530 static int airoha_en7581_get_sport(struct airoha_gdm_port *port, int nbq)
3531 {
3532 switch (port->id) {
3533 case AIROHA_GDM3_IDX:
3534 /* 7581 SoC supports PCIe serdes on GDM3 port */
3535 if (nbq == 4)
3536 return HSGMII_LAN_7581_PCIE0_SRCPORT;
3537 if (nbq == 5)
3538 return HSGMII_LAN_7581_PCIE1_SRCPORT;
3539 break;
3540 case AIROHA_GDM4_IDX:
3541 /* 7581 SoC supports eth and usb serdes on GDM4 port */
3542 if (!nbq)
3543 return HSGMII_LAN_7581_ETH_SRCPORT;
3544 if (nbq == 1)
3545 return HSGMII_LAN_7581_USB_SRCPORT;
3546 break;
3547 default:
3548 break;
3549 }
3550
3551 return -EINVAL;
3552 }
3553
airoha_en7581_get_vip_port(struct airoha_gdm_port * port,int nbq)3554 static u32 airoha_en7581_get_vip_port(struct airoha_gdm_port *port, int nbq)
3555 {
3556 switch (port->id) {
3557 case AIROHA_GDM3_IDX:
3558 if (nbq == 4)
3559 return XSI_PCIE0_VIP_PORT_MASK;
3560 if (nbq == 5)
3561 return XSI_PCIE1_VIP_PORT_MASK;
3562 break;
3563 case AIROHA_GDM4_IDX:
3564 if (!nbq)
3565 return XSI_ETH_VIP_PORT_MASK;
3566 if (nbq == 1)
3567 return XSI_USB_VIP_PORT_MASK;
3568 break;
3569 default:
3570 break;
3571 }
3572
3573 return 0;
3574 }
3575
airoha_en7581_get_dev_from_sport(struct airoha_qdma_desc * desc,u16 * port,u16 * dev)3576 static int airoha_en7581_get_dev_from_sport(struct airoha_qdma_desc *desc,
3577 u16 *port, u16 *dev)
3578 {
3579 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK,
3580 le32_to_cpu(READ_ONCE(desc->msg1)));
3581
3582 *dev = 0;
3583 switch (sport) {
3584 case 0x10 ... 0x14:
3585 *port = 0; /* GDM1 */
3586 break;
3587 case 0x2 ... 0x4:
3588 *port = sport - 1;
3589 break;
3590 case HSGMII_LAN_7581_PCIE1_SRCPORT:
3591 *dev = 1;
3592 fallthrough;
3593 case HSGMII_LAN_7581_PCIE0_SRCPORT:
3594 *port = 2; /* GDM3 */
3595 break;
3596 case HSGMII_LAN_7581_USB_SRCPORT:
3597 *dev = 1;
3598 fallthrough;
3599 case HSGMII_LAN_7581_ETH_SRCPORT:
3600 *port = 3; /* GDM4 */
3601 break;
3602 default:
3603 return -EINVAL;
3604 }
3605
3606 return 0;
3607 }
3608
3609 static const char * const an7583_xsi_rsts_names[] = {
3610 "xsi-mac",
3611 "hsi0-mac",
3612 "hsi1-mac",
3613 "xfp-mac",
3614 };
3615
airoha_an7583_get_sport(struct airoha_gdm_port * port,int nbq)3616 static int airoha_an7583_get_sport(struct airoha_gdm_port *port, int nbq)
3617 {
3618 switch (port->id) {
3619 case AIROHA_GDM3_IDX:
3620 /* 7583 SoC supports eth serdes on GDM3 port */
3621 if (!nbq)
3622 return HSGMII_LAN_7583_ETH_SRCPORT;
3623 break;
3624 case AIROHA_GDM4_IDX:
3625 /* 7583 SoC supports PCIe and USB serdes on GDM4 port */
3626 if (!nbq)
3627 return HSGMII_LAN_7583_PCIE_SRCPORT;
3628 if (nbq == 1)
3629 return HSGMII_LAN_7583_USB_SRCPORT;
3630 break;
3631 default:
3632 break;
3633 }
3634
3635 return -EINVAL;
3636 }
3637
airoha_an7583_get_vip_port(struct airoha_gdm_port * port,int nbq)3638 static u32 airoha_an7583_get_vip_port(struct airoha_gdm_port *port, int nbq)
3639 {
3640 switch (port->id) {
3641 case AIROHA_GDM3_IDX:
3642 if (!nbq)
3643 return XSI_ETH_VIP_PORT_MASK;
3644 break;
3645 case AIROHA_GDM4_IDX:
3646 if (!nbq)
3647 return XSI_PCIE0_VIP_PORT_MASK;
3648 if (nbq == 1)
3649 return XSI_USB_VIP_PORT_MASK;
3650 break;
3651 default:
3652 break;
3653 }
3654
3655 return 0;
3656 }
3657
airoha_an7583_get_dev_from_sport(struct airoha_qdma_desc * desc,u16 * port,u16 * dev)3658 static int airoha_an7583_get_dev_from_sport(struct airoha_qdma_desc *desc,
3659 u16 *port, u16 *dev)
3660 {
3661 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK,
3662 le32_to_cpu(READ_ONCE(desc->msg1)));
3663
3664 *dev = 0;
3665 switch (sport) {
3666 case 0x10 ... 0x14:
3667 *port = 0; /* GDM1 */
3668 break;
3669 case 0x2 ... 0x4:
3670 *port = sport - 1;
3671 break;
3672 case HSGMII_LAN_7583_ETH_SRCPORT:
3673 *port = 2; /* GDM3 */
3674 break;
3675 case HSGMII_LAN_7583_USB_SRCPORT:
3676 *dev = 1;
3677 fallthrough;
3678 case HSGMII_LAN_7583_PCIE_SRCPORT:
3679 *port = 3; /* GDM4 */
3680 break;
3681 default:
3682 return -EINVAL;
3683 }
3684
3685 return 0;
3686 }
3687
3688 static const struct airoha_eth_soc_data en7581_soc_data = {
3689 .version = 0x7581,
3690 .xsi_rsts_names = en7581_xsi_rsts_names,
3691 .num_xsi_rsts = ARRAY_SIZE(en7581_xsi_rsts_names),
3692 .num_ppe = 2,
3693 .ops = {
3694 .get_sport = airoha_en7581_get_sport,
3695 .get_vip_port = airoha_en7581_get_vip_port,
3696 .get_dev_from_sport = airoha_en7581_get_dev_from_sport,
3697 },
3698 };
3699
3700 static const struct airoha_eth_soc_data an7583_soc_data = {
3701 .version = 0x7583,
3702 .xsi_rsts_names = an7583_xsi_rsts_names,
3703 .num_xsi_rsts = ARRAY_SIZE(an7583_xsi_rsts_names),
3704 .num_ppe = 1,
3705 .ops = {
3706 .get_sport = airoha_an7583_get_sport,
3707 .get_vip_port = airoha_an7583_get_vip_port,
3708 .get_dev_from_sport = airoha_an7583_get_dev_from_sport,
3709 },
3710 };
3711
3712 static const struct of_device_id of_airoha_match[] = {
3713 { .compatible = "airoha,en7581-eth", .data = &en7581_soc_data },
3714 { .compatible = "airoha,an7583-eth", .data = &an7583_soc_data },
3715 { /* sentinel */ }
3716 };
3717 MODULE_DEVICE_TABLE(of, of_airoha_match);
3718
3719 static struct platform_driver airoha_driver = {
3720 .probe = airoha_probe,
3721 .remove = airoha_remove,
3722 .driver = {
3723 .name = KBUILD_MODNAME,
3724 .of_match_table = of_airoha_match,
3725 },
3726 };
3727 module_platform_driver(airoha_driver);
3728
3729 MODULE_LICENSE("GPL v2");
3730 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>");
3731 MODULE_DESCRIPTION("Ethernet driver for Airoha SoC");
3732