1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2024 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 #include <linux/of.h> 7 #include <linux/of_net.h> 8 #include <linux/of_reserved_mem.h> 9 #include <linux/platform_device.h> 10 #include <linux/tcp.h> 11 #include <linux/u64_stats_sync.h> 12 #include <net/dst_metadata.h> 13 #include <net/page_pool/helpers.h> 14 #include <net/pkt_cls.h> 15 #include <uapi/linux/ppp_defs.h> 16 17 #include "airoha_regs.h" 18 #include "airoha_eth.h" 19 20 u32 airoha_rr(void __iomem *base, u32 offset) 21 { 22 return readl(base + offset); 23 } 24 25 void airoha_wr(void __iomem *base, u32 offset, u32 val) 26 { 27 writel(val, base + offset); 28 } 29 30 u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val) 31 { 32 val |= (airoha_rr(base, offset) & ~mask); 33 airoha_wr(base, offset, val); 34 35 return val; 36 } 37 38 static void airoha_qdma_set_irqmask(struct airoha_irq_bank *irq_bank, 39 int index, u32 clear, u32 set) 40 { 41 struct airoha_qdma *qdma = irq_bank->qdma; 42 int bank = irq_bank - &qdma->irq_banks[0]; 43 unsigned long flags; 44 45 if (WARN_ON_ONCE(index >= ARRAY_SIZE(irq_bank->irqmask))) 46 return; 47 48 spin_lock_irqsave(&irq_bank->irq_lock, flags); 49 50 irq_bank->irqmask[index] &= ~clear; 51 irq_bank->irqmask[index] |= set; 52 airoha_qdma_wr(qdma, REG_INT_ENABLE(bank, index), 53 irq_bank->irqmask[index]); 54 /* Read irq_enable register in order to guarantee the update above 55 * completes in the spinlock critical section. 56 */ 57 airoha_qdma_rr(qdma, REG_INT_ENABLE(bank, index)); 58 59 spin_unlock_irqrestore(&irq_bank->irq_lock, flags); 60 } 61 62 static void airoha_qdma_irq_enable(struct airoha_irq_bank *irq_bank, 63 int index, u32 mask) 64 { 65 airoha_qdma_set_irqmask(irq_bank, index, 0, mask); 66 } 67 68 static void airoha_qdma_irq_disable(struct airoha_irq_bank *irq_bank, 69 int index, u32 mask) 70 { 71 airoha_qdma_set_irqmask(irq_bank, index, mask, 0); 72 } 73 74 static int airoha_set_macaddr(struct airoha_gdm_dev *dev, const u8 *addr) 75 { 76 u8 ref_addr[ETH_ALEN] __aligned(2); 77 struct airoha_eth *eth = dev->eth; 78 u32 reg, val, lmin, lmax; 79 int i; 80 81 eth_zero_addr(ref_addr); 82 lmin = (addr[3] << 16) | (addr[4] << 8) | addr[5]; 83 lmax = lmin; 84 85 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 86 struct airoha_gdm_port *port = eth->ports[i]; 87 int j; 88 89 if (!port) 90 continue; 91 92 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 93 struct airoha_gdm_dev *iter_dev; 94 struct net_device *netdev; 95 96 iter_dev = port->devs[j]; 97 if (!iter_dev || iter_dev == dev) 98 continue; 99 100 if (airoha_is_lan_gdm_dev(iter_dev) != 101 airoha_is_lan_gdm_dev(dev)) 102 continue; 103 104 netdev = netdev_from_priv(iter_dev); 105 if (netdev->reg_state != NETREG_REGISTERED) 106 continue; 107 108 ether_addr_copy(ref_addr, netdev->dev_addr); 109 val = (netdev->dev_addr[3] << 16) | 110 (netdev->dev_addr[4] << 8) | netdev->dev_addr[5]; 111 if (val < lmin) 112 lmin = val; 113 if (val > lmax) 114 lmax = val; 115 } 116 } 117 118 if (!is_zero_ether_addr(ref_addr) && memcmp(ref_addr, addr, 3)) { 119 /* According to the HW design, hw mac address MSBs must be 120 * the same for each net_device with the same LAN/WAN 121 * configuration. 122 */ 123 struct net_device *netdev = netdev_from_priv(dev); 124 125 dev_warn(eth->dev, 126 "%s: wrong mac addr, MSBs must be %02x:%02x:%02x\n", 127 netdev->name, ref_addr[0], ref_addr[1], 128 ref_addr[2]); 129 dev_warn(eth->dev, "FE hw forwarding won't work properly\n"); 130 131 return -EINVAL; 132 } 133 134 reg = airoha_is_lan_gdm_dev(dev) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H; 135 val = (addr[0] << 16) | (addr[1] << 8) | addr[2]; 136 airoha_fe_wr(eth, reg, val); 137 138 airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), lmin); 139 airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), lmax); 140 141 airoha_ppe_init_upd_mem(dev, addr); 142 143 return 0; 144 } 145 146 static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr, 147 u32 val) 148 { 149 airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK, 150 FIELD_PREP(GDM_OCFQ_MASK, val)); 151 airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK, 152 FIELD_PREP(GDM_MCFQ_MASK, val)); 153 airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK, 154 FIELD_PREP(GDM_BCFQ_MASK, val)); 155 airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK, 156 FIELD_PREP(GDM_UCFQ_MASK, val)); 157 } 158 159 static int airoha_set_vip_for_gdm_port(struct airoha_gdm_dev *dev, bool enable) 160 { 161 struct airoha_gdm_port *port = dev->port; 162 struct airoha_eth *eth = dev->eth; 163 u32 vip_port; 164 165 vip_port = eth->soc->ops.get_vip_port(port, dev->nbq); 166 if (enable) { 167 airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port); 168 airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port); 169 } else { 170 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port); 171 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port); 172 } 173 174 return 0; 175 } 176 177 static void airoha_fe_maccr_init(struct airoha_eth *eth) 178 { 179 int p; 180 181 for (p = 1; p <= ARRAY_SIZE(eth->ports); p++) { 182 airoha_fe_set(eth, REG_GDM_FWD_CFG(p), 183 GDM_TCP_CKSUM_MASK | GDM_UDP_CKSUM_MASK | 184 GDM_IP4_CKSUM_MASK | GDM_DROP_CRC_ERR_MASK); 185 airoha_fe_rmw(eth, REG_GDM_LEN_CFG(p), 186 GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK, 187 FIELD_PREP(GDM_SHORT_LEN_MASK, 60) | 188 FIELD_PREP(GDM_LONG_LEN_MASK, AIROHA_MAX_RX_SIZE)); 189 } 190 191 airoha_fe_rmw(eth, REG_CDM_VLAN_CTRL(1), CDM_VLAN_MASK, 192 FIELD_PREP(CDM_VLAN_MASK, 0x8100)); 193 194 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD); 195 } 196 197 static void airoha_fe_vip_setup(struct airoha_eth *eth) 198 { 199 airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC); 200 airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK); 201 202 airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP); 203 airoha_fe_wr(eth, REG_FE_VIP_EN(4), 204 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 205 PATN_EN_MASK); 206 207 airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP); 208 airoha_fe_wr(eth, REG_FE_VIP_EN(6), 209 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 210 PATN_EN_MASK); 211 212 airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP); 213 airoha_fe_wr(eth, REG_FE_VIP_EN(7), 214 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 215 PATN_EN_MASK); 216 217 /* BOOTP (0x43) */ 218 airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43); 219 airoha_fe_wr(eth, REG_FE_VIP_EN(8), 220 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK | 221 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 222 223 /* BOOTP (0x44) */ 224 airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44); 225 airoha_fe_wr(eth, REG_FE_VIP_EN(9), 226 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK | 227 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 228 229 /* ISAKMP */ 230 airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4); 231 airoha_fe_wr(eth, REG_FE_VIP_EN(10), 232 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK | 233 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 234 235 airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP); 236 airoha_fe_wr(eth, REG_FE_VIP_EN(11), 237 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 238 PATN_EN_MASK); 239 240 /* DHCPv6 */ 241 airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223); 242 airoha_fe_wr(eth, REG_FE_VIP_EN(12), 243 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK | 244 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 245 246 airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP); 247 airoha_fe_wr(eth, REG_FE_VIP_EN(19), 248 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 249 PATN_EN_MASK); 250 251 /* ETH->ETH_P_1905 (0x893a) */ 252 airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a); 253 airoha_fe_wr(eth, REG_FE_VIP_EN(20), 254 PATN_FCPU_EN_MASK | PATN_EN_MASK); 255 256 airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP); 257 airoha_fe_wr(eth, REG_FE_VIP_EN(21), 258 PATN_FCPU_EN_MASK | PATN_EN_MASK); 259 } 260 261 static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth, 262 u32 port, u32 queue) 263 { 264 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR, 265 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK, 266 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) | 267 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue)); 268 269 return airoha_fe_get(eth, REG_FE_PSE_QUEUE_CFG_VAL, 270 PSE_CFG_OQ_RSV_MASK); 271 } 272 273 static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth, 274 u32 port, u32 queue, u32 val) 275 { 276 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK, 277 FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val)); 278 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR, 279 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK | 280 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK, 281 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) | 282 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) | 283 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK); 284 } 285 286 static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth) 287 { 288 return airoha_fe_get(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK); 289 } 290 291 static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth, 292 u32 port, u32 queue, u32 val) 293 { 294 u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue); 295 u32 tmp, all_rsv, fq_limit; 296 297 airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val); 298 299 /* modify all rsv */ 300 all_rsv = airoha_fe_get_pse_all_rsv(eth); 301 all_rsv += (val - orig_val); 302 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, 303 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); 304 305 /* modify hthd */ 306 fq_limit = airoha_fe_get(eth, PSE_FQ_CFG, PSE_FQ_LIMIT_MASK); 307 tmp = fq_limit - all_rsv - 0x20; 308 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD, 309 PSE_SHARE_USED_HTHD_MASK, 310 FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp)); 311 312 tmp = fq_limit - all_rsv - 0x100; 313 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD, 314 PSE_SHARE_USED_MTHD_MASK, 315 FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp)); 316 tmp = (3 * tmp) >> 2; 317 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, 318 PSE_SHARE_USED_LTHD_MASK, 319 FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp)); 320 321 return 0; 322 } 323 324 static void airoha_fe_pse_ports_init(struct airoha_eth *eth) 325 { 326 const u32 pse_port_num_queues[] = { 327 [FE_PSE_PORT_CDM1] = 6, 328 [FE_PSE_PORT_GDM1] = 6, 329 [FE_PSE_PORT_GDM2] = 32, 330 [FE_PSE_PORT_GDM3] = 6, 331 [FE_PSE_PORT_PPE1] = 4, 332 [FE_PSE_PORT_CDM2] = 6, 333 [FE_PSE_PORT_CDM3] = 8, 334 [FE_PSE_PORT_CDM4] = 10, 335 [FE_PSE_PORT_PPE2] = 4, 336 [FE_PSE_PORT_GDM4] = 2, 337 [FE_PSE_PORT_CDM5] = 2, 338 }; 339 int q; 340 341 if (airoha_ppe_is_enabled(eth, 1)) { 342 u32 all_rsv; 343 344 /* hw misses PPE2 oq rsv */ 345 all_rsv = airoha_fe_get_pse_all_rsv(eth); 346 all_rsv += PSE_RSV_PAGES * 347 pse_port_num_queues[FE_PSE_PORT_PPE2]; 348 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, 349 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); 350 } 351 352 /* CDM1 */ 353 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++) 354 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q, 355 PSE_QUEUE_RSV_PAGES); 356 /* GDM1 */ 357 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++) 358 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q, 359 PSE_QUEUE_RSV_PAGES); 360 /* GDM2 */ 361 for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++) 362 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0); 363 /* GDM3 */ 364 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++) 365 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q, 366 PSE_QUEUE_RSV_PAGES); 367 /* PPE1 */ 368 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) { 369 if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2) 370 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 371 PSE_QUEUE_RSV_PAGES); 372 else 373 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0); 374 } 375 /* CDM2 */ 376 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++) 377 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q, 378 PSE_QUEUE_RSV_PAGES); 379 /* CDM3 */ 380 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++) 381 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0); 382 /* CDM4 */ 383 for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++) 384 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q, 385 PSE_QUEUE_RSV_PAGES); 386 if (airoha_ppe_is_enabled(eth, 1)) { 387 /* PPE2 */ 388 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) { 389 if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2) 390 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, 391 q, 392 PSE_QUEUE_RSV_PAGES); 393 else 394 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, 395 q, 0); 396 } 397 } 398 /* GDM4 */ 399 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++) 400 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q, 401 PSE_QUEUE_RSV_PAGES); 402 /* CDM5 */ 403 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++) 404 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q, 405 PSE_QUEUE_RSV_PAGES); 406 } 407 408 static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth) 409 { 410 int i; 411 412 for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) { 413 int err, j; 414 u32 val; 415 416 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0); 417 418 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) | 419 MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK; 420 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val); 421 err = read_poll_timeout(airoha_fe_rr, val, 422 val & MC_VLAN_CFG_CMD_DONE_MASK, 423 USEC_PER_MSEC, 5 * USEC_PER_MSEC, 424 false, eth, REG_MC_VLAN_CFG); 425 if (err) 426 return err; 427 428 for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) { 429 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0); 430 431 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) | 432 FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) | 433 MC_VLAN_CFG_RW_MASK; 434 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val); 435 err = read_poll_timeout(airoha_fe_rr, val, 436 val & MC_VLAN_CFG_CMD_DONE_MASK, 437 USEC_PER_MSEC, 438 5 * USEC_PER_MSEC, false, eth, 439 REG_MC_VLAN_CFG); 440 if (err) 441 return err; 442 } 443 } 444 445 return 0; 446 } 447 448 static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth) 449 { 450 /* CDM1_CRSN_QSEL */ 451 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_22 >> 2), 452 CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 453 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 454 CDM_CRSN_QSEL_Q1)); 455 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_08 >> 2), 456 CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 457 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 458 CDM_CRSN_QSEL_Q1)); 459 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_21 >> 2), 460 CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 461 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 462 CDM_CRSN_QSEL_Q1)); 463 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_24 >> 2), 464 CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 465 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 466 CDM_CRSN_QSEL_Q6)); 467 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_25 >> 2), 468 CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 469 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 470 CDM_CRSN_QSEL_Q1)); 471 /* CDM2_CRSN_QSEL */ 472 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_08 >> 2), 473 CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 474 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 475 CDM_CRSN_QSEL_Q1)); 476 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_21 >> 2), 477 CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 478 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 479 CDM_CRSN_QSEL_Q1)); 480 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_22 >> 2), 481 CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 482 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 483 CDM_CRSN_QSEL_Q1)); 484 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_24 >> 2), 485 CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 486 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 487 CDM_CRSN_QSEL_Q6)); 488 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_25 >> 2), 489 CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 490 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 491 CDM_CRSN_QSEL_Q1)); 492 } 493 494 static int airoha_fe_init(struct airoha_eth *eth) 495 { 496 airoha_fe_maccr_init(eth); 497 498 /* PSE IQ reserve */ 499 airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK, 500 FIELD_PREP(PSE_IQ_RES1_P2_MASK, 0x10)); 501 airoha_fe_rmw(eth, REG_PSE_IQ_REV2, 502 PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK, 503 FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) | 504 FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34)); 505 506 /* enable FE copy engine for KA/DPI */ 507 airoha_fe_wr(eth, REG_FE_PCE_CFG, PCE_DPI_EN_MASK | PCE_KA_EN_MASK); 508 /* set vip queue selection to ring 1 */ 509 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(1), CDM_VIP_QSEL_MASK, 510 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4)); 511 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_VIP_QSEL_MASK, 512 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4)); 513 /* set GDM4 source interface offset to 8 */ 514 airoha_fe_rmw(eth, REG_GDM_SRC_PORT_SET(4), 515 GDM_SPORT_OFF2_MASK | 516 GDM_SPORT_OFF1_MASK | 517 GDM_SPORT_OFF0_MASK, 518 FIELD_PREP(GDM_SPORT_OFF2_MASK, 8) | 519 FIELD_PREP(GDM_SPORT_OFF1_MASK, 8) | 520 FIELD_PREP(GDM_SPORT_OFF0_MASK, 8)); 521 522 /* set PSE Page as 128B */ 523 airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG, 524 FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK, 525 FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, 2) | 526 FE_DMA_GLO_PG_SZ_MASK); 527 airoha_fe_wr(eth, REG_FE_RST_GLO_CFG, 528 FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK | 529 FE_RST_GDM4_MBI_ARB_MASK); 530 usleep_range(1000, 2000); 531 532 /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1 533 * connect other rings to PSE Port0 OQ-0 534 */ 535 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4)); 536 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28)); 537 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4)); 538 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28)); 539 540 airoha_fe_vip_setup(eth); 541 airoha_fe_pse_ports_init(eth); 542 543 airoha_fe_set(eth, REG_GDM_MISC_CFG, 544 GDM2_RDM_ACK_WAIT_PREF_MASK | 545 GDM2_CHN_VLD_MODE_MASK); 546 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_OAM_QSEL_MASK, 547 FIELD_PREP(CDM_OAM_QSEL_MASK, 15)); 548 549 /* init fragment and assemble Force Port */ 550 /* NPU Core-3, NPU Bridge Channel-3 */ 551 airoha_fe_rmw(eth, REG_IP_FRAG_FP, 552 IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK, 553 FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) | 554 FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3)); 555 /* QDMA LAN, RX Ring-22 */ 556 airoha_fe_rmw(eth, REG_IP_FRAG_FP, 557 IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK, 558 FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) | 559 FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22)); 560 561 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM3_IDX), GDM_PAD_EN_MASK); 562 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM4_IDX), GDM_PAD_EN_MASK); 563 564 /* Enable split for MIB counters for GDM3 and GDM4 */ 565 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM3_IDX), 566 FE_GDM_TX_MIB_SPLIT_EN_MASK | 567 FE_GDM_RX_MIB_SPLIT_EN_MASK); 568 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM4_IDX), 569 FE_GDM_TX_MIB_SPLIT_EN_MASK | 570 FE_GDM_RX_MIB_SPLIT_EN_MASK); 571 572 airoha_fe_crsn_qsel_init(eth); 573 574 airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK); 575 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK); 576 577 /* default aging mode for mbi unlock issue */ 578 airoha_fe_rmw(eth, REG_GDM_CHN_RLS(2), 579 MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK, 580 FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) | 581 FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3)); 582 583 /* disable IFC by default */ 584 airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK); 585 586 /* enable 1:N vlan action, init vlan table */ 587 airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK); 588 589 return airoha_fe_mc_vlan_clear(eth); 590 } 591 592 static int airoha_qdma_fill_rx_queue(struct airoha_queue *q) 593 { 594 struct airoha_qdma *qdma = q->qdma; 595 int qid = q - &qdma->q_rx[0]; 596 int nframes = 0; 597 598 while (q->queued < q->ndesc - 1) { 599 struct airoha_queue_entry *e = &q->entry[q->head]; 600 struct airoha_qdma_desc *desc = &q->desc[q->head]; 601 struct page *page; 602 int offset; 603 u32 val; 604 605 page = page_pool_dev_alloc_frag(q->page_pool, &offset, 606 q->buf_size); 607 if (!page) 608 break; 609 610 q->head = (q->head + 1) % q->ndesc; 611 q->queued++; 612 nframes++; 613 614 offset += AIROHA_RX_HEADROOM; 615 e->buf = page_address(page) + offset; 616 e->dma_addr = page_pool_get_dma_addr(page) + offset; 617 e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size)); 618 619 val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len); 620 WRITE_ONCE(desc->ctrl, cpu_to_le32(val)); 621 WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr)); 622 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head); 623 WRITE_ONCE(desc->data, cpu_to_le32(val)); 624 WRITE_ONCE(desc->msg0, 0); 625 WRITE_ONCE(desc->msg1, 0); 626 WRITE_ONCE(desc->msg2, 0); 627 WRITE_ONCE(desc->msg3, 0); 628 } 629 630 if (nframes) 631 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), 632 RX_RING_CPU_IDX_MASK, 633 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); 634 635 return nframes; 636 } 637 638 static struct airoha_gdm_dev * 639 airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc *desc) 640 { 641 struct airoha_gdm_port *port; 642 u16 p, d; 643 644 if (eth->soc->ops.get_dev_from_sport(desc, &p, &d)) 645 return ERR_PTR(-ENODEV); 646 647 if (p >= ARRAY_SIZE(eth->ports)) 648 return ERR_PTR(-ENODEV); 649 650 port = eth->ports[p]; 651 if (!port) 652 return ERR_PTR(-ENODEV); 653 654 if (d >= ARRAY_SIZE(port->devs)) 655 return ERR_PTR(-ENODEV); 656 657 return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV); 658 } 659 660 static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) 661 { 662 enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool); 663 struct airoha_qdma *qdma = q->qdma; 664 struct airoha_eth *eth = qdma->eth; 665 int qid = q - &qdma->q_rx[0]; 666 int done = 0; 667 668 while (done < budget) { 669 struct airoha_queue_entry *e = &q->entry[q->tail]; 670 struct airoha_qdma_desc *desc = &q->desc[q->tail]; 671 u32 hash, reason, msg1, desc_ctrl; 672 struct airoha_gdm_dev *dev; 673 struct net_device *netdev; 674 int data_len, len; 675 struct page *page; 676 677 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); 678 if (!(desc_ctrl & QDMA_DESC_DONE_MASK)) 679 break; 680 681 dma_rmb(); 682 683 q->tail = (q->tail + 1) % q->ndesc; 684 q->queued--; 685 686 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, 687 dir); 688 689 page = virt_to_head_page(e->buf); 690 len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl); 691 data_len = q->skb ? AIROHA_RX_LEN(q->buf_size) : e->dma_len; 692 if (!len || data_len < len) 693 goto free_frag; 694 695 dev = airoha_qdma_get_gdm_dev(eth, desc); 696 if (IS_ERR(dev)) 697 goto free_frag; 698 699 netdev = netdev_from_priv(dev); 700 if (!q->skb) { /* first buffer */ 701 q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM, 702 q->buf_size); 703 if (!q->skb) 704 goto free_frag; 705 706 skb_reserve(q->skb, AIROHA_RX_HEADROOM); 707 __skb_put(q->skb, len); 708 skb_mark_for_recycle(q->skb); 709 q->skb->dev = netdev; 710 q->skb->protocol = eth_type_trans(q->skb, netdev); 711 q->skb->ip_summed = CHECKSUM_UNNECESSARY; 712 skb_record_rx_queue(q->skb, qid); 713 } else { /* scattered frame */ 714 struct skb_shared_info *shinfo = skb_shinfo(q->skb); 715 int nr_frags = shinfo->nr_frags; 716 717 if (nr_frags >= ARRAY_SIZE(shinfo->frags)) 718 goto free_frag; 719 720 skb_add_rx_frag(q->skb, nr_frags, page, 721 e->buf - page_address(page), len, 722 q->buf_size); 723 } 724 725 if (FIELD_GET(QDMA_DESC_MORE_MASK, desc_ctrl)) 726 continue; 727 728 if (netdev_uses_dsa(netdev)) { 729 struct airoha_gdm_port *port = dev->port; 730 731 /* PPE module requires untagged packets to work 732 * properly and it provides DSA port index via the 733 * DMA descriptor. Report DSA tag to the DSA stack 734 * via skb dst info. 735 */ 736 u32 msg0 = le32_to_cpu(READ_ONCE(desc->msg0)); 737 u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0); 738 739 if (sptag < ARRAY_SIZE(port->dsa_meta) && 740 port->dsa_meta[sptag]) 741 skb_dst_set_noref(q->skb, 742 &port->dsa_meta[sptag]->dst); 743 } 744 745 msg1 = le32_to_cpu(READ_ONCE(desc->msg1)); 746 hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1); 747 if (hash != AIROHA_RXD4_FOE_ENTRY) 748 skb_set_hash(q->skb, jhash_1word(hash, 0), 749 PKT_HASH_TYPE_L4); 750 751 reason = FIELD_GET(AIROHA_RXD4_PPE_CPU_REASON, msg1); 752 if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) 753 airoha_ppe_check_skb(ð->ppe->dev, q->skb, hash, 754 false); 755 756 done++; 757 napi_gro_receive(&q->napi, q->skb); 758 q->skb = NULL; 759 continue; 760 free_frag: 761 if (q->skb) { 762 dev_kfree_skb(q->skb); 763 q->skb = NULL; 764 } 765 page_pool_put_full_page(q->page_pool, page, true); 766 } 767 airoha_qdma_fill_rx_queue(q); 768 769 return done; 770 } 771 772 static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget) 773 { 774 struct airoha_queue *q = container_of(napi, struct airoha_queue, napi); 775 int cur, done = 0; 776 777 do { 778 cur = airoha_qdma_rx_process(q, budget - done); 779 done += cur; 780 } while (cur && done < budget); 781 782 if (done < budget && napi_complete(napi)) { 783 struct airoha_qdma *qdma = q->qdma; 784 int i, qid = q - &qdma->q_rx[0]; 785 int intr_reg = qid < RX_DONE_HIGH_OFFSET ? QDMA_INT_REG_IDX1 786 : QDMA_INT_REG_IDX2; 787 788 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 789 if (!(BIT(qid) & RX_IRQ_BANK_PIN_MASK(i))) 790 continue; 791 792 airoha_qdma_irq_enable(&qdma->irq_banks[i], intr_reg, 793 BIT(qid % RX_DONE_HIGH_OFFSET)); 794 } 795 } 796 797 return done; 798 } 799 800 static int airoha_qdma_init_rx_queue(struct airoha_queue *q, 801 struct airoha_qdma *qdma, int ndesc) 802 { 803 const struct page_pool_params pp_params = { 804 .order = 0, 805 .pool_size = 256, 806 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 807 .dma_dir = DMA_FROM_DEVICE, 808 .max_len = PAGE_SIZE, 809 .nid = NUMA_NO_NODE, 810 .dev = qdma->eth->dev, 811 .napi = &q->napi, 812 }; 813 struct airoha_eth *eth = qdma->eth; 814 int qid = q - &qdma->q_rx[0], thr; 815 dma_addr_t dma_addr; 816 817 q->buf_size = PAGE_SIZE / 2; 818 q->qdma = qdma; 819 820 q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry), 821 GFP_KERNEL); 822 if (!q->entry) 823 return -ENOMEM; 824 825 q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc), 826 &dma_addr, GFP_KERNEL); 827 if (!q->desc) 828 return -ENOMEM; 829 830 q->page_pool = page_pool_create(&pp_params); 831 if (IS_ERR(q->page_pool)) { 832 int err = PTR_ERR(q->page_pool); 833 834 q->page_pool = NULL; 835 return err; 836 } 837 838 q->ndesc = ndesc; 839 netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll); 840 841 airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr); 842 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), 843 RX_RING_SIZE_MASK, 844 FIELD_PREP(RX_RING_SIZE_MASK, ndesc)); 845 846 thr = clamp(ndesc >> 3, 1, 32); 847 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK, 848 FIELD_PREP(RX_RING_THR_MASK, thr)); 849 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, 850 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head)); 851 airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid), RX_RING_SG_EN_MASK); 852 853 airoha_qdma_fill_rx_queue(q); 854 855 return 0; 856 } 857 858 static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q) 859 { 860 struct airoha_qdma *qdma = q->qdma; 861 struct airoha_eth *eth = qdma->eth; 862 int qid = q - &qdma->q_rx[0]; 863 864 while (q->queued) { 865 struct airoha_queue_entry *e = &q->entry[q->tail]; 866 struct airoha_qdma_desc *desc = &q->desc[q->tail]; 867 struct page *page = virt_to_head_page(e->buf); 868 869 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, 870 page_pool_get_dma_dir(q->page_pool)); 871 page_pool_put_full_page(q->page_pool, page, false); 872 /* Reset DMA descriptor */ 873 WRITE_ONCE(desc->ctrl, 0); 874 WRITE_ONCE(desc->addr, 0); 875 WRITE_ONCE(desc->data, 0); 876 WRITE_ONCE(desc->msg0, 0); 877 WRITE_ONCE(desc->msg1, 0); 878 WRITE_ONCE(desc->msg2, 0); 879 WRITE_ONCE(desc->msg3, 0); 880 881 q->tail = (q->tail + 1) % q->ndesc; 882 q->queued--; 883 } 884 885 q->head = q->tail; 886 /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is 887 * empty. 888 */ 889 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK, 890 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); 891 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, 892 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail)); 893 } 894 895 static int airoha_qdma_init_rx(struct airoha_qdma *qdma) 896 { 897 int i; 898 899 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 900 int err; 901 902 if (!(RX_DONE_INT_MASK & BIT(i))) { 903 /* rx-queue not binded to irq */ 904 continue; 905 } 906 907 err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma, 908 RX_DSCP_NUM(i)); 909 if (err) 910 return err; 911 } 912 913 return 0; 914 } 915 916 static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q) 917 { 918 struct airoha_qdma *qdma = q->qdma; 919 struct airoha_eth *eth = qdma->eth; 920 int i, qid = q - &qdma->q_tx[0]; 921 922 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 923 struct airoha_gdm_port *port = eth->ports[i]; 924 int d; 925 926 if (!port) 927 continue; 928 929 for (d = 0; d < ARRAY_SIZE(port->devs); d++) { 930 struct airoha_gdm_dev *dev = port->devs[d]; 931 struct net_device *netdev; 932 int j; 933 934 if (!dev) 935 continue; 936 937 if (rcu_access_pointer(dev->qdma) != qdma) 938 continue; 939 940 netdev = netdev_from_priv(dev); 941 for (j = 0; j < netdev->num_tx_queues; j++) { 942 if (airoha_qdma_get_txq(qdma, j) != qid) 943 continue; 944 945 netif_wake_subqueue(netdev, j); 946 } 947 } 948 } 949 q->txq_stopped = false; 950 } 951 952 static void airoha_unmap_xmit_buf(struct airoha_eth *eth, 953 struct airoha_queue_entry *e) 954 { 955 switch (e->dma_type) { 956 case AIROHA_DMA_MAP_PAGE: 957 dma_unmap_page(eth->dev, e->dma_addr, e->dma_len, 958 DMA_TO_DEVICE); 959 break; 960 case AIROHA_DMA_MAP_SINGLE: 961 dma_unmap_single(eth->dev, e->dma_addr, e->dma_len, 962 DMA_TO_DEVICE); 963 break; 964 case AIROHA_DMA_UNMAPPED: 965 default: 966 break; 967 } 968 e->dma_type = AIROHA_DMA_UNMAPPED; 969 } 970 971 static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget) 972 { 973 struct airoha_tx_irq_queue *irq_q; 974 int id, done = 0, irq_queued; 975 struct airoha_qdma *qdma; 976 struct airoha_eth *eth; 977 u32 status, head; 978 979 irq_q = container_of(napi, struct airoha_tx_irq_queue, napi); 980 qdma = irq_q->qdma; 981 id = irq_q - &qdma->q_tx_irq[0]; 982 eth = qdma->eth; 983 984 status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id)); 985 head = FIELD_GET(IRQ_HEAD_IDX_MASK, status); 986 head = head % irq_q->size; 987 irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status); 988 989 while (irq_queued > 0 && done < budget) { 990 u32 qid, val = irq_q->q[head]; 991 struct airoha_qdma_desc *desc; 992 struct airoha_queue_entry *e; 993 struct airoha_queue *q; 994 u32 index, desc_ctrl; 995 struct sk_buff *skb; 996 997 if (val == 0xff) 998 break; 999 1000 irq_q->q[head] = 0xff; /* mark as done */ 1001 head = (head + 1) % irq_q->size; 1002 irq_queued--; 1003 done++; 1004 1005 qid = FIELD_GET(IRQ_RING_IDX_MASK, val); 1006 if (qid >= ARRAY_SIZE(qdma->q_tx)) 1007 continue; 1008 1009 q = &qdma->q_tx[qid]; 1010 if (!q->ndesc) 1011 continue; 1012 1013 index = FIELD_GET(IRQ_DESC_IDX_MASK, val); 1014 if (index >= q->ndesc) 1015 continue; 1016 1017 spin_lock_bh(&q->lock); 1018 1019 if (!q->queued) 1020 goto unlock; 1021 1022 desc = &q->desc[index]; 1023 desc_ctrl = le32_to_cpu(desc->ctrl); 1024 1025 if (!(desc_ctrl & QDMA_DESC_DONE_MASK) && 1026 !(desc_ctrl & QDMA_DESC_DROP_MASK)) 1027 goto unlock; 1028 1029 e = &q->entry[index]; 1030 skb = e->skb; 1031 e->skb = NULL; 1032 1033 airoha_unmap_xmit_buf(eth, e); 1034 list_add_tail(&e->list, &q->tx_list); 1035 1036 WRITE_ONCE(desc->msg0, 0); 1037 WRITE_ONCE(desc->msg1, 0); 1038 q->queued--; 1039 1040 if (skb) { 1041 struct airoha_gdm_dev *dev = netdev_priv(skb->dev); 1042 u16 qidx = skb_get_queue_mapping(skb); 1043 struct netdev_queue *txq; 1044 1045 txq = skb_get_tx_queue(skb->dev, skb); 1046 1047 spin_lock_bh(&dev->txq_lock[qidx]); 1048 netdev_tx_completed_queue(txq, 1, skb->len); 1049 spin_unlock_bh(&dev->txq_lock[qidx]); 1050 1051 dev_kfree_skb_any(skb); 1052 } 1053 1054 if (q->txq_stopped && q->ndesc - q->queued >= q->free_thr) { 1055 /* Since multiple net_device TX queues can share the 1056 * same hw QDMA TX queue, there is no guarantee we have 1057 * inflight packets queued in hw belonging to a 1058 * net_device TX queue stopped in the xmit path. 1059 * In order to avoid any potential net_device TX queue 1060 * stall, we need to wake all the net_device TX queues 1061 * feeding the same hw QDMA TX queue. 1062 */ 1063 airoha_qdma_wake_netdev_txqs(q); 1064 } 1065 1066 unlock: 1067 spin_unlock_bh(&q->lock); 1068 } 1069 1070 if (done) { 1071 int i, len = done >> 7; 1072 1073 for (i = 0; i < len; i++) 1074 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id), 1075 IRQ_CLEAR_LEN_MASK, 0x80); 1076 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id), 1077 IRQ_CLEAR_LEN_MASK, (done & 0x7f)); 1078 } 1079 1080 if (done < budget && napi_complete(napi)) 1081 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0, 1082 TX_DONE_INT_MASK(id)); 1083 1084 return done; 1085 } 1086 1087 static int airoha_qdma_init_tx_queue(struct airoha_queue *q, 1088 struct airoha_qdma *qdma, int size) 1089 { 1090 struct airoha_eth *eth = qdma->eth; 1091 int i, qid = q - &qdma->q_tx[0]; 1092 dma_addr_t dma_addr; 1093 1094 spin_lock_init(&q->lock); 1095 q->qdma = qdma; 1096 q->free_thr = 1 + MAX_SKB_FRAGS; 1097 INIT_LIST_HEAD(&q->tx_list); 1098 1099 q->entry = devm_kzalloc(eth->dev, size * sizeof(*q->entry), 1100 GFP_KERNEL); 1101 if (!q->entry) 1102 return -ENOMEM; 1103 1104 q->desc = dmam_alloc_coherent(eth->dev, size * sizeof(*q->desc), 1105 &dma_addr, GFP_KERNEL); 1106 if (!q->desc) 1107 return -ENOMEM; 1108 1109 for (i = 0; i < size; i++) { 1110 u32 val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1); 1111 1112 list_add_tail(&q->entry[i].list, &q->tx_list); 1113 WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val)); 1114 } 1115 q->ndesc = size; 1116 1117 /* xmit ring drop default setting */ 1118 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(qid), 1119 TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK); 1120 1121 airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr); 1122 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK, 1123 FIELD_PREP(TX_RING_CPU_IDX_MASK, 0)); 1124 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK, 1125 FIELD_PREP(TX_RING_DMA_IDX_MASK, 0)); 1126 1127 return 0; 1128 } 1129 1130 static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q, 1131 struct airoha_qdma *qdma, int size) 1132 { 1133 int id = irq_q - &qdma->q_tx_irq[0]; 1134 struct airoha_eth *eth = qdma->eth; 1135 dma_addr_t dma_addr; 1136 1137 irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32), 1138 &dma_addr, GFP_KERNEL); 1139 if (!irq_q->q) 1140 return -ENOMEM; 1141 1142 memset(irq_q->q, 0xff, size * sizeof(u32)); 1143 irq_q->size = size; 1144 irq_q->qdma = qdma; 1145 1146 netif_napi_add_tx(eth->napi_dev, &irq_q->napi, 1147 airoha_qdma_tx_napi_poll); 1148 1149 airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr); 1150 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK, 1151 FIELD_PREP(TX_IRQ_DEPTH_MASK, size)); 1152 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK, 1153 FIELD_PREP(TX_IRQ_THR_MASK, 1)); 1154 1155 return 0; 1156 } 1157 1158 static int airoha_qdma_init_tx(struct airoha_qdma *qdma) 1159 { 1160 int i, err; 1161 1162 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1163 err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma, 1164 IRQ_QUEUE_LEN(i)); 1165 if (err) 1166 return err; 1167 } 1168 1169 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1170 err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma, 1171 TX_DSCP_NUM); 1172 if (err) 1173 return err; 1174 } 1175 1176 return 0; 1177 } 1178 1179 static void airoha_qdma_tx_cleanup(struct airoha_qdma *qdma) 1180 { 1181 u32 status; 1182 int i; 1183 1184 airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG, 1185 GLOBAL_CFG_TX_DMA_EN_MASK); 1186 if (read_poll_timeout(airoha_qdma_rr, status, 1187 !(status & GLOBAL_CFG_TX_DMA_BUSY_MASK), 1188 USEC_PER_MSEC, 50 * USEC_PER_MSEC, true, 1189 qdma, REG_QDMA_GLOBAL_CFG)) 1190 dev_warn(qdma->eth->dev, "QDMA TX DMA busy timeout\n"); 1191 1192 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1193 struct airoha_queue *q = &qdma->q_tx[i]; 1194 u16 index = 0; 1195 int j; 1196 1197 if (!q->ndesc) 1198 continue; 1199 1200 spin_lock_bh(&q->lock); 1201 1202 q->flushing = true; 1203 for (j = 0; j < q->ndesc; j++) { 1204 struct airoha_queue_entry *e = &q->entry[j]; 1205 struct airoha_qdma_desc *desc = &q->desc[j]; 1206 struct sk_buff *skb = e->skb; 1207 1208 if (e->dma_type == AIROHA_DMA_UNMAPPED) 1209 continue; 1210 1211 airoha_unmap_xmit_buf(qdma->eth, e); 1212 list_add_tail(&e->list, &q->tx_list); 1213 1214 WRITE_ONCE(desc->ctrl, 0); 1215 WRITE_ONCE(desc->addr, 0); 1216 WRITE_ONCE(desc->data, 0); 1217 WRITE_ONCE(desc->msg0, 0); 1218 WRITE_ONCE(desc->msg1, 0); 1219 WRITE_ONCE(desc->msg2, 0); 1220 1221 if (skb) { 1222 struct netdev_queue *txq; 1223 1224 txq = skb_get_tx_queue(skb->dev, skb); 1225 netdev_tx_completed_queue(txq, 1, skb->len); 1226 dev_kfree_skb_any(skb); 1227 e->skb = NULL; 1228 } 1229 1230 q->queued--; 1231 } 1232 1233 if (!list_empty(&q->tx_list)) { 1234 struct airoha_queue_entry *e; 1235 1236 e = list_first_entry(&q->tx_list, 1237 struct airoha_queue_entry, list); 1238 index = e - q->entry; 1239 } 1240 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(i), TX_RING_CPU_IDX_MASK, 1241 FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); 1242 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(i), TX_RING_DMA_IDX_MASK, 1243 FIELD_PREP(TX_RING_DMA_IDX_MASK, index)); 1244 1245 spin_unlock_bh(&q->lock); 1246 } 1247 } 1248 1249 static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma) 1250 { 1251 int size, index, num_desc = HW_DSCP_NUM; 1252 struct airoha_eth *eth = qdma->eth; 1253 int id = qdma - ð->qdma[0]; 1254 u32 status, buf_size; 1255 dma_addr_t dma_addr; 1256 const char *name; 1257 1258 name = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d-buf", id); 1259 if (!name) 1260 return -ENOMEM; 1261 1262 buf_size = id ? AIROHA_MAX_PACKET_SIZE / 2 : AIROHA_MAX_PACKET_SIZE; 1263 index = of_property_match_string(eth->dev->of_node, 1264 "memory-region-names", name); 1265 if (index >= 0) { 1266 struct reserved_mem *rmem; 1267 struct device_node *np; 1268 1269 /* Consume reserved memory for hw forwarding buffers queue if 1270 * available in the DTS 1271 */ 1272 np = of_parse_phandle(eth->dev->of_node, "memory-region", 1273 index); 1274 if (!np) 1275 return -ENODEV; 1276 1277 rmem = of_reserved_mem_lookup(np); 1278 of_node_put(np); 1279 if (!rmem) 1280 return -ENODEV; 1281 1282 dma_addr = rmem->base; 1283 /* Compute the number of hw descriptors according to the 1284 * reserved memory size and the payload buffer size 1285 */ 1286 num_desc = div_u64(rmem->size, buf_size); 1287 } else { 1288 size = buf_size * num_desc; 1289 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, 1290 GFP_KERNEL)) 1291 return -ENOMEM; 1292 } 1293 1294 airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr); 1295 1296 size = num_desc * sizeof(struct airoha_qdma_fwd_desc); 1297 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, GFP_KERNEL)) 1298 return -ENOMEM; 1299 1300 airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr); 1301 /* QDMA0: 2KB. QDMA1: 1KB */ 1302 airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG, 1303 HW_FWD_DSCP_PAYLOAD_SIZE_MASK, 1304 FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK, !!id)); 1305 airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK, 1306 FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128)); 1307 airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG, 1308 LMGR_INIT_START | LMGR_SRAM_MODE_MASK | 1309 HW_FWD_DESC_NUM_MASK, 1310 FIELD_PREP(HW_FWD_DESC_NUM_MASK, num_desc) | 1311 LMGR_INIT_START | LMGR_SRAM_MODE_MASK); 1312 1313 return read_poll_timeout(airoha_qdma_rr, status, 1314 !(status & LMGR_INIT_START), USEC_PER_MSEC, 1315 30 * USEC_PER_MSEC, true, qdma, 1316 REG_LMGR_INIT_CFG); 1317 } 1318 1319 static void airoha_qdma_init_qos(struct airoha_qdma *qdma) 1320 { 1321 airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK); 1322 airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK); 1323 1324 airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG, 1325 PSE_BUF_ESTIMATE_EN_MASK); 1326 1327 airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG, 1328 EGRESS_RATE_METER_EN_MASK | 1329 EGRESS_RATE_METER_EQ_RATE_EN_MASK); 1330 /* 2047us x 31 = 63.457ms */ 1331 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG, 1332 EGRESS_RATE_METER_WINDOW_SZ_MASK, 1333 FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK, 0x1f)); 1334 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG, 1335 EGRESS_RATE_METER_TIMESLICE_MASK, 1336 FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK, 0x7ff)); 1337 1338 /* ratelimit init */ 1339 airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK); 1340 /* fast-tick 25us */ 1341 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK, 1342 FIELD_PREP(GLB_FAST_TICK_MASK, 25)); 1343 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK, 1344 FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40)); 1345 1346 airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK); 1347 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK, 1348 FIELD_PREP(EGRESS_FAST_TICK_MASK, 25)); 1349 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, 1350 EGRESS_SLOW_TICK_RATIO_MASK, 1351 FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40)); 1352 1353 airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK); 1354 airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG, 1355 INGRESS_TRTCM_MODE_MASK); 1356 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK, 1357 FIELD_PREP(INGRESS_FAST_TICK_MASK, 125)); 1358 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, 1359 INGRESS_SLOW_TICK_RATIO_MASK, 1360 FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8)); 1361 1362 airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK); 1363 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK, 1364 FIELD_PREP(SLA_FAST_TICK_MASK, 25)); 1365 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK, 1366 FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40)); 1367 } 1368 1369 static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma) 1370 { 1371 int i; 1372 1373 for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) { 1374 /* Tx-cpu transferred count */ 1375 airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0); 1376 airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1), 1377 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK | 1378 CNTR_ALL_DSCP_RING_EN_MASK | 1379 FIELD_PREP(CNTR_CHAN_MASK, i)); 1380 /* Tx-fwd transferred count */ 1381 airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0); 1382 airoha_qdma_wr(qdma, REG_CNTR_CFG((i << 1) + 1), 1383 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK | 1384 CNTR_ALL_DSCP_RING_EN_MASK | 1385 FIELD_PREP(CNTR_SRC_MASK, 1) | 1386 FIELD_PREP(CNTR_CHAN_MASK, i)); 1387 } 1388 } 1389 1390 static int airoha_qdma_hw_init(struct airoha_qdma *qdma) 1391 { 1392 int i; 1393 1394 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 1395 /* clear pending irqs */ 1396 airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff); 1397 /* setup rx irqs */ 1398 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX0, 1399 INT_RX0_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1400 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX1, 1401 INT_RX1_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1402 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX2, 1403 INT_RX2_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1404 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX3, 1405 INT_RX3_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1406 } 1407 /* setup tx irqs */ 1408 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0, 1409 TX_COHERENT_LOW_INT_MASK | INT_TX_MASK); 1410 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX4, 1411 TX_COHERENT_HIGH_INT_MASK); 1412 1413 /* setup irq binding */ 1414 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1415 if (!qdma->q_tx[i].ndesc) 1416 continue; 1417 1418 if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i)) 1419 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i), 1420 TX_RING_IRQ_BLOCKING_CFG_MASK); 1421 else 1422 airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i), 1423 TX_RING_IRQ_BLOCKING_CFG_MASK); 1424 } 1425 1426 airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG, 1427 FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) | 1428 GLOBAL_CFG_CPU_TXR_RR_MASK | 1429 GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK | 1430 GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK | 1431 GLOBAL_CFG_MULTICAST_EN_MASK | 1432 GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK | 1433 GLOBAL_CFG_TX_WB_DONE_MASK | 1434 FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2)); 1435 1436 airoha_qdma_init_qos(qdma); 1437 1438 /* disable qdma rx delay interrupt */ 1439 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1440 if (!qdma->q_rx[i].ndesc) 1441 continue; 1442 1443 airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i), 1444 RX_DELAY_INT_MASK); 1445 } 1446 1447 airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG, 1448 TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN); 1449 airoha_qdma_init_qos_stats(qdma); 1450 1451 return 0; 1452 } 1453 1454 static irqreturn_t airoha_irq_handler(int irq, void *dev_instance) 1455 { 1456 struct airoha_irq_bank *irq_bank = dev_instance; 1457 struct airoha_qdma *qdma = irq_bank->qdma; 1458 u32 rx_intr_mask = 0, rx_intr1, rx_intr2; 1459 u32 intr[ARRAY_SIZE(irq_bank->irqmask)]; 1460 int i; 1461 1462 for (i = 0; i < ARRAY_SIZE(intr); i++) { 1463 intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i)); 1464 intr[i] &= irq_bank->irqmask[i]; 1465 airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]); 1466 } 1467 1468 if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state)) 1469 return IRQ_NONE; 1470 1471 rx_intr1 = intr[1] & RX_DONE_LOW_INT_MASK; 1472 if (rx_intr1) { 1473 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX1, rx_intr1); 1474 rx_intr_mask |= rx_intr1; 1475 } 1476 1477 rx_intr2 = intr[2] & RX_DONE_HIGH_INT_MASK; 1478 if (rx_intr2) { 1479 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX2, rx_intr2); 1480 rx_intr_mask |= (rx_intr2 << 16); 1481 } 1482 1483 for (i = 0; rx_intr_mask && i < ARRAY_SIZE(qdma->q_rx); i++) { 1484 if (!qdma->q_rx[i].ndesc) 1485 continue; 1486 1487 if (rx_intr_mask & BIT(i)) 1488 napi_schedule(&qdma->q_rx[i].napi); 1489 } 1490 1491 if (intr[0] & INT_TX_MASK) { 1492 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1493 if (!(intr[0] & TX_DONE_INT_MASK(i))) 1494 continue; 1495 1496 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX0, 1497 TX_DONE_INT_MASK(i)); 1498 napi_schedule(&qdma->q_tx_irq[i].napi); 1499 } 1500 } 1501 1502 return IRQ_HANDLED; 1503 } 1504 1505 static int airoha_qdma_init_irq_banks(struct platform_device *pdev, 1506 struct airoha_qdma *qdma) 1507 { 1508 struct airoha_eth *eth = qdma->eth; 1509 int i, id = qdma - ð->qdma[0]; 1510 1511 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 1512 struct airoha_irq_bank *irq_bank = &qdma->irq_banks[i]; 1513 int err, irq_index = 4 * id + i; 1514 const char *name; 1515 1516 spin_lock_init(&irq_bank->irq_lock); 1517 irq_bank->qdma = qdma; 1518 1519 irq_bank->irq = platform_get_irq(pdev, irq_index); 1520 if (irq_bank->irq < 0) 1521 return irq_bank->irq; 1522 1523 name = devm_kasprintf(eth->dev, GFP_KERNEL, 1524 KBUILD_MODNAME ".%d", irq_index); 1525 if (!name) 1526 return -ENOMEM; 1527 1528 err = devm_request_irq(eth->dev, irq_bank->irq, 1529 airoha_irq_handler, IRQF_SHARED, name, 1530 irq_bank); 1531 if (err) 1532 return err; 1533 } 1534 1535 return 0; 1536 } 1537 1538 static int airoha_qdma_init(struct platform_device *pdev, 1539 struct airoha_eth *eth, 1540 struct airoha_qdma *qdma) 1541 { 1542 int err, id = qdma - ð->qdma[0]; 1543 const char *res; 1544 1545 qdma->eth = eth; 1546 res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id); 1547 if (!res) 1548 return -ENOMEM; 1549 1550 qdma->regs = devm_platform_ioremap_resource_byname(pdev, res); 1551 if (IS_ERR(qdma->regs)) 1552 return dev_err_probe(eth->dev, PTR_ERR(qdma->regs), 1553 "failed to iomap qdma%d regs\n", id); 1554 1555 err = airoha_qdma_init_irq_banks(pdev, qdma); 1556 if (err) 1557 return err; 1558 1559 err = airoha_qdma_init_rx(qdma); 1560 if (err) 1561 return err; 1562 1563 err = airoha_qdma_init_tx(qdma); 1564 if (err) 1565 return err; 1566 1567 err = airoha_qdma_init_hfwd_queues(qdma); 1568 if (err) 1569 return err; 1570 1571 return airoha_qdma_hw_init(qdma); 1572 } 1573 1574 static void airoha_qdma_cleanup(struct airoha_eth *eth, 1575 struct airoha_qdma *qdma) 1576 { 1577 int i; 1578 1579 if (test_bit(DEV_STATE_INITIALIZED, ð->state)) { 1580 u32 status; 1581 1582 airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG, 1583 GLOBAL_CFG_RX_DMA_EN_MASK); 1584 if (read_poll_timeout(airoha_qdma_rr, status, 1585 !(status & GLOBAL_CFG_RX_DMA_BUSY_MASK), 1586 USEC_PER_MSEC, 50 * USEC_PER_MSEC, true, 1587 qdma, REG_QDMA_GLOBAL_CFG)) 1588 dev_warn(eth->dev, "QDMA RX DMA busy timeout\n"); 1589 } 1590 1591 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1592 if (!qdma->q_rx[i].ndesc) 1593 continue; 1594 1595 netif_napi_del(&qdma->q_rx[i].napi); 1596 airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]); 1597 if (qdma->q_rx[i].page_pool) { 1598 page_pool_destroy(qdma->q_rx[i].page_pool); 1599 qdma->q_rx[i].page_pool = NULL; 1600 } 1601 } 1602 1603 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1604 if (!qdma->q_tx_irq[i].size) 1605 continue; 1606 1607 netif_napi_del(&qdma->q_tx_irq[i].napi); 1608 } 1609 1610 } 1611 1612 static int airoha_hw_init(struct platform_device *pdev, 1613 struct airoha_eth *eth) 1614 { 1615 int err, i; 1616 1617 /* disable xsi */ 1618 err = reset_control_bulk_assert(eth->soc->num_xsi_rsts, eth->xsi_rsts); 1619 if (err) 1620 return err; 1621 1622 err = reset_control_bulk_assert(ARRAY_SIZE(eth->rsts), eth->rsts); 1623 if (err) 1624 return err; 1625 1626 msleep(20); 1627 err = reset_control_bulk_deassert(ARRAY_SIZE(eth->rsts), eth->rsts); 1628 if (err) 1629 return err; 1630 1631 msleep(20); 1632 err = airoha_fe_init(eth); 1633 if (err) 1634 return err; 1635 1636 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 1637 err = airoha_qdma_init(pdev, eth, ð->qdma[i]); 1638 if (err) 1639 goto error; 1640 } 1641 1642 err = airoha_ppe_init(eth); 1643 if (err) 1644 goto error; 1645 1646 set_bit(DEV_STATE_INITIALIZED, ð->state); 1647 1648 return 0; 1649 error: 1650 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 1651 airoha_qdma_cleanup(eth, ð->qdma[i]); 1652 1653 return err; 1654 } 1655 1656 static void airoha_hw_cleanup(struct airoha_eth *eth) 1657 { 1658 int i; 1659 1660 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 1661 airoha_qdma_cleanup(eth, ð->qdma[i]); 1662 airoha_ppe_deinit(eth); 1663 } 1664 1665 static void airoha_qdma_start_napi(struct airoha_qdma *qdma) 1666 { 1667 int i; 1668 1669 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) 1670 napi_enable(&qdma->q_tx_irq[i].napi); 1671 1672 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1673 if (!qdma->q_rx[i].ndesc) 1674 continue; 1675 1676 napi_enable(&qdma->q_rx[i].napi); 1677 } 1678 } 1679 1680 static void airoha_qdma_stop_napi(struct airoha_qdma *qdma) 1681 { 1682 int i; 1683 1684 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) 1685 napi_disable(&qdma->q_tx_irq[i].napi); 1686 1687 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1688 if (!qdma->q_rx[i].ndesc) 1689 continue; 1690 1691 napi_disable(&qdma->q_rx[i].napi); 1692 } 1693 } 1694 1695 static void airoha_update_hw_stats(struct airoha_gdm_dev *dev) 1696 { 1697 struct airoha_gdm_port *port = dev->port; 1698 struct airoha_eth *eth = dev->eth; 1699 u32 val, i = 0; 1700 u64 data; 1701 1702 spin_lock(&port->stats_lock); 1703 1704 /* Read relevant MIB for GDM with multiple port attached */ 1705 if (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX) 1706 airoha_fe_rmw(eth, REG_FE_GDM_MIB_CFG(port->id), 1707 FE_TX_MIB_ID_MASK | FE_RX_MIB_ID_MASK, 1708 FIELD_PREP(FE_TX_MIB_ID_MASK, dev->nbq) | 1709 FIELD_PREP(FE_RX_MIB_ID_MASK, dev->nbq)); 1710 1711 u64_stats_update_begin(&dev->stats.syncp); 1712 1713 /* TX - 64-bit H+L registers: hw accumulates the total, read directly. 1714 * Use local variable to prevent readers from seeing intermediate values. 1715 * Clamp to prevent regression from torn reads between H and L. 1716 */ 1717 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_H(port->id)); 1718 data = (u64)val << 32; 1719 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_L(port->id)); 1720 data += val; 1721 dev->stats.tx_ok_pkts = max(data, dev->stats.tx_ok_pkts); 1722 1723 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_H(port->id)); 1724 data = (u64)val << 32; 1725 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_L(port->id)); 1726 data += val; 1727 dev->stats.tx_ok_bytes = max(data, dev->stats.tx_ok_bytes); 1728 1729 /* TX - 32-bit registers: accumulate delta to handle wrap-around. */ 1730 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_DROP_CNT(port->id)); 1731 dev->stats.tx_drops += (u32)(val - dev->stats.mib_prev.tx_drops); 1732 dev->stats.mib_prev.tx_drops = val; 1733 1734 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_BC_CNT(port->id)); 1735 dev->stats.tx_broadcast += (u32)(val - dev->stats.mib_prev.tx_broadcast); 1736 dev->stats.mib_prev.tx_broadcast = val; 1737 1738 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_MC_CNT(port->id)); 1739 dev->stats.tx_multicast += (u32)(val - dev->stats.mib_prev.tx_multicast); 1740 dev->stats.mib_prev.tx_multicast = val; 1741 1742 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_RUNT_CNT(port->id)); 1743 dev->stats.mib_prev.tx_runt64 += 1744 (u32)(val - dev->stats.mib_prev.tx_runt); 1745 dev->stats.mib_prev.tx_runt = val; 1746 1747 /* tx_len[0]: RUNT (32-bit, delta) + E64 (64-bit, absolute). */ 1748 data = dev->stats.mib_prev.tx_runt64; 1749 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_H(port->id)); 1750 data += (u64)val << 32; 1751 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_L(port->id)); 1752 data += val; 1753 dev->stats.tx_len[i] = max(data, dev->stats.tx_len[i]); 1754 i++; 1755 1756 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_H(port->id)); 1757 data = (u64)val << 32; 1758 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_L(port->id)); 1759 data += val; 1760 dev->stats.tx_len[i] = max(data, dev->stats.tx_len[i]); 1761 i++; 1762 1763 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_H(port->id)); 1764 data = (u64)val << 32; 1765 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_L(port->id)); 1766 data += val; 1767 dev->stats.tx_len[i] = max(data, dev->stats.tx_len[i]); 1768 i++; 1769 1770 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_H(port->id)); 1771 data = (u64)val << 32; 1772 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_L(port->id)); 1773 data += val; 1774 dev->stats.tx_len[i] = max(data, dev->stats.tx_len[i]); 1775 i++; 1776 1777 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_H(port->id)); 1778 data = (u64)val << 32; 1779 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_L(port->id)); 1780 data += val; 1781 dev->stats.tx_len[i] = max(data, dev->stats.tx_len[i]); 1782 i++; 1783 1784 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_H(port->id)); 1785 data = (u64)val << 32; 1786 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_L(port->id)); 1787 data += val; 1788 dev->stats.tx_len[i] = max(data, dev->stats.tx_len[i]); 1789 i++; 1790 1791 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_LONG_CNT(port->id)); 1792 dev->stats.tx_len[i++] += (u32)(val - dev->stats.mib_prev.tx_long); 1793 dev->stats.mib_prev.tx_long = val; 1794 1795 /* RX */ 1796 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_H(port->id)); 1797 data = (u64)val << 32; 1798 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_L(port->id)); 1799 data += val; 1800 dev->stats.rx_ok_pkts = max(data, dev->stats.rx_ok_pkts); 1801 1802 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_H(port->id)); 1803 data = (u64)val << 32; 1804 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_L(port->id)); 1805 data += val; 1806 dev->stats.rx_ok_bytes = max(data, dev->stats.rx_ok_bytes); 1807 1808 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_DROP_CNT(port->id)); 1809 dev->stats.rx_drops += (u32)(val - dev->stats.mib_prev.rx_drops); 1810 dev->stats.mib_prev.rx_drops = val; 1811 1812 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_BC_CNT(port->id)); 1813 dev->stats.rx_broadcast += (u32)(val - dev->stats.mib_prev.rx_broadcast); 1814 dev->stats.mib_prev.rx_broadcast = val; 1815 1816 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_MC_CNT(port->id)); 1817 dev->stats.rx_multicast += (u32)(val - dev->stats.mib_prev.rx_multicast); 1818 dev->stats.mib_prev.rx_multicast = val; 1819 1820 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ERROR_DROP_CNT(port->id)); 1821 dev->stats.rx_errors += (u32)(val - dev->stats.mib_prev.rx_errors); 1822 dev->stats.mib_prev.rx_errors = val; 1823 1824 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_CRC_ERR_CNT(port->id)); 1825 dev->stats.rx_crc_error += (u32)(val - dev->stats.mib_prev.rx_crc_error); 1826 dev->stats.mib_prev.rx_crc_error = val; 1827 1828 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OVERFLOW_DROP_CNT(port->id)); 1829 dev->stats.rx_over_errors += (u32)(val - dev->stats.mib_prev.rx_over_errors); 1830 dev->stats.mib_prev.rx_over_errors = val; 1831 1832 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_FRAG_CNT(port->id)); 1833 dev->stats.rx_fragment += (u32)(val - dev->stats.mib_prev.rx_fragment); 1834 dev->stats.mib_prev.rx_fragment = val; 1835 1836 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_JABBER_CNT(port->id)); 1837 dev->stats.rx_jabber += (u32)(val - dev->stats.mib_prev.rx_jabber); 1838 dev->stats.mib_prev.rx_jabber = val; 1839 1840 i = 0; 1841 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_RUNT_CNT(port->id)); 1842 dev->stats.mib_prev.rx_runt64 += 1843 (u32)(val - dev->stats.mib_prev.rx_runt); 1844 dev->stats.mib_prev.rx_runt = val; 1845 1846 /* rx_len[0]: RUNT (32-bit, delta) + E64 (64-bit, absolute). */ 1847 data = dev->stats.mib_prev.rx_runt64; 1848 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_H(port->id)); 1849 data += (u64)val << 32; 1850 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_L(port->id)); 1851 data += val; 1852 dev->stats.rx_len[i] = max(data, dev->stats.rx_len[i]); 1853 i++; 1854 1855 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_H(port->id)); 1856 data = (u64)val << 32; 1857 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_L(port->id)); 1858 data += val; 1859 dev->stats.rx_len[i] = max(data, dev->stats.rx_len[i]); 1860 i++; 1861 1862 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_H(port->id)); 1863 data = (u64)val << 32; 1864 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_L(port->id)); 1865 data += val; 1866 dev->stats.rx_len[i] = max(data, dev->stats.rx_len[i]); 1867 i++; 1868 1869 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_H(port->id)); 1870 data = (u64)val << 32; 1871 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_L(port->id)); 1872 data += val; 1873 dev->stats.rx_len[i] = max(data, dev->stats.rx_len[i]); 1874 i++; 1875 1876 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_H(port->id)); 1877 data = (u64)val << 32; 1878 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_L(port->id)); 1879 data += val; 1880 dev->stats.rx_len[i] = max(data, dev->stats.rx_len[i]); 1881 i++; 1882 1883 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_H(port->id)); 1884 data = (u64)val << 32; 1885 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_L(port->id)); 1886 data += val; 1887 dev->stats.rx_len[i] = max(data, dev->stats.rx_len[i]); 1888 i++; 1889 1890 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_LONG_CNT(port->id)); 1891 dev->stats.rx_len[i] += (u32)(val - dev->stats.mib_prev.rx_long); 1892 dev->stats.mib_prev.rx_long = val; 1893 1894 u64_stats_update_end(&dev->stats.syncp); 1895 1896 spin_unlock(&port->stats_lock); 1897 } 1898 1899 static void airoha_dev_set_xmit_frame_size(struct net_device *netdev) 1900 { 1901 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1902 1903 airoha_ppe_set_xmit_frame_size(dev); 1904 if (!airoha_is_lan_gdm_dev(dev)) 1905 airoha_fe_rmw(dev->eth, REG_WAN_MTU0, WAN_MTU0_MASK, 1906 FIELD_PREP(WAN_MTU0_MASK, 1907 VLAN_ETH_HLEN + netdev->mtu)); 1908 } 1909 1910 static int airoha_dev_open(struct net_device *netdev) 1911 { 1912 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1913 struct airoha_gdm_port *port = dev->port; 1914 u32 pse_port = FE_PSE_PORT_PPE1; 1915 struct airoha_qdma *qdma; 1916 int err; 1917 1918 netif_tx_start_all_queues(netdev); 1919 err = airoha_set_vip_for_gdm_port(dev, true); 1920 if (err) 1921 return err; 1922 1923 qdma = airoha_qdma_deref(dev); 1924 if (netdev_uses_dsa(netdev)) 1925 airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id), 1926 GDM_STAG_EN_MASK); 1927 else 1928 airoha_fe_clear(qdma->eth, REG_GDM_INGRESS_CFG(port->id), 1929 GDM_STAG_EN_MASK); 1930 1931 airoha_dev_set_xmit_frame_size(netdev); 1932 port->users++; 1933 1934 if (!airoha_is_lan_gdm_dev(dev) && 1935 airoha_ppe_is_enabled(qdma->eth, 1)) 1936 pse_port = FE_PSE_PORT_PPE2; 1937 airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id), 1938 pse_port); 1939 1940 return 0; 1941 } 1942 1943 static int airoha_dev_stop(struct net_device *netdev) 1944 { 1945 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1946 struct airoha_gdm_port *port = dev->port; 1947 1948 netif_tx_disable(netdev); 1949 airoha_set_vip_for_gdm_port(dev, false); 1950 1951 if (--port->users) 1952 airoha_ppe_set_xmit_frame_size(dev); 1953 else 1954 airoha_set_gdm_port_fwd_cfg(dev->eth, 1955 REG_GDM_FWD_CFG(port->id), 1956 FE_PSE_PORT_DROP); 1957 return 0; 1958 } 1959 1960 static int airoha_dev_set_macaddr(struct net_device *netdev, void *p) 1961 { 1962 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1963 struct sockaddr *addr = p; 1964 int err; 1965 1966 err = eth_prepare_mac_addr_change(netdev, p); 1967 if (err) 1968 return err; 1969 1970 err = airoha_set_macaddr(dev, addr->sa_data); 1971 if (err) 1972 return err; 1973 1974 eth_commit_mac_addr_change(netdev, p); 1975 1976 return 0; 1977 } 1978 1979 static int airoha_enable_gdm2_loopback(struct airoha_gdm_dev *dev) 1980 { 1981 struct airoha_gdm_port *port = dev->port; 1982 struct airoha_eth *eth = dev->eth; 1983 u32 val, pse_port, chan; 1984 int i, src_port; 1985 1986 src_port = eth->soc->ops.get_sport(port, dev->nbq); 1987 if (src_port < 0) 1988 return src_port; 1989 1990 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1991 FE_PSE_PORT_DROP); 1992 airoha_fe_clear(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1993 GDM_STRIP_CRC_MASK); 1994 1995 /* Enable GDM2 loopback */ 1996 airoha_fe_wr(eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), 0xffffffff); 1997 airoha_fe_wr(eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX), 0xffff); 1998 1999 chan = port->id == AIROHA_GDM3_IDX ? airoha_is_7581(eth) ? 4 : 3 : 0; 2000 airoha_fe_rmw(eth, REG_GDM_LPBK_CFG(AIROHA_GDM2_IDX), 2001 LPBK_CHAN_MASK | LPBK_MODE_MASK | LPBK_EN_MASK, 2002 FIELD_PREP(LPBK_CHAN_MASK, chan) | 2003 LBK_GAP_MODE_MASK | LBK_LEN_MODE_MASK | 2004 LBK_CHAN_MODE_MASK | LPBK_EN_MASK); 2005 /* Forward the traffic to the proper GDM port */ 2006 pse_port = port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3 2007 : FE_PSE_PORT_GDM4; 2008 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 2009 pse_port); 2010 2011 /* Disable VIP and IFC for GDM2 */ 2012 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX)); 2013 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX)); 2014 2015 airoha_fe_rmw(eth, REG_FE_WAN_PORT, 2016 WAN1_EN_MASK | WAN1_MASK | WAN0_MASK, 2017 FIELD_PREP(WAN0_MASK, src_port)); 2018 val = src_port & SP_CPORT_DFT_MASK; 2019 airoha_fe_rmw(eth, 2020 REG_SP_DFT_CPORT(src_port >> fls(SP_CPORT_DFT_MASK)), 2021 SP_CPORT_MASK(val), 2022 __field_prep(SP_CPORT_MASK(val), FE_PSE_PORT_CDM2)); 2023 2024 for (i = 0; i < eth->soc->num_ppe; i++) 2025 airoha_ppe_set_cpu_port(dev, i, AIROHA_GDM2_IDX); 2026 2027 if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) { 2028 u32 mask = FC_ID_OF_SRC_PORT_MASK(dev->nbq); 2029 2030 airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask, 2031 __field_prep(mask, AIROHA_GDM2_IDX)); 2032 } 2033 2034 return 0; 2035 } 2036 2037 static int airoha_disable_gdm2_loopback(struct airoha_gdm_dev *dev) 2038 { 2039 struct airoha_gdm_port *port = dev->port; 2040 struct airoha_eth *eth = dev->eth; 2041 int i, src_port; 2042 u32 pse_port; 2043 2044 src_port = eth->soc->ops.get_sport(dev->port, dev->nbq); 2045 if (src_port < 0) 2046 return src_port; 2047 2048 airoha_fe_clear(eth, 2049 REG_SP_DFT_CPORT(src_port >> fls(SP_CPORT_DFT_MASK)), 2050 SP_CPORT_MASK(src_port & SP_CPORT_DFT_MASK)); 2051 2052 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 2053 GDM_STRIP_CRC_MASK); 2054 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 2055 FE_PSE_PORT_DROP); 2056 airoha_fe_clear(eth, REG_GDM_LPBK_CFG(AIROHA_GDM2_IDX), 2057 LPBK_CHAN_MASK | LPBK_MODE_MASK | LPBK_EN_MASK); 2058 pse_port = airoha_ppe_is_enabled(eth, 1) ? FE_PSE_PORT_PPE2 2059 : FE_PSE_PORT_PPE1; 2060 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 2061 pse_port); 2062 2063 airoha_fe_rmw(eth, REG_FE_WAN_PORT, WAN0_MASK, 2064 FIELD_PREP(WAN0_MASK, AIROHA_GDM2_IDX)); 2065 2066 for (i = 0; i < eth->soc->num_ppe; i++) 2067 airoha_fe_clear(eth, REG_PPE_DFT_CPORT(i, AIROHA_GDM2_IDX), 2068 DFT_CPORT_MASK(AIROHA_GDM2_IDX)); 2069 2070 /* Enable VIP and IFC for GDM2 */ 2071 airoha_fe_set(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX)); 2072 airoha_fe_set(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX)); 2073 2074 if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) { 2075 u32 mask = FC_ID_OF_SRC_PORT_MASK(dev->nbq); 2076 2077 airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask, 2078 FC_MAP6_DEF_VALUE & mask); 2079 } 2080 2081 return 0; 2082 } 2083 2084 static struct airoha_gdm_dev * 2085 airoha_get_wan_gdm_dev(struct airoha_eth *eth) 2086 { 2087 int i; 2088 2089 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 2090 struct airoha_gdm_port *port = eth->ports[i]; 2091 int j; 2092 2093 if (!port) 2094 continue; 2095 2096 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 2097 struct airoha_gdm_dev *dev = port->devs[j]; 2098 2099 if (dev && !airoha_is_lan_gdm_dev(dev)) 2100 return dev; 2101 } 2102 } 2103 2104 return NULL; 2105 } 2106 2107 static void airoha_dev_set_qdma(struct airoha_gdm_dev *dev) 2108 { 2109 struct net_device *netdev = netdev_from_priv(dev); 2110 struct airoha_qdma *cur_qdma, *qdma; 2111 struct airoha_eth *eth = dev->eth; 2112 int i, ppe_id; 2113 2114 /* QDMA0 is used for lan ports while QDMA1 is used for WAN ports */ 2115 qdma = ð->qdma[!airoha_is_lan_gdm_dev(dev)]; 2116 cur_qdma = airoha_qdma_deref(dev); 2117 2118 if (cur_qdma) 2119 netif_tx_stop_all_queues(netdev); 2120 2121 rcu_assign_pointer(dev->qdma, qdma); 2122 netdev->irq = qdma->irq_banks[0].irq; 2123 synchronize_rcu(); 2124 2125 ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1); 2126 airoha_ppe_set_cpu_port(dev, ppe_id, airoha_get_fe_port(dev)); 2127 2128 /* Seed qos_stats baselines with the new QDMA block's current 2129 * counter values to avoid a spurious spike on the first stats 2130 * query after init or migration. 2131 */ 2132 for (i = 0; i < ARRAY_SIZE(dev->qos_stats); i++) { 2133 dev->qos_stats[i].cpu_tx_packets = 2134 airoha_qdma_rr(qdma, REG_CNTR_VAL(i << 1)); 2135 dev->qos_stats[i].fwd_tx_packets = 2136 airoha_qdma_rr(qdma, REG_CNTR_VAL((i << 1) + 1)); 2137 } 2138 2139 if (cur_qdma) 2140 netif_tx_wake_all_queues(netdev); 2141 } 2142 2143 static int airoha_dev_init(struct net_device *netdev) 2144 { 2145 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2146 struct airoha_gdm_port *port = dev->port; 2147 2148 switch (port->id) { 2149 case AIROHA_GDM3_IDX: 2150 case AIROHA_GDM4_IDX: 2151 if (airoha_get_wan_gdm_dev(dev->eth)) 2152 break; 2153 fallthrough; 2154 case AIROHA_GDM2_IDX: 2155 /* GDM2 is always used as wan */ 2156 set_bit(AIROHA_DEV_F_WAN, &dev->flags); 2157 break; 2158 default: 2159 break; 2160 } 2161 2162 airoha_dev_set_qdma(dev); 2163 airoha_set_macaddr(dev, netdev->dev_addr); 2164 2165 if (!airoha_is_lan_gdm_dev(dev) && 2166 (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX)) { 2167 int err; 2168 2169 err = airoha_enable_gdm2_loopback(dev); 2170 if (err) 2171 return err; 2172 } 2173 2174 return 0; 2175 } 2176 2177 static void airoha_dev_get_stats64(struct net_device *netdev, 2178 struct rtnl_link_stats64 *storage) 2179 { 2180 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2181 unsigned int start; 2182 2183 airoha_update_hw_stats(dev); 2184 do { 2185 start = u64_stats_fetch_begin(&dev->stats.syncp); 2186 storage->rx_packets = dev->stats.rx_ok_pkts; 2187 storage->tx_packets = dev->stats.tx_ok_pkts; 2188 storage->rx_bytes = dev->stats.rx_ok_bytes; 2189 storage->tx_bytes = dev->stats.tx_ok_bytes; 2190 storage->multicast = dev->stats.rx_multicast; 2191 storage->rx_errors = dev->stats.rx_errors; 2192 storage->rx_dropped = dev->stats.rx_drops; 2193 storage->tx_dropped = dev->stats.tx_drops; 2194 storage->rx_crc_errors = dev->stats.rx_crc_error; 2195 storage->rx_over_errors = dev->stats.rx_over_errors; 2196 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2197 } 2198 2199 static int airoha_dev_change_mtu(struct net_device *netdev, int mtu) 2200 { 2201 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2202 struct airoha_gdm_port *port = dev->port; 2203 2204 WRITE_ONCE(netdev->mtu, mtu); 2205 if (port->users) 2206 airoha_dev_set_xmit_frame_size(netdev); 2207 2208 return 0; 2209 } 2210 2211 static u16 airoha_dev_select_queue(struct net_device *netdev, 2212 struct sk_buff *skb, 2213 struct net_device *sb_dev) 2214 { 2215 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2216 struct airoha_gdm_port *port = dev->port; 2217 int queue, channel; 2218 2219 /* For dsa device select QoS channel according to the dsa user port 2220 * index, rely on port id otherwise. Select QoS queue based on the 2221 * skb priority. 2222 */ 2223 channel = netdev_uses_dsa(netdev) ? skb_get_queue_mapping(skb) : port->id; 2224 channel = channel % AIROHA_NUM_QOS_CHANNELS; 2225 queue = skb->priority % AIROHA_NUM_QOS_QUEUES; 2226 queue = channel * AIROHA_NUM_QOS_QUEUES + queue; 2227 2228 return queue < netdev->num_tx_queues ? queue : 0; 2229 } 2230 2231 static u32 airoha_get_dsa_tag(struct sk_buff *skb, struct net_device *dev) 2232 { 2233 #if IS_ENABLED(CONFIG_NET_DSA) 2234 struct ethhdr *ehdr; 2235 u8 xmit_tpid; 2236 u16 tag; 2237 2238 if (!netdev_uses_dsa(dev)) 2239 return 0; 2240 2241 if (dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 2242 return 0; 2243 2244 if (skb_cow_head(skb, 0)) 2245 return 0; 2246 2247 ehdr = (struct ethhdr *)skb->data; 2248 tag = be16_to_cpu(ehdr->h_proto); 2249 xmit_tpid = tag >> 8; 2250 2251 switch (xmit_tpid) { 2252 case MTK_HDR_XMIT_TAGGED_TPID_8100: 2253 ehdr->h_proto = cpu_to_be16(ETH_P_8021Q); 2254 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_8100 << 8); 2255 break; 2256 case MTK_HDR_XMIT_TAGGED_TPID_88A8: 2257 ehdr->h_proto = cpu_to_be16(ETH_P_8021AD); 2258 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_88A8 << 8); 2259 break; 2260 default: 2261 /* PPE module requires untagged DSA packets to work properly, 2262 * so move DSA tag to DMA descriptor. 2263 */ 2264 memmove(skb->data + MTK_HDR_LEN, skb->data, 2 * ETH_ALEN); 2265 __skb_pull(skb, MTK_HDR_LEN); 2266 break; 2267 } 2268 2269 return tag; 2270 #else 2271 return 0; 2272 #endif 2273 } 2274 2275 int airoha_get_fe_port(struct airoha_gdm_dev *dev) 2276 { 2277 struct airoha_gdm_port *port = dev->port; 2278 struct airoha_eth *eth = dev->eth; 2279 2280 switch (eth->soc->version) { 2281 case 0x7583: 2282 return port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3 2283 : port->id; 2284 case 0x7581: 2285 default: 2286 return port->id == AIROHA_GDM4_IDX ? FE_PSE_PORT_GDM4 2287 : port->id; 2288 } 2289 } 2290 2291 static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, 2292 struct net_device *netdev) 2293 { 2294 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2295 u32 nr_frags, tag, msg0, msg1, len; 2296 struct airoha_queue_entry *e; 2297 struct airoha_qdma *qdma; 2298 struct netdev_queue *txq; 2299 struct airoha_queue *q; 2300 LIST_HEAD(tx_list); 2301 dma_addr_t addr; 2302 int i = 0, qid; 2303 u16 index; 2304 u8 fport; 2305 2306 rcu_read_lock(); 2307 qdma = rcu_dereference(dev->qdma); 2308 qid = airoha_qdma_get_txq(qdma, skb_get_queue_mapping(skb)); 2309 tag = airoha_get_dsa_tag(skb, netdev); 2310 2311 msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK, 2312 qid / AIROHA_NUM_QOS_QUEUES) | 2313 FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK, 2314 qid % AIROHA_NUM_QOS_QUEUES) | 2315 FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK, tag); 2316 if (skb->ip_summed == CHECKSUM_PARTIAL) 2317 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TCO_MASK, 1) | 2318 FIELD_PREP(QDMA_ETH_TXMSG_UCO_MASK, 1) | 2319 FIELD_PREP(QDMA_ETH_TXMSG_ICO_MASK, 1); 2320 2321 /* TSO: fill MSS info in tcp checksum field */ 2322 if (skb_is_gso(skb)) { 2323 if (skb_cow_head(skb, 0)) 2324 goto error; 2325 2326 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | 2327 SKB_GSO_TCPV6)) { 2328 __be16 csum = cpu_to_be16(skb_shinfo(skb)->gso_size); 2329 2330 tcp_hdr(skb)->check = (__force __sum16)csum; 2331 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TSO_MASK, 1); 2332 } 2333 } 2334 2335 fport = airoha_get_fe_port(dev); 2336 msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) | 2337 FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) | 2338 FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f); 2339 2340 q = &qdma->q_tx[qid]; 2341 if (WARN_ON_ONCE(!q->ndesc)) 2342 goto error; 2343 2344 spin_lock_bh(&q->lock); 2345 2346 if (q->flushing) 2347 goto error_unlock; 2348 2349 txq = skb_get_tx_queue(netdev, skb); 2350 nr_frags = 1 + skb_shinfo(skb)->nr_frags; 2351 2352 if (q->queued + nr_frags >= q->ndesc) { 2353 /* not enough space in the queue */ 2354 netif_tx_stop_queue(txq); 2355 q->txq_stopped = true; 2356 spin_unlock_bh(&q->lock); 2357 rcu_read_unlock(); 2358 2359 return NETDEV_TX_BUSY; 2360 } 2361 2362 e = list_first_entry(&q->tx_list, struct airoha_queue_entry, 2363 list); 2364 len = skb_headlen(skb); 2365 addr = dma_map_single(netdev->dev.parent, skb->data, len, 2366 DMA_TO_DEVICE); 2367 if (unlikely(dma_mapping_error(netdev->dev.parent, addr))) 2368 goto error_unlock; 2369 2370 e->dma_type = AIROHA_DMA_MAP_SINGLE; 2371 index = e - q->entry; 2372 2373 while (true) { 2374 struct airoha_qdma_desc *desc = &q->desc[index]; 2375 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2376 u32 val; 2377 2378 list_move_tail(&e->list, &tx_list); 2379 e->skb = i == nr_frags - 1 ? skb : NULL; 2380 e->dma_addr = addr; 2381 e->dma_len = len; 2382 2383 e = list_first_entry(&q->tx_list, struct airoha_queue_entry, 2384 list); 2385 index = e - q->entry; 2386 2387 val = FIELD_PREP(QDMA_DESC_LEN_MASK, len); 2388 if (i < nr_frags - 1) 2389 val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1); 2390 WRITE_ONCE(desc->ctrl, cpu_to_le32(val)); 2391 WRITE_ONCE(desc->addr, cpu_to_le32(addr)); 2392 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, index); 2393 WRITE_ONCE(desc->data, cpu_to_le32(val)); 2394 WRITE_ONCE(desc->msg0, cpu_to_le32(msg0)); 2395 WRITE_ONCE(desc->msg1, cpu_to_le32(msg1)); 2396 WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff)); 2397 2398 if (++i == nr_frags) 2399 break; 2400 2401 len = skb_frag_size(frag); 2402 addr = skb_frag_dma_map(netdev->dev.parent, frag, 0, len, 2403 DMA_TO_DEVICE); 2404 if (unlikely(dma_mapping_error(netdev->dev.parent, addr))) 2405 goto error_unmap; 2406 2407 e->dma_type = AIROHA_DMA_MAP_PAGE; 2408 } 2409 q->queued += i; 2410 2411 skb_tx_timestamp(skb); 2412 netdev_tx_sent_queue(txq, skb->len); 2413 if (q->ndesc - q->queued < q->free_thr) { 2414 netif_tx_stop_queue(txq); 2415 q->txq_stopped = true; 2416 } 2417 2418 if (netif_xmit_stopped(txq) || !netdev_xmit_more()) 2419 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), 2420 TX_RING_CPU_IDX_MASK, 2421 FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); 2422 2423 spin_unlock_bh(&q->lock); 2424 rcu_read_unlock(); 2425 2426 return NETDEV_TX_OK; 2427 2428 error_unmap: 2429 list_for_each_entry(e, &tx_list, list) 2430 airoha_unmap_xmit_buf(dev->eth, e); 2431 list_splice(&tx_list, &q->tx_list); 2432 error_unlock: 2433 spin_unlock_bh(&q->lock); 2434 error: 2435 dev_kfree_skb_any(skb); 2436 netdev->stats.tx_dropped++; 2437 rcu_read_unlock(); 2438 2439 return NETDEV_TX_OK; 2440 } 2441 2442 static void airoha_ethtool_get_drvinfo(struct net_device *netdev, 2443 struct ethtool_drvinfo *info) 2444 { 2445 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2446 struct airoha_eth *eth = dev->eth; 2447 2448 strscpy(info->driver, eth->dev->driver->name, sizeof(info->driver)); 2449 strscpy(info->bus_info, dev_name(eth->dev), sizeof(info->bus_info)); 2450 } 2451 2452 static void airoha_ethtool_get_mac_stats(struct net_device *netdev, 2453 struct ethtool_eth_mac_stats *stats) 2454 { 2455 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2456 unsigned int start; 2457 2458 airoha_update_hw_stats(dev); 2459 do { 2460 start = u64_stats_fetch_begin(&dev->stats.syncp); 2461 stats->FramesTransmittedOK = dev->stats.tx_ok_pkts; 2462 stats->OctetsTransmittedOK = dev->stats.tx_ok_bytes; 2463 stats->MulticastFramesXmittedOK = dev->stats.tx_multicast; 2464 stats->BroadcastFramesXmittedOK = dev->stats.tx_broadcast; 2465 stats->FramesReceivedOK = dev->stats.rx_ok_pkts; 2466 stats->OctetsReceivedOK = dev->stats.rx_ok_bytes; 2467 stats->BroadcastFramesReceivedOK = dev->stats.rx_broadcast; 2468 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2469 } 2470 2471 static const struct ethtool_rmon_hist_range airoha_ethtool_rmon_ranges[] = { 2472 { 0, 64 }, 2473 { 65, 127 }, 2474 { 128, 255 }, 2475 { 256, 511 }, 2476 { 512, 1023 }, 2477 { 1024, 1518 }, 2478 { 1519, 10239 }, 2479 {}, 2480 }; 2481 2482 static void 2483 airoha_ethtool_get_rmon_stats(struct net_device *netdev, 2484 struct ethtool_rmon_stats *stats, 2485 const struct ethtool_rmon_hist_range **ranges) 2486 { 2487 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2488 struct airoha_hw_stats *hw_stats = &dev->stats; 2489 unsigned int start; 2490 2491 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) != 2492 ARRAY_SIZE(hw_stats->tx_len) + 1); 2493 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) != 2494 ARRAY_SIZE(hw_stats->rx_len) + 1); 2495 2496 *ranges = airoha_ethtool_rmon_ranges; 2497 airoha_update_hw_stats(dev); 2498 do { 2499 int i; 2500 2501 start = u64_stats_fetch_begin(&dev->stats.syncp); 2502 stats->fragments = hw_stats->rx_fragment; 2503 stats->jabbers = hw_stats->rx_jabber; 2504 for (i = 0; i < ARRAY_SIZE(airoha_ethtool_rmon_ranges) - 1; 2505 i++) { 2506 stats->hist[i] = hw_stats->rx_len[i]; 2507 stats->hist_tx[i] = hw_stats->tx_len[i]; 2508 } 2509 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2510 } 2511 2512 static int airoha_qdma_set_chan_tx_sched(struct net_device *netdev, 2513 int channel, enum tx_sched_mode mode, 2514 const u16 *weights, u8 n_weights) 2515 { 2516 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2517 struct airoha_qdma *qdma; 2518 int i; 2519 2520 qdma = airoha_qdma_deref(dev); 2521 for (i = 0; i < AIROHA_NUM_QOS_QUEUES; i++) 2522 airoha_qdma_clear(qdma, REG_QUEUE_CLOSE_CFG(channel), 2523 TXQ_DISABLE_CHAN_QUEUE_MASK(channel, i)); 2524 2525 for (i = 0; i < n_weights; i++) { 2526 u32 status; 2527 int err; 2528 2529 airoha_qdma_wr(qdma, REG_TXWRR_WEIGHT_CFG, 2530 TWRR_RW_CMD_MASK | 2531 FIELD_PREP(TWRR_CHAN_IDX_MASK, channel) | 2532 FIELD_PREP(TWRR_QUEUE_IDX_MASK, i) | 2533 FIELD_PREP(TWRR_VALUE_MASK, weights[i])); 2534 err = read_poll_timeout(airoha_qdma_rr, status, 2535 status & TWRR_RW_CMD_DONE, 2536 USEC_PER_MSEC, 10 * USEC_PER_MSEC, 2537 true, qdma, REG_TXWRR_WEIGHT_CFG); 2538 if (err) 2539 return err; 2540 } 2541 2542 airoha_qdma_rmw(qdma, REG_CHAN_QOS_MODE(channel >> 3), 2543 CHAN_QOS_MODE_MASK(channel), 2544 __field_prep(CHAN_QOS_MODE_MASK(channel), mode)); 2545 2546 return 0; 2547 } 2548 2549 static int airoha_qdma_set_tx_prio_sched(struct net_device *dev, int channel) 2550 { 2551 static const u16 w[AIROHA_NUM_QOS_QUEUES] = {}; 2552 2553 return airoha_qdma_set_chan_tx_sched(dev, channel, TC_SCH_SP, w, 2554 ARRAY_SIZE(w)); 2555 } 2556 2557 static int airoha_qdma_set_tx_ets_sched(struct net_device *dev, int channel, 2558 struct tc_ets_qopt_offload *opt) 2559 { 2560 struct tc_ets_qopt_offload_replace_params *p = &opt->replace_params; 2561 enum tx_sched_mode mode = TC_SCH_SP; 2562 u16 w[AIROHA_NUM_QOS_QUEUES] = {}; 2563 int i, nstrict = 0; 2564 2565 if (p->bands > AIROHA_NUM_QOS_QUEUES) 2566 return -EINVAL; 2567 2568 for (i = 0; i < p->bands; i++) { 2569 if (!p->quanta[i]) 2570 nstrict++; 2571 } 2572 2573 /* this configuration is not supported by the hw */ 2574 if (nstrict == AIROHA_NUM_QOS_QUEUES - 1) 2575 return -EINVAL; 2576 2577 /* EN7581 SoC supports fixed QoS band priority where WRR queues have 2578 * lowest priorities with respect to SP ones. 2579 * e.g: WRR0, WRR1, .., WRRm, SP0, SP1, .., SPn 2580 */ 2581 for (i = 0; i < nstrict; i++) { 2582 if (p->priomap[p->bands - i - 1] != i) 2583 return -EINVAL; 2584 } 2585 2586 for (i = 0; i < p->bands - nstrict; i++) { 2587 if (p->priomap[i] != nstrict + i) 2588 return -EINVAL; 2589 2590 w[i] = p->weights[nstrict + i]; 2591 } 2592 2593 if (!nstrict) 2594 mode = TC_SCH_WRR8; 2595 else if (nstrict < AIROHA_NUM_QOS_QUEUES - 1) 2596 mode = nstrict + 1; 2597 2598 return airoha_qdma_set_chan_tx_sched(dev, channel, mode, w, 2599 ARRAY_SIZE(w)); 2600 } 2601 2602 static int airoha_qdma_get_tx_ets_stats(struct net_device *netdev, int channel, 2603 struct tc_ets_qopt_offload *opt) 2604 { 2605 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2606 u32 cpu_tx_packets, fwd_tx_packets; 2607 struct airoha_qdma *qdma; 2608 u64 tx_packets; 2609 2610 qdma = airoha_qdma_deref(dev); 2611 cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1)); 2612 fwd_tx_packets = airoha_qdma_rr(qdma, 2613 REG_CNTR_VAL((channel << 1) + 1)); 2614 tx_packets = (u32)(cpu_tx_packets - 2615 dev->qos_stats[channel].cpu_tx_packets); 2616 tx_packets += (u32)(fwd_tx_packets - 2617 dev->qos_stats[channel].fwd_tx_packets); 2618 2619 _bstats_update(opt->stats.bstats, 0, tx_packets); 2620 dev->qos_stats[channel].cpu_tx_packets = cpu_tx_packets; 2621 dev->qos_stats[channel].fwd_tx_packets = fwd_tx_packets; 2622 2623 return 0; 2624 } 2625 2626 static int airoha_tc_setup_qdisc_ets(struct net_device *dev, 2627 struct tc_ets_qopt_offload *opt) 2628 { 2629 int channel; 2630 2631 if (opt->parent == TC_H_ROOT) 2632 return -EINVAL; 2633 2634 channel = TC_H_MIN(opt->parent) % AIROHA_NUM_QOS_CHANNELS; 2635 2636 switch (opt->command) { 2637 case TC_ETS_REPLACE: 2638 return airoha_qdma_set_tx_ets_sched(dev, channel, opt); 2639 case TC_ETS_DESTROY: 2640 /* PRIO is default qdisc scheduler */ 2641 return airoha_qdma_set_tx_prio_sched(dev, channel); 2642 case TC_ETS_STATS: 2643 return airoha_qdma_get_tx_ets_stats(dev, channel, opt); 2644 default: 2645 return -EOPNOTSUPP; 2646 } 2647 } 2648 2649 static int airoha_qdma_get_rl_param(struct airoha_qdma *qdma, int queue_id, 2650 u32 addr, enum trtcm_param_type param, 2651 u32 *val_low, u32 *val_high) 2652 { 2653 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id); 2654 u32 val, config = FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) | 2655 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) | 2656 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx); 2657 2658 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2659 if (read_poll_timeout(airoha_qdma_rr, val, 2660 val & RATE_LIMIT_PARAM_RW_DONE_MASK, 2661 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, qdma, 2662 REG_TRTCM_CFG_PARAM(addr))) 2663 return -ETIMEDOUT; 2664 2665 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr)); 2666 if (val_high) 2667 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr)); 2668 2669 return 0; 2670 } 2671 2672 static int airoha_qdma_set_rl_param(struct airoha_qdma *qdma, int queue_id, 2673 u32 addr, enum trtcm_param_type param, 2674 u32 val) 2675 { 2676 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id); 2677 u32 config = RATE_LIMIT_PARAM_RW_MASK | 2678 FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) | 2679 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) | 2680 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx); 2681 2682 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); 2683 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2684 2685 return read_poll_timeout(airoha_qdma_rr, val, 2686 val & RATE_LIMIT_PARAM_RW_DONE_MASK, 2687 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2688 qdma, REG_TRTCM_CFG_PARAM(addr)); 2689 } 2690 2691 static int airoha_qdma_set_rl_config(struct airoha_qdma *qdma, int queue_id, 2692 u32 addr, bool enable, u32 enable_mask) 2693 { 2694 u32 val; 2695 int err; 2696 2697 err = airoha_qdma_get_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE, 2698 &val, NULL); 2699 if (err) 2700 return err; 2701 2702 val = enable ? val | enable_mask : val & ~enable_mask; 2703 2704 return airoha_qdma_set_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE, 2705 val); 2706 } 2707 2708 static int airoha_qdma_set_rl_token_bucket(struct airoha_qdma *qdma, 2709 int queue_id, u32 rate_val, 2710 u32 bucket_size) 2711 { 2712 u32 val, config, tick, unit, rate, rate_frac; 2713 int err; 2714 2715 err = airoha_qdma_get_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2716 TRTCM_MISC_MODE, &config, NULL); 2717 if (err) 2718 return err; 2719 2720 val = airoha_qdma_rr(qdma, REG_INGRESS_TRTCM_CFG); 2721 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val); 2722 if (config & TRTCM_TICK_SEL) 2723 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val); 2724 if (!tick) 2725 return -EINVAL; 2726 2727 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick; 2728 if (!unit) 2729 return -EINVAL; 2730 2731 rate = rate_val / unit; 2732 rate_frac = rate_val % unit; 2733 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit; 2734 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) | 2735 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac); 2736 2737 err = airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2738 TRTCM_TOKEN_RATE_MODE, rate); 2739 if (err) 2740 return err; 2741 2742 val = bucket_size; 2743 if (!(config & TRTCM_PKT_MODE)) 2744 val = max_t(u32, val, MIN_TOKEN_SIZE); 2745 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET); 2746 2747 return airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2748 TRTCM_BUCKETSIZE_SHIFT_MODE, val); 2749 } 2750 2751 static int airoha_qdma_init_rl_config(struct airoha_qdma *qdma, int queue_id, 2752 bool enable, enum trtcm_unit_type unit) 2753 { 2754 bool tick_sel = queue_id == 0 || queue_id == 2 || queue_id == 8; 2755 enum trtcm_param mode = TRTCM_METER_MODE; 2756 int err; 2757 2758 mode |= unit == TRTCM_PACKET_UNIT ? TRTCM_PKT_MODE : 0; 2759 err = airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2760 enable, mode); 2761 if (err) 2762 return err; 2763 2764 return airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2765 tick_sel, TRTCM_TICK_SEL); 2766 } 2767 2768 static int airoha_qdma_get_trtcm_param(struct airoha_qdma *qdma, int channel, 2769 u32 addr, enum trtcm_param_type param, 2770 enum trtcm_mode_type mode, 2771 u32 *val_low, u32 *val_high) 2772 { 2773 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel); 2774 u32 val, config = FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) | 2775 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) | 2776 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) | 2777 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode); 2778 2779 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2780 if (read_poll_timeout(airoha_qdma_rr, val, 2781 val & TRTCM_PARAM_RW_DONE_MASK, 2782 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2783 qdma, REG_TRTCM_CFG_PARAM(addr))) 2784 return -ETIMEDOUT; 2785 2786 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr)); 2787 if (val_high) 2788 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr)); 2789 2790 return 0; 2791 } 2792 2793 static int airoha_qdma_set_trtcm_param(struct airoha_qdma *qdma, int channel, 2794 u32 addr, enum trtcm_param_type param, 2795 enum trtcm_mode_type mode, u32 val) 2796 { 2797 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel); 2798 u32 config = TRTCM_PARAM_RW_MASK | 2799 FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) | 2800 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) | 2801 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) | 2802 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode); 2803 2804 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); 2805 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2806 2807 return read_poll_timeout(airoha_qdma_rr, val, 2808 val & TRTCM_PARAM_RW_DONE_MASK, 2809 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2810 qdma, REG_TRTCM_CFG_PARAM(addr)); 2811 } 2812 2813 static int airoha_qdma_set_trtcm_config(struct airoha_qdma *qdma, int channel, 2814 u32 addr, enum trtcm_mode_type mode, 2815 bool enable, u32 enable_mask) 2816 { 2817 u32 val; 2818 2819 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2820 mode, &val, NULL)) 2821 return -EINVAL; 2822 2823 val = enable ? val | enable_mask : val & ~enable_mask; 2824 2825 return airoha_qdma_set_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2826 mode, val); 2827 } 2828 2829 static int airoha_qdma_set_trtcm_token_bucket(struct airoha_qdma *qdma, 2830 int channel, u32 addr, 2831 enum trtcm_mode_type mode, 2832 u32 rate_val, u32 bucket_size) 2833 { 2834 u32 val, config, tick, unit, rate, rate_frac; 2835 int err; 2836 2837 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2838 mode, &config, NULL)) 2839 return -EINVAL; 2840 2841 val = airoha_qdma_rr(qdma, addr); 2842 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val); 2843 if (config & TRTCM_TICK_SEL) 2844 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val); 2845 if (!tick) 2846 return -EINVAL; 2847 2848 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick; 2849 if (!unit) 2850 return -EINVAL; 2851 2852 rate = rate_val / unit; 2853 rate_frac = rate_val % unit; 2854 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit; 2855 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) | 2856 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac); 2857 2858 err = airoha_qdma_set_trtcm_param(qdma, channel, addr, 2859 TRTCM_TOKEN_RATE_MODE, mode, rate); 2860 if (err) 2861 return err; 2862 2863 val = max_t(u32, bucket_size, MIN_TOKEN_SIZE); 2864 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET); 2865 2866 return airoha_qdma_set_trtcm_param(qdma, channel, addr, 2867 TRTCM_BUCKETSIZE_SHIFT_MODE, 2868 mode, val); 2869 } 2870 2871 static int airoha_qdma_set_tx_rate_limit(struct net_device *netdev, 2872 int channel, u32 rate, 2873 u32 bucket_size) 2874 { 2875 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2876 struct airoha_qdma *qdma; 2877 int i, err; 2878 2879 qdma = airoha_qdma_deref(dev); 2880 for (i = 0; i <= TRTCM_PEAK_MODE; i++) { 2881 err = airoha_qdma_set_trtcm_config(qdma, channel, 2882 REG_EGRESS_TRTCM_CFG, i, 2883 !!rate, TRTCM_METER_MODE); 2884 if (err) 2885 return err; 2886 2887 err = airoha_qdma_set_trtcm_token_bucket(qdma, channel, 2888 REG_EGRESS_TRTCM_CFG, 2889 i, rate, bucket_size); 2890 if (err) 2891 return err; 2892 } 2893 2894 return 0; 2895 } 2896 2897 static int airoha_tc_htb_modify_queue(struct net_device *dev, 2898 struct tc_htb_qopt_offload *opt) 2899 { 2900 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 2901 u32 rate = div_u64(opt->rate, 1000) << 3; /* kbps */ 2902 int err; 2903 2904 if (opt->parent_classid != TC_HTB_CLASSID_ROOT) { 2905 NL_SET_ERR_MSG_MOD(opt->extack, "invalid parent classid"); 2906 return -EINVAL; 2907 } 2908 2909 err = airoha_qdma_set_tx_rate_limit(dev, channel, rate, opt->quantum); 2910 if (err) 2911 NL_SET_ERR_MSG_MOD(opt->extack, 2912 "failed configuring htb offload"); 2913 2914 return err; 2915 } 2916 2917 static int airoha_tc_htb_alloc_leaf_queue(struct net_device *netdev, 2918 struct tc_htb_qopt_offload *opt) 2919 { 2920 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 2921 int err, num_tx_queues = AIROHA_NUM_TX_RING + channel + 1; 2922 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2923 struct airoha_qdma *qdma; 2924 2925 /* Here we need to check the requested QDMA channel is not already 2926 * in use by another net_device running on the same QDMA block. 2927 */ 2928 qdma = airoha_qdma_deref(dev); 2929 if (test_and_set_bit(channel, qdma->qos_channel_map)) { 2930 NL_SET_ERR_MSG_MOD(opt->extack, 2931 "qdma qos channel already in use"); 2932 return -EBUSY; 2933 } 2934 2935 err = airoha_tc_htb_modify_queue(netdev, opt); 2936 if (err) 2937 goto error; 2938 2939 if (num_tx_queues > netdev->real_num_tx_queues) { 2940 err = netif_set_real_num_tx_queues(netdev, num_tx_queues); 2941 if (err) { 2942 airoha_qdma_set_tx_rate_limit(netdev, channel, 0, 2943 opt->quantum); 2944 NL_SET_ERR_MSG_MOD(opt->extack, 2945 "failed setting real_num_tx_queues"); 2946 goto error; 2947 } 2948 } 2949 2950 set_bit(channel, dev->qos_sq_bmap); 2951 opt->qid = AIROHA_NUM_TX_RING + channel; 2952 2953 return 0; 2954 error: 2955 clear_bit(channel, qdma->qos_channel_map); 2956 2957 return err; 2958 } 2959 2960 static int airoha_qdma_set_rx_meter(struct airoha_gdm_dev *dev, 2961 u32 rate, u32 bucket_size, 2962 enum trtcm_unit_type unit_type) 2963 { 2964 struct airoha_qdma *qdma = airoha_qdma_deref(dev); 2965 int i; 2966 2967 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 2968 int err; 2969 2970 if (!qdma->q_rx[i].ndesc) 2971 continue; 2972 2973 err = airoha_qdma_init_rl_config(qdma, i, !!rate, unit_type); 2974 if (err) 2975 return err; 2976 2977 err = airoha_qdma_set_rl_token_bucket(qdma, i, rate, 2978 bucket_size); 2979 if (err) 2980 return err; 2981 } 2982 2983 return 0; 2984 } 2985 2986 static int airoha_tc_matchall_act_validate(struct tc_cls_matchall_offload *f) 2987 { 2988 const struct flow_action *actions = &f->rule->action; 2989 const struct flow_action_entry *act; 2990 2991 if (!flow_action_has_entries(actions)) { 2992 NL_SET_ERR_MSG_MOD(f->common.extack, 2993 "filter run with no actions"); 2994 return -EINVAL; 2995 } 2996 2997 if (!flow_offload_has_one_action(actions)) { 2998 NL_SET_ERR_MSG_MOD(f->common.extack, 2999 "only once action per filter is supported"); 3000 return -EOPNOTSUPP; 3001 } 3002 3003 act = &actions->entries[0]; 3004 if (act->id != FLOW_ACTION_POLICE) { 3005 NL_SET_ERR_MSG_MOD(f->common.extack, "unsupported action"); 3006 return -EOPNOTSUPP; 3007 } 3008 3009 if (act->police.exceed.act_id != FLOW_ACTION_DROP) { 3010 NL_SET_ERR_MSG_MOD(f->common.extack, 3011 "invalid exceed action id"); 3012 return -EOPNOTSUPP; 3013 } 3014 3015 if (act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { 3016 NL_SET_ERR_MSG_MOD(f->common.extack, 3017 "invalid notexceed action id"); 3018 return -EOPNOTSUPP; 3019 } 3020 3021 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && 3022 !flow_action_is_last_entry(actions, act)) { 3023 NL_SET_ERR_MSG_MOD(f->common.extack, 3024 "action accept must be last"); 3025 return -EOPNOTSUPP; 3026 } 3027 3028 if (act->police.peakrate_bytes_ps || act->police.avrate || 3029 act->police.overhead || act->police.mtu) { 3030 NL_SET_ERR_MSG_MOD(f->common.extack, 3031 "peakrate/avrate/overhead/mtu unsupported"); 3032 return -EOPNOTSUPP; 3033 } 3034 3035 return 0; 3036 } 3037 3038 static int airoha_dev_tc_matchall(struct net_device *netdev, 3039 struct tc_cls_matchall_offload *f) 3040 { 3041 enum trtcm_unit_type unit_type = TRTCM_BYTE_UNIT; 3042 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3043 u32 rate = 0, bucket_size = 0; 3044 3045 switch (f->command) { 3046 case TC_CLSMATCHALL_REPLACE: { 3047 const struct flow_action_entry *act; 3048 int err; 3049 3050 err = airoha_tc_matchall_act_validate(f); 3051 if (err) 3052 return err; 3053 3054 act = &f->rule->action.entries[0]; 3055 if (act->police.rate_pkt_ps) { 3056 rate = act->police.rate_pkt_ps; 3057 bucket_size = act->police.burst_pkt; 3058 unit_type = TRTCM_PACKET_UNIT; 3059 } else { 3060 rate = div_u64(act->police.rate_bytes_ps, 1000); 3061 rate = rate << 3; /* Kbps */ 3062 bucket_size = act->police.burst; 3063 } 3064 fallthrough; 3065 } 3066 case TC_CLSMATCHALL_DESTROY: 3067 return airoha_qdma_set_rx_meter(dev, rate, bucket_size, 3068 unit_type); 3069 default: 3070 return -EOPNOTSUPP; 3071 } 3072 } 3073 3074 static int airoha_dev_setup_tc_block_cb(enum tc_setup_type type, 3075 void *type_data, void *cb_priv) 3076 { 3077 struct net_device *netdev = cb_priv; 3078 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3079 struct airoha_eth *eth = dev->eth; 3080 3081 if (!tc_can_offload(netdev)) 3082 return -EOPNOTSUPP; 3083 3084 switch (type) { 3085 case TC_SETUP_CLSFLOWER: 3086 return airoha_ppe_setup_tc_block_cb(ð->ppe->dev, type_data); 3087 case TC_SETUP_CLSMATCHALL: 3088 return airoha_dev_tc_matchall(netdev, type_data); 3089 default: 3090 return -EOPNOTSUPP; 3091 } 3092 } 3093 3094 static int airoha_dev_setup_tc_block(struct net_device *dev, 3095 struct flow_block_offload *f) 3096 { 3097 flow_setup_cb_t *cb = airoha_dev_setup_tc_block_cb; 3098 static LIST_HEAD(block_cb_list); 3099 struct flow_block_cb *block_cb; 3100 3101 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 3102 return -EOPNOTSUPP; 3103 3104 f->driver_block_list = &block_cb_list; 3105 switch (f->command) { 3106 case FLOW_BLOCK_BIND: 3107 block_cb = flow_block_cb_lookup(f->block, cb, dev); 3108 if (block_cb) { 3109 flow_block_cb_incref(block_cb); 3110 return 0; 3111 } 3112 block_cb = flow_block_cb_alloc(cb, dev, dev, NULL); 3113 if (IS_ERR(block_cb)) 3114 return PTR_ERR(block_cb); 3115 3116 flow_block_cb_incref(block_cb); 3117 flow_block_cb_add(block_cb, f); 3118 list_add_tail(&block_cb->driver_list, &block_cb_list); 3119 return 0; 3120 case FLOW_BLOCK_UNBIND: 3121 block_cb = flow_block_cb_lookup(f->block, cb, dev); 3122 if (!block_cb) 3123 return -ENOENT; 3124 3125 if (!flow_block_cb_decref(block_cb)) { 3126 flow_block_cb_remove(block_cb, f); 3127 list_del(&block_cb->driver_list); 3128 } 3129 return 0; 3130 default: 3131 return -EOPNOTSUPP; 3132 } 3133 } 3134 3135 static void airoha_tc_remove_htb_queue(struct net_device *netdev, int queue) 3136 { 3137 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3138 int num_tx_queues = AIROHA_NUM_TX_RING; 3139 struct airoha_qdma *qdma; 3140 3141 airoha_qdma_set_tx_rate_limit(netdev, queue, 0, 0); 3142 3143 qdma = airoha_qdma_deref(dev); 3144 clear_bit(queue, qdma->qos_channel_map); 3145 clear_bit(queue, dev->qos_sq_bmap); 3146 3147 if (!bitmap_empty(dev->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS)) 3148 num_tx_queues += find_last_bit(dev->qos_sq_bmap, 3149 AIROHA_NUM_QOS_CHANNELS) + 1; 3150 netif_set_real_num_tx_queues(netdev, num_tx_queues); 3151 } 3152 3153 static int airoha_tc_htb_delete_leaf_queue(struct net_device *netdev, 3154 struct tc_htb_qopt_offload *opt) 3155 { 3156 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 3157 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3158 3159 if (!test_bit(channel, dev->qos_sq_bmap)) { 3160 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); 3161 return -EINVAL; 3162 } 3163 3164 airoha_tc_remove_htb_queue(netdev, channel); 3165 3166 return 0; 3167 } 3168 3169 static void airoha_disable_qos_for_gdm34(struct net_device *netdev) 3170 { 3171 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3172 struct airoha_gdm_port *port = dev->port; 3173 int err; 3174 3175 if (port->id != AIROHA_GDM3_IDX && 3176 port->id != AIROHA_GDM4_IDX) 3177 return; 3178 3179 err = airoha_disable_gdm2_loopback(dev); 3180 if (err) 3181 netdev_warn(netdev, 3182 "failed disabling GDM2 loopback: %d\n", err); 3183 3184 clear_bit(AIROHA_DEV_F_WAN, &dev->flags); 3185 airoha_dev_set_qdma(dev); 3186 3187 airoha_set_macaddr(dev, netdev->dev_addr); 3188 airoha_ppe_set_xmit_frame_size(dev); 3189 3190 if (netif_running(netdev)) 3191 airoha_set_gdm_port_fwd_cfg(dev->eth, 3192 REG_GDM_FWD_CFG(port->id), 3193 FE_PSE_PORT_PPE1); 3194 } 3195 3196 static int airoha_enable_qos_for_gdm34(struct net_device *netdev, 3197 struct netlink_ext_ack *extack) 3198 { 3199 struct airoha_gdm_dev *wan_dev, *dev = netdev_priv(netdev); 3200 struct airoha_gdm_port *port = dev->port; 3201 struct airoha_eth *eth = dev->eth; 3202 int err = -EBUSY; 3203 3204 if (port->id != AIROHA_GDM3_IDX && 3205 port->id != AIROHA_GDM4_IDX) { 3206 /* HW QoS is always supported by GDM1 and GDM2 */ 3207 return 0; 3208 } 3209 3210 if (!airoha_is_lan_gdm_dev(dev)) /* Already enabled */ 3211 return 0; 3212 3213 mutex_lock(&flow_offload_mutex); 3214 3215 wan_dev = airoha_get_wan_gdm_dev(eth); 3216 if (wan_dev) { 3217 if (test_bit(AIROHA_DEV_F_TX_QOS, &wan_dev->flags) || 3218 wan_dev->port->id == AIROHA_GDM2_IDX) { 3219 NL_SET_ERR_MSG_MOD(extack, 3220 "QoS configured for WAN device"); 3221 goto error_unlock; 3222 } 3223 airoha_disable_qos_for_gdm34(netdev_from_priv(wan_dev)); 3224 } 3225 3226 set_bit(AIROHA_DEV_F_WAN, &dev->flags); 3227 airoha_dev_set_qdma(dev); 3228 err = airoha_enable_gdm2_loopback(dev); 3229 if (err) 3230 goto error_disable_wan; 3231 3232 err = airoha_set_macaddr(dev, netdev->dev_addr); 3233 if (err) 3234 goto error_disable_loopback; 3235 3236 airoha_dev_set_xmit_frame_size(netdev); 3237 if (netif_running(netdev)) { 3238 u32 pse_port; 3239 3240 pse_port = airoha_ppe_is_enabled(eth, 1) ? FE_PSE_PORT_PPE2 3241 : FE_PSE_PORT_PPE1; 3242 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(port->id), 3243 pse_port); 3244 } 3245 3246 mutex_unlock(&flow_offload_mutex); 3247 3248 return 0; 3249 3250 error_disable_loopback: 3251 airoha_disable_gdm2_loopback(dev); 3252 error_disable_wan: 3253 clear_bit(AIROHA_DEV_F_WAN, &dev->flags); 3254 airoha_dev_set_qdma(dev); 3255 error_unlock: 3256 mutex_unlock(&flow_offload_mutex); 3257 3258 return err; 3259 } 3260 3261 static int airoha_tc_htb_destroy(struct net_device *netdev) 3262 { 3263 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3264 int q; 3265 3266 for_each_set_bit(q, dev->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS) 3267 airoha_tc_remove_htb_queue(netdev, q); 3268 3269 clear_bit(AIROHA_DEV_F_TX_QOS, &dev->flags); 3270 3271 return 0; 3272 } 3273 3274 static int airoha_tc_get_htb_get_leaf_queue(struct net_device *netdev, 3275 struct tc_htb_qopt_offload *opt) 3276 { 3277 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 3278 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3279 3280 if (!test_bit(channel, dev->qos_sq_bmap)) { 3281 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); 3282 return -EINVAL; 3283 } 3284 3285 opt->qid = AIROHA_NUM_TX_RING + channel; 3286 3287 return 0; 3288 } 3289 3290 static int airoha_tc_setup_qdisc_htb(struct net_device *netdev, 3291 struct tc_htb_qopt_offload *opt) 3292 { 3293 switch (opt->command) { 3294 case TC_HTB_CREATE: { 3295 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3296 int err; 3297 3298 err = airoha_enable_qos_for_gdm34(netdev, opt->extack); 3299 if (err) 3300 return err; 3301 3302 set_bit(AIROHA_DEV_F_TX_QOS, &dev->flags); 3303 break; 3304 } 3305 case TC_HTB_DESTROY: 3306 return airoha_tc_htb_destroy(netdev); 3307 case TC_HTB_NODE_MODIFY: 3308 return airoha_tc_htb_modify_queue(netdev, opt); 3309 case TC_HTB_LEAF_ALLOC_QUEUE: 3310 return airoha_tc_htb_alloc_leaf_queue(netdev, opt); 3311 case TC_HTB_LEAF_DEL: 3312 case TC_HTB_LEAF_DEL_LAST: 3313 case TC_HTB_LEAF_DEL_LAST_FORCE: 3314 return airoha_tc_htb_delete_leaf_queue(netdev, opt); 3315 case TC_HTB_LEAF_QUERY_QUEUE: 3316 return airoha_tc_get_htb_get_leaf_queue(netdev, opt); 3317 default: 3318 return -EOPNOTSUPP; 3319 } 3320 3321 return 0; 3322 } 3323 3324 static int airoha_dev_tc_setup(struct net_device *dev, 3325 enum tc_setup_type type, void *type_data) 3326 { 3327 switch (type) { 3328 case TC_SETUP_QDISC_ETS: 3329 return airoha_tc_setup_qdisc_ets(dev, type_data); 3330 case TC_SETUP_QDISC_HTB: 3331 return airoha_tc_setup_qdisc_htb(dev, type_data); 3332 case TC_SETUP_BLOCK: 3333 case TC_SETUP_FT: 3334 return airoha_dev_setup_tc_block(dev, type_data); 3335 default: 3336 return -EOPNOTSUPP; 3337 } 3338 } 3339 3340 static const struct net_device_ops airoha_netdev_ops = { 3341 .ndo_init = airoha_dev_init, 3342 .ndo_open = airoha_dev_open, 3343 .ndo_stop = airoha_dev_stop, 3344 .ndo_change_mtu = airoha_dev_change_mtu, 3345 .ndo_select_queue = airoha_dev_select_queue, 3346 .ndo_start_xmit = airoha_dev_xmit, 3347 .ndo_get_stats64 = airoha_dev_get_stats64, 3348 .ndo_set_mac_address = airoha_dev_set_macaddr, 3349 .ndo_setup_tc = airoha_dev_tc_setup, 3350 }; 3351 3352 static const struct ethtool_ops airoha_ethtool_ops = { 3353 .get_drvinfo = airoha_ethtool_get_drvinfo, 3354 .get_eth_mac_stats = airoha_ethtool_get_mac_stats, 3355 .get_rmon_stats = airoha_ethtool_get_rmon_stats, 3356 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3357 .get_link = ethtool_op_get_link, 3358 }; 3359 3360 static int airoha_metadata_dst_alloc(struct airoha_gdm_port *port) 3361 { 3362 int i; 3363 3364 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) { 3365 struct metadata_dst *md_dst; 3366 3367 md_dst = metadata_dst_alloc(0, METADATA_HW_PORT_MUX, 3368 GFP_KERNEL); 3369 if (!md_dst) 3370 return -ENOMEM; 3371 3372 md_dst->u.port_info.port_id = i; 3373 port->dsa_meta[i] = md_dst; 3374 } 3375 3376 return 0; 3377 } 3378 3379 static void airoha_metadata_dst_free(struct airoha_gdm_port *port) 3380 { 3381 int i; 3382 3383 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) { 3384 if (!port->dsa_meta[i]) 3385 continue; 3386 3387 dst_release(&port->dsa_meta[i]->dst); 3388 } 3389 } 3390 3391 bool airoha_is_valid_gdm_dev(struct airoha_eth *eth, 3392 struct airoha_gdm_dev *dev) 3393 { 3394 int i; 3395 3396 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3397 struct airoha_gdm_port *port = eth->ports[i]; 3398 int j; 3399 3400 if (!port) 3401 continue; 3402 3403 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3404 if (port->devs[j] == dev) 3405 return true; 3406 } 3407 } 3408 3409 return false; 3410 } 3411 3412 static int airoha_alloc_gdm_device(struct airoha_eth *eth, 3413 struct airoha_gdm_port *port, 3414 int nbq, struct device_node *np) 3415 { 3416 struct net_device *netdev; 3417 struct airoha_gdm_dev *dev; 3418 int err, i; 3419 u8 index; 3420 3421 netdev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*dev), 3422 AIROHA_NUM_NETDEV_TX_RINGS, 3423 AIROHA_NUM_RX_RING); 3424 if (!netdev) { 3425 dev_err(eth->dev, "alloc_etherdev failed\n"); 3426 return -ENOMEM; 3427 } 3428 3429 netdev->netdev_ops = &airoha_netdev_ops; 3430 netdev->ethtool_ops = &airoha_ethtool_ops; 3431 netdev->max_mtu = AIROHA_MAX_MTU; 3432 netdev->watchdog_timeo = 5 * HZ; 3433 netdev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_TSO6 | 3434 NETIF_F_IPV6_CSUM | NETIF_F_SG | NETIF_F_TSO | 3435 NETIF_F_HW_TC; 3436 netdev->features |= netdev->hw_features; 3437 netdev->vlan_features = netdev->hw_features; 3438 SET_NETDEV_DEV(netdev, eth->dev); 3439 3440 /* reserve hw queues for HTB offloading */ 3441 err = netif_set_real_num_tx_queues(netdev, AIROHA_NUM_TX_RING); 3442 if (err) 3443 return err; 3444 3445 err = of_get_ethdev_address(np, netdev); 3446 if (err) { 3447 if (err == -EPROBE_DEFER) 3448 return err; 3449 3450 eth_hw_addr_random(netdev); 3451 dev_info(eth->dev, "generated random MAC address %pM\n", 3452 netdev->dev_addr); 3453 } 3454 3455 /* Allowed nbq for EN7581 on GDM3 port are 4 and 5 for PCIE0 3456 * and PCIE1 respectively. 3457 */ 3458 index = nbq; 3459 if (index && airoha_is_7581(eth) && port->id == AIROHA_GDM3_IDX) 3460 index -= 4; 3461 3462 if (index >= ARRAY_SIZE(port->devs) || port->devs[index]) { 3463 dev_err(eth->dev, "invalid nbq id: %d\n", nbq); 3464 return -EINVAL; 3465 } 3466 3467 netdev->dev.of_node = of_node_get(np); 3468 dev = netdev_priv(netdev); 3469 u64_stats_init(&dev->stats.syncp); 3470 for (i = 0; i < AIROHA_NUM_NETDEV_TX_RINGS; i++) 3471 spin_lock_init(&dev->txq_lock[i]); 3472 dev->port = port; 3473 dev->eth = eth; 3474 dev->nbq = nbq; 3475 port->devs[index] = dev; 3476 3477 return 0; 3478 } 3479 3480 static int airoha_alloc_gdm_port(struct airoha_eth *eth, 3481 struct device_node *np) 3482 { 3483 const __be32 *id_ptr = of_get_property(np, "reg", NULL); 3484 struct airoha_gdm_port *port; 3485 struct device_node *node; 3486 int err, nbq, p, d = 0; 3487 u32 id; 3488 3489 if (!id_ptr) { 3490 dev_err(eth->dev, "missing gdm port id\n"); 3491 return -EINVAL; 3492 } 3493 3494 id = be32_to_cpup(id_ptr); 3495 p = id - 1; 3496 3497 if (!id || id > ARRAY_SIZE(eth->ports)) { 3498 dev_err(eth->dev, "invalid gdm port id: %d\n", id); 3499 return -EINVAL; 3500 } 3501 3502 if (eth->ports[p]) { 3503 dev_err(eth->dev, "duplicate gdm port id: %d\n", id); 3504 return -EINVAL; 3505 } 3506 3507 port = devm_kzalloc(eth->dev, sizeof(*port), GFP_KERNEL); 3508 if (!port) 3509 return -ENOMEM; 3510 3511 port->id = id; 3512 spin_lock_init(&port->stats_lock); 3513 eth->ports[p] = port; 3514 3515 err = airoha_metadata_dst_alloc(port); 3516 if (err) 3517 return err; 3518 3519 /* Default nbq value to ensure backward compatibility */ 3520 nbq = id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0; 3521 3522 for_each_child_of_node(np, node) { 3523 /* Multiple external serdes connected to the FE GDM port via an 3524 * external arbiter. 3525 */ 3526 const __be32 *nbq_ptr; 3527 3528 if (!of_device_is_compatible(node, "airoha,eth-port")) 3529 continue; 3530 3531 d++; 3532 if (!of_device_is_available(node)) 3533 continue; 3534 3535 nbq_ptr = of_get_property(node, "reg", NULL); 3536 if (!nbq_ptr) { 3537 dev_err(eth->dev, "missing nbq id\n"); 3538 of_node_put(node); 3539 return -EINVAL; 3540 } 3541 3542 /* Verify the provided nbq parameter is valid */ 3543 nbq = be32_to_cpup(nbq_ptr); 3544 err = eth->soc->ops.get_sport(port, nbq); 3545 if (err < 0) { 3546 of_node_put(node); 3547 return err; 3548 } 3549 3550 err = airoha_alloc_gdm_device(eth, port, nbq, node); 3551 if (err) { 3552 of_node_put(node); 3553 return err; 3554 } 3555 } 3556 3557 return !d ? airoha_alloc_gdm_device(eth, port, nbq, np) : 0; 3558 } 3559 3560 static int airoha_register_gdm_devices(struct airoha_eth *eth) 3561 { 3562 int i; 3563 3564 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3565 struct airoha_gdm_port *port = eth->ports[i]; 3566 int j; 3567 3568 if (!port) 3569 continue; 3570 3571 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3572 struct airoha_gdm_dev *dev = port->devs[j]; 3573 int err; 3574 3575 if (!dev) 3576 continue; 3577 3578 err = register_netdev(netdev_from_priv(dev)); 3579 if (err) 3580 return err; 3581 } 3582 } 3583 3584 set_bit(DEV_STATE_REGISTERED, ð->state); 3585 3586 return 0; 3587 } 3588 3589 static int airoha_probe(struct platform_device *pdev) 3590 { 3591 struct reset_control_bulk_data *xsi_rsts; 3592 struct device_node *np; 3593 struct airoha_eth *eth; 3594 int i, err; 3595 3596 eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL); 3597 if (!eth) 3598 return -ENOMEM; 3599 3600 eth->soc = of_device_get_match_data(&pdev->dev); 3601 if (!eth->soc) 3602 return -EINVAL; 3603 3604 eth->dev = &pdev->dev; 3605 3606 err = dma_set_mask_and_coherent(eth->dev, DMA_BIT_MASK(32)); 3607 if (err) { 3608 dev_err(eth->dev, "failed configuring DMA mask\n"); 3609 return err; 3610 } 3611 3612 eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe"); 3613 if (IS_ERR(eth->fe_regs)) 3614 return dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs), 3615 "failed to iomap fe regs\n"); 3616 3617 eth->rsts[0].id = "fe"; 3618 eth->rsts[1].id = "pdma"; 3619 eth->rsts[2].id = "qdma"; 3620 err = devm_reset_control_bulk_get_exclusive(eth->dev, 3621 ARRAY_SIZE(eth->rsts), 3622 eth->rsts); 3623 if (err) { 3624 dev_err(eth->dev, "failed to get bulk reset lines\n"); 3625 return err; 3626 } 3627 3628 xsi_rsts = devm_kcalloc(eth->dev, 3629 eth->soc->num_xsi_rsts, sizeof(*xsi_rsts), 3630 GFP_KERNEL); 3631 if (!xsi_rsts) 3632 return -ENOMEM; 3633 3634 eth->xsi_rsts = xsi_rsts; 3635 for (i = 0; i < eth->soc->num_xsi_rsts; i++) 3636 eth->xsi_rsts[i].id = eth->soc->xsi_rsts_names[i]; 3637 3638 err = devm_reset_control_bulk_get_exclusive(eth->dev, 3639 eth->soc->num_xsi_rsts, 3640 eth->xsi_rsts); 3641 if (err) { 3642 dev_err(eth->dev, "failed to get bulk xsi reset lines\n"); 3643 return err; 3644 } 3645 3646 eth->napi_dev = alloc_netdev_dummy(0); 3647 if (!eth->napi_dev) 3648 return -ENOMEM; 3649 3650 /* Enable threaded NAPI by default */ 3651 eth->napi_dev->threaded = true; 3652 strscpy(eth->napi_dev->name, "qdma_eth", sizeof(eth->napi_dev->name)); 3653 platform_set_drvdata(pdev, eth); 3654 3655 err = airoha_hw_init(pdev, eth); 3656 if (err) 3657 goto error_netdev_free; 3658 3659 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 3660 airoha_qdma_start_napi(ð->qdma[i]); 3661 airoha_qdma_set(ð->qdma[i], REG_QDMA_GLOBAL_CFG, 3662 GLOBAL_CFG_TX_DMA_EN_MASK | 3663 GLOBAL_CFG_RX_DMA_EN_MASK); 3664 } 3665 3666 for_each_child_of_node(pdev->dev.of_node, np) { 3667 if (!of_device_is_compatible(np, "airoha,eth-mac")) 3668 continue; 3669 3670 if (!of_device_is_available(np)) 3671 continue; 3672 3673 err = airoha_alloc_gdm_port(eth, np); 3674 if (err) { 3675 of_node_put(np); 3676 goto error_napi_stop; 3677 } 3678 } 3679 3680 err = airoha_register_gdm_devices(eth); 3681 if (err) 3682 goto error_napi_stop; 3683 3684 return 0; 3685 3686 error_napi_stop: 3687 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 3688 airoha_qdma_stop_napi(ð->qdma[i]); 3689 airoha_qdma_tx_cleanup(ð->qdma[i]); 3690 } 3691 3692 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3693 struct airoha_gdm_port *port = eth->ports[i]; 3694 int j; 3695 3696 if (!port) 3697 continue; 3698 3699 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3700 struct airoha_gdm_dev *dev = port->devs[j]; 3701 struct net_device *netdev; 3702 3703 if (!dev) 3704 continue; 3705 3706 netdev = netdev_from_priv(dev); 3707 if (netdev->reg_state == NETREG_REGISTERED) 3708 unregister_netdev(netdev); 3709 of_node_put(netdev->dev.of_node); 3710 } 3711 airoha_metadata_dst_free(port); 3712 } 3713 airoha_hw_cleanup(eth); 3714 error_netdev_free: 3715 free_netdev(eth->napi_dev); 3716 platform_set_drvdata(pdev, NULL); 3717 3718 return err; 3719 } 3720 3721 static void airoha_remove(struct platform_device *pdev) 3722 { 3723 struct airoha_eth *eth = platform_get_drvdata(pdev); 3724 int i; 3725 3726 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 3727 airoha_qdma_stop_napi(ð->qdma[i]); 3728 airoha_qdma_tx_cleanup(ð->qdma[i]); 3729 } 3730 3731 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3732 struct airoha_gdm_port *port = eth->ports[i]; 3733 int j; 3734 3735 if (!port) 3736 continue; 3737 3738 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3739 struct airoha_gdm_dev *dev = port->devs[j]; 3740 struct net_device *netdev; 3741 3742 if (!dev) 3743 continue; 3744 3745 netdev = netdev_from_priv(dev); 3746 unregister_netdev(netdev); 3747 of_node_put(netdev->dev.of_node); 3748 } 3749 airoha_metadata_dst_free(port); 3750 } 3751 airoha_hw_cleanup(eth); 3752 3753 free_netdev(eth->napi_dev); 3754 platform_set_drvdata(pdev, NULL); 3755 } 3756 3757 static const char * const en7581_xsi_rsts_names[] = { 3758 "xsi-mac", 3759 "hsi0-mac", 3760 "hsi1-mac", 3761 "hsi-mac", 3762 "xfp-mac", 3763 }; 3764 3765 static int airoha_en7581_get_sport(struct airoha_gdm_port *port, int nbq) 3766 { 3767 switch (port->id) { 3768 case AIROHA_GDM3_IDX: 3769 /* 7581 SoC supports PCIe serdes on GDM3 port */ 3770 if (nbq == 4) 3771 return HSGMII_LAN_7581_PCIE0_SRCPORT; 3772 if (nbq == 5) 3773 return HSGMII_LAN_7581_PCIE1_SRCPORT; 3774 break; 3775 case AIROHA_GDM4_IDX: 3776 /* 7581 SoC supports eth and usb serdes on GDM4 port */ 3777 if (!nbq) 3778 return HSGMII_LAN_7581_ETH_SRCPORT; 3779 if (nbq == 1) 3780 return HSGMII_LAN_7581_USB_SRCPORT; 3781 break; 3782 default: 3783 break; 3784 } 3785 3786 return -EINVAL; 3787 } 3788 3789 static u32 airoha_en7581_get_vip_port(struct airoha_gdm_port *port, int nbq) 3790 { 3791 switch (port->id) { 3792 case AIROHA_GDM3_IDX: 3793 if (nbq == 4) 3794 return XSI_PCIE0_VIP_PORT_MASK; 3795 if (nbq == 5) 3796 return XSI_PCIE1_VIP_PORT_MASK; 3797 break; 3798 case AIROHA_GDM4_IDX: 3799 if (!nbq) 3800 return XSI_ETH_VIP_PORT_MASK; 3801 if (nbq == 1) 3802 return XSI_USB_VIP_PORT_MASK; 3803 break; 3804 default: 3805 break; 3806 } 3807 3808 return 0; 3809 } 3810 3811 static int airoha_en7581_get_dev_from_sport(struct airoha_qdma_desc *desc, 3812 u16 *port, u16 *dev) 3813 { 3814 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, 3815 le32_to_cpu(READ_ONCE(desc->msg1))); 3816 3817 *dev = 0; 3818 switch (sport) { 3819 case 0x10 ... 0x14: 3820 *port = 0; /* GDM1 */ 3821 break; 3822 case 0x2 ... 0x4: 3823 *port = sport - 1; 3824 break; 3825 case HSGMII_LAN_7581_PCIE1_SRCPORT: 3826 *dev = 1; 3827 fallthrough; 3828 case HSGMII_LAN_7581_PCIE0_SRCPORT: 3829 *port = 2; /* GDM3 */ 3830 break; 3831 case HSGMII_LAN_7581_USB_SRCPORT: 3832 *dev = 1; 3833 fallthrough; 3834 case HSGMII_LAN_7581_ETH_SRCPORT: 3835 *port = 3; /* GDM4 */ 3836 break; 3837 default: 3838 return -EINVAL; 3839 } 3840 3841 return 0; 3842 } 3843 3844 static const char * const an7583_xsi_rsts_names[] = { 3845 "xsi-mac", 3846 "hsi0-mac", 3847 "hsi1-mac", 3848 "xfp-mac", 3849 }; 3850 3851 static int airoha_an7583_get_sport(struct airoha_gdm_port *port, int nbq) 3852 { 3853 switch (port->id) { 3854 case AIROHA_GDM3_IDX: 3855 /* 7583 SoC supports eth serdes on GDM3 port */ 3856 if (!nbq) 3857 return HSGMII_LAN_7583_ETH_SRCPORT; 3858 break; 3859 case AIROHA_GDM4_IDX: 3860 /* 7583 SoC supports PCIe and USB serdes on GDM4 port */ 3861 if (!nbq) 3862 return HSGMII_LAN_7583_PCIE_SRCPORT; 3863 if (nbq == 1) 3864 return HSGMII_LAN_7583_USB_SRCPORT; 3865 break; 3866 default: 3867 break; 3868 } 3869 3870 return -EINVAL; 3871 } 3872 3873 static u32 airoha_an7583_get_vip_port(struct airoha_gdm_port *port, int nbq) 3874 { 3875 switch (port->id) { 3876 case AIROHA_GDM3_IDX: 3877 if (!nbq) 3878 return XSI_ETH_VIP_PORT_MASK; 3879 break; 3880 case AIROHA_GDM4_IDX: 3881 if (!nbq) 3882 return XSI_PCIE0_VIP_PORT_MASK; 3883 if (nbq == 1) 3884 return XSI_USB_VIP_PORT_MASK; 3885 break; 3886 default: 3887 break; 3888 } 3889 3890 return 0; 3891 } 3892 3893 static int airoha_an7583_get_dev_from_sport(struct airoha_qdma_desc *desc, 3894 u16 *port, u16 *dev) 3895 { 3896 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, 3897 le32_to_cpu(READ_ONCE(desc->msg1))); 3898 3899 *dev = 0; 3900 switch (sport) { 3901 case 0x10 ... 0x14: 3902 *port = 0; /* GDM1 */ 3903 break; 3904 case 0x2 ... 0x4: 3905 *port = sport - 1; 3906 break; 3907 case HSGMII_LAN_7583_ETH_SRCPORT: 3908 *port = 2; /* GDM3 */ 3909 break; 3910 case HSGMII_LAN_7583_USB_SRCPORT: 3911 *dev = 1; 3912 fallthrough; 3913 case HSGMII_LAN_7583_PCIE_SRCPORT: 3914 *port = 3; /* GDM4 */ 3915 break; 3916 default: 3917 return -EINVAL; 3918 } 3919 3920 return 0; 3921 } 3922 3923 static const struct airoha_eth_soc_data en7581_soc_data = { 3924 .version = 0x7581, 3925 .xsi_rsts_names = en7581_xsi_rsts_names, 3926 .num_xsi_rsts = ARRAY_SIZE(en7581_xsi_rsts_names), 3927 .num_ppe = 2, 3928 .ops = { 3929 .get_sport = airoha_en7581_get_sport, 3930 .get_vip_port = airoha_en7581_get_vip_port, 3931 .get_dev_from_sport = airoha_en7581_get_dev_from_sport, 3932 }, 3933 }; 3934 3935 static const struct airoha_eth_soc_data an7583_soc_data = { 3936 .version = 0x7583, 3937 .xsi_rsts_names = an7583_xsi_rsts_names, 3938 .num_xsi_rsts = ARRAY_SIZE(an7583_xsi_rsts_names), 3939 .num_ppe = 1, 3940 .ops = { 3941 .get_sport = airoha_an7583_get_sport, 3942 .get_vip_port = airoha_an7583_get_vip_port, 3943 .get_dev_from_sport = airoha_an7583_get_dev_from_sport, 3944 }, 3945 }; 3946 3947 static const struct of_device_id of_airoha_match[] = { 3948 { .compatible = "airoha,en7581-eth", .data = &en7581_soc_data }, 3949 { .compatible = "airoha,an7583-eth", .data = &an7583_soc_data }, 3950 { /* sentinel */ } 3951 }; 3952 MODULE_DEVICE_TABLE(of, of_airoha_match); 3953 3954 static struct platform_driver airoha_driver = { 3955 .probe = airoha_probe, 3956 .remove = airoha_remove, 3957 .driver = { 3958 .name = KBUILD_MODNAME, 3959 .of_match_table = of_airoha_match, 3960 }, 3961 }; 3962 module_platform_driver(airoha_driver); 3963 3964 MODULE_LICENSE("GPL v2"); 3965 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>"); 3966 MODULE_DESCRIPTION("Ethernet driver for Airoha SoC"); 3967