1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2024 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 #include <linux/of.h> 7 #include <linux/of_net.h> 8 #include <linux/of_reserved_mem.h> 9 #include <linux/platform_device.h> 10 #include <linux/tcp.h> 11 #include <linux/u64_stats_sync.h> 12 #include <net/dst_metadata.h> 13 #include <net/page_pool/helpers.h> 14 #include <net/pkt_cls.h> 15 #include <uapi/linux/ppp_defs.h> 16 17 #include "airoha_regs.h" 18 #include "airoha_eth.h" 19 20 u32 airoha_rr(void __iomem *base, u32 offset) 21 { 22 return readl(base + offset); 23 } 24 25 void airoha_wr(void __iomem *base, u32 offset, u32 val) 26 { 27 writel(val, base + offset); 28 } 29 30 u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val) 31 { 32 val |= (airoha_rr(base, offset) & ~mask); 33 airoha_wr(base, offset, val); 34 35 return val; 36 } 37 38 static void airoha_qdma_set_irqmask(struct airoha_irq_bank *irq_bank, 39 int index, u32 clear, u32 set) 40 { 41 struct airoha_qdma *qdma = irq_bank->qdma; 42 int bank = irq_bank - &qdma->irq_banks[0]; 43 unsigned long flags; 44 45 if (WARN_ON_ONCE(index >= ARRAY_SIZE(irq_bank->irqmask))) 46 return; 47 48 spin_lock_irqsave(&irq_bank->irq_lock, flags); 49 50 irq_bank->irqmask[index] &= ~clear; 51 irq_bank->irqmask[index] |= set; 52 airoha_qdma_wr(qdma, REG_INT_ENABLE(bank, index), 53 irq_bank->irqmask[index]); 54 /* Read irq_enable register in order to guarantee the update above 55 * completes in the spinlock critical section. 56 */ 57 airoha_qdma_rr(qdma, REG_INT_ENABLE(bank, index)); 58 59 spin_unlock_irqrestore(&irq_bank->irq_lock, flags); 60 } 61 62 static void airoha_qdma_irq_enable(struct airoha_irq_bank *irq_bank, 63 int index, u32 mask) 64 { 65 airoha_qdma_set_irqmask(irq_bank, index, 0, mask); 66 } 67 68 static void airoha_qdma_irq_disable(struct airoha_irq_bank *irq_bank, 69 int index, u32 mask) 70 { 71 airoha_qdma_set_irqmask(irq_bank, index, mask, 0); 72 } 73 74 static int airoha_set_macaddr(struct airoha_gdm_dev *dev, const u8 *addr) 75 { 76 u8 ref_addr[ETH_ALEN] __aligned(2); 77 struct airoha_eth *eth = dev->eth; 78 u32 reg, val, lmin, lmax; 79 int i; 80 81 eth_zero_addr(ref_addr); 82 lmin = (addr[3] << 16) | (addr[4] << 8) | addr[5]; 83 lmax = lmin; 84 85 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 86 struct airoha_gdm_port *port = eth->ports[i]; 87 int j; 88 89 if (!port) 90 continue; 91 92 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 93 struct airoha_gdm_dev *iter_dev; 94 struct net_device *netdev; 95 96 iter_dev = port->devs[j]; 97 if (!iter_dev || iter_dev == dev) 98 continue; 99 100 if (airoha_is_lan_gdm_dev(iter_dev) != 101 airoha_is_lan_gdm_dev(dev)) 102 continue; 103 104 netdev = netdev_from_priv(iter_dev); 105 if (netdev->reg_state != NETREG_REGISTERED) 106 continue; 107 108 ether_addr_copy(ref_addr, netdev->dev_addr); 109 val = (netdev->dev_addr[3] << 16) | 110 (netdev->dev_addr[4] << 8) | netdev->dev_addr[5]; 111 if (val < lmin) 112 lmin = val; 113 if (val > lmax) 114 lmax = val; 115 } 116 } 117 118 if (!is_zero_ether_addr(ref_addr) && memcmp(ref_addr, addr, 3)) { 119 /* According to the HW design, hw mac address MSBs must be 120 * the same for each net_device with the same LAN/WAN 121 * configuration. 122 */ 123 struct net_device *netdev = netdev_from_priv(dev); 124 125 dev_warn(eth->dev, 126 "%s: wrong mac addr, MSBs must be %02x:%02x:%02x\n", 127 netdev->name, ref_addr[0], ref_addr[1], 128 ref_addr[2]); 129 dev_warn(eth->dev, "FE hw forwarding won't work properly\n"); 130 131 return -EINVAL; 132 } 133 134 reg = airoha_is_lan_gdm_dev(dev) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H; 135 val = (addr[0] << 16) | (addr[1] << 8) | addr[2]; 136 airoha_fe_wr(eth, reg, val); 137 138 airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), lmin); 139 airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), lmax); 140 141 airoha_ppe_init_upd_mem(dev, addr); 142 143 return 0; 144 } 145 146 static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr, 147 u32 val) 148 { 149 airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK, 150 FIELD_PREP(GDM_OCFQ_MASK, val)); 151 airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK, 152 FIELD_PREP(GDM_MCFQ_MASK, val)); 153 airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK, 154 FIELD_PREP(GDM_BCFQ_MASK, val)); 155 airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK, 156 FIELD_PREP(GDM_UCFQ_MASK, val)); 157 } 158 159 static int airoha_set_vip_for_gdm_port(struct airoha_gdm_dev *dev, bool enable) 160 { 161 struct airoha_gdm_port *port = dev->port; 162 struct airoha_eth *eth = dev->eth; 163 u32 vip_port; 164 165 vip_port = eth->soc->ops.get_vip_port(port, dev->nbq); 166 if (enable) { 167 airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port); 168 airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port); 169 } else { 170 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port); 171 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port); 172 } 173 174 return 0; 175 } 176 177 static void airoha_fe_maccr_init(struct airoha_eth *eth) 178 { 179 int p; 180 181 for (p = 1; p <= ARRAY_SIZE(eth->ports); p++) 182 airoha_fe_set(eth, REG_GDM_FWD_CFG(p), 183 GDM_TCP_CKSUM_MASK | GDM_UDP_CKSUM_MASK | 184 GDM_IP4_CKSUM_MASK | GDM_DROP_CRC_ERR_MASK); 185 186 airoha_fe_rmw(eth, REG_CDM_VLAN_CTRL(1), CDM_VLAN_MASK, 187 FIELD_PREP(CDM_VLAN_MASK, 0x8100)); 188 189 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD); 190 } 191 192 static void airoha_fe_vip_setup(struct airoha_eth *eth) 193 { 194 airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC); 195 airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK); 196 197 airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP); 198 airoha_fe_wr(eth, REG_FE_VIP_EN(4), 199 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 200 PATN_EN_MASK); 201 202 airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP); 203 airoha_fe_wr(eth, REG_FE_VIP_EN(6), 204 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 205 PATN_EN_MASK); 206 207 airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP); 208 airoha_fe_wr(eth, REG_FE_VIP_EN(7), 209 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 210 PATN_EN_MASK); 211 212 /* BOOTP (0x43) */ 213 airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43); 214 airoha_fe_wr(eth, REG_FE_VIP_EN(8), 215 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK | 216 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 217 218 /* BOOTP (0x44) */ 219 airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44); 220 airoha_fe_wr(eth, REG_FE_VIP_EN(9), 221 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK | 222 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 223 224 /* ISAKMP */ 225 airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4); 226 airoha_fe_wr(eth, REG_FE_VIP_EN(10), 227 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK | 228 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 229 230 airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP); 231 airoha_fe_wr(eth, REG_FE_VIP_EN(11), 232 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 233 PATN_EN_MASK); 234 235 /* DHCPv6 */ 236 airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223); 237 airoha_fe_wr(eth, REG_FE_VIP_EN(12), 238 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK | 239 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 240 241 airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP); 242 airoha_fe_wr(eth, REG_FE_VIP_EN(19), 243 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 244 PATN_EN_MASK); 245 246 /* ETH->ETH_P_1905 (0x893a) */ 247 airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a); 248 airoha_fe_wr(eth, REG_FE_VIP_EN(20), 249 PATN_FCPU_EN_MASK | PATN_EN_MASK); 250 251 airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP); 252 airoha_fe_wr(eth, REG_FE_VIP_EN(21), 253 PATN_FCPU_EN_MASK | PATN_EN_MASK); 254 } 255 256 static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth, 257 u32 port, u32 queue) 258 { 259 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR, 260 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK, 261 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) | 262 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue)); 263 264 return airoha_fe_get(eth, REG_FE_PSE_QUEUE_CFG_VAL, 265 PSE_CFG_OQ_RSV_MASK); 266 } 267 268 static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth, 269 u32 port, u32 queue, u32 val) 270 { 271 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK, 272 FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val)); 273 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR, 274 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK | 275 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK, 276 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) | 277 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) | 278 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK); 279 } 280 281 static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth) 282 { 283 return airoha_fe_get(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK); 284 } 285 286 static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth, 287 u32 port, u32 queue, u32 val) 288 { 289 u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue); 290 u32 tmp, all_rsv, fq_limit; 291 292 airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val); 293 294 /* modify all rsv */ 295 all_rsv = airoha_fe_get_pse_all_rsv(eth); 296 all_rsv += (val - orig_val); 297 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, 298 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); 299 300 /* modify hthd */ 301 fq_limit = airoha_fe_get(eth, PSE_FQ_CFG, PSE_FQ_LIMIT_MASK); 302 tmp = fq_limit - all_rsv - 0x20; 303 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD, 304 PSE_SHARE_USED_HTHD_MASK, 305 FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp)); 306 307 tmp = fq_limit - all_rsv - 0x100; 308 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD, 309 PSE_SHARE_USED_MTHD_MASK, 310 FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp)); 311 tmp = (3 * tmp) >> 2; 312 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, 313 PSE_SHARE_USED_LTHD_MASK, 314 FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp)); 315 316 return 0; 317 } 318 319 static void airoha_fe_pse_ports_init(struct airoha_eth *eth) 320 { 321 const u32 pse_port_num_queues[] = { 322 [FE_PSE_PORT_CDM1] = 6, 323 [FE_PSE_PORT_GDM1] = 6, 324 [FE_PSE_PORT_GDM2] = 32, 325 [FE_PSE_PORT_GDM3] = 6, 326 [FE_PSE_PORT_PPE1] = 4, 327 [FE_PSE_PORT_CDM2] = 6, 328 [FE_PSE_PORT_CDM3] = 8, 329 [FE_PSE_PORT_CDM4] = 10, 330 [FE_PSE_PORT_PPE2] = 4, 331 [FE_PSE_PORT_GDM4] = 2, 332 [FE_PSE_PORT_CDM5] = 2, 333 }; 334 int q; 335 336 if (airoha_ppe_is_enabled(eth, 1)) { 337 u32 all_rsv; 338 339 /* hw misses PPE2 oq rsv */ 340 all_rsv = airoha_fe_get_pse_all_rsv(eth); 341 all_rsv += PSE_RSV_PAGES * 342 pse_port_num_queues[FE_PSE_PORT_PPE2]; 343 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, 344 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); 345 } 346 347 /* CDM1 */ 348 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++) 349 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q, 350 PSE_QUEUE_RSV_PAGES); 351 /* GDM1 */ 352 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++) 353 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q, 354 PSE_QUEUE_RSV_PAGES); 355 /* GDM2 */ 356 for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++) 357 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0); 358 /* GDM3 */ 359 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++) 360 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q, 361 PSE_QUEUE_RSV_PAGES); 362 /* PPE1 */ 363 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) { 364 if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2) 365 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 366 PSE_QUEUE_RSV_PAGES); 367 else 368 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0); 369 } 370 /* CDM2 */ 371 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++) 372 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q, 373 PSE_QUEUE_RSV_PAGES); 374 /* CDM3 */ 375 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++) 376 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0); 377 /* CDM4 */ 378 for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++) 379 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q, 380 PSE_QUEUE_RSV_PAGES); 381 if (airoha_ppe_is_enabled(eth, 1)) { 382 /* PPE2 */ 383 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) { 384 if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2) 385 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, 386 q, 387 PSE_QUEUE_RSV_PAGES); 388 else 389 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, 390 q, 0); 391 } 392 } 393 /* GDM4 */ 394 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++) 395 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q, 396 PSE_QUEUE_RSV_PAGES); 397 /* CDM5 */ 398 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++) 399 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q, 400 PSE_QUEUE_RSV_PAGES); 401 } 402 403 static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth) 404 { 405 int i; 406 407 for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) { 408 int err, j; 409 u32 val; 410 411 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0); 412 413 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) | 414 MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK; 415 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val); 416 err = read_poll_timeout(airoha_fe_rr, val, 417 val & MC_VLAN_CFG_CMD_DONE_MASK, 418 USEC_PER_MSEC, 5 * USEC_PER_MSEC, 419 false, eth, REG_MC_VLAN_CFG); 420 if (err) 421 return err; 422 423 for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) { 424 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0); 425 426 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) | 427 FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) | 428 MC_VLAN_CFG_RW_MASK; 429 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val); 430 err = read_poll_timeout(airoha_fe_rr, val, 431 val & MC_VLAN_CFG_CMD_DONE_MASK, 432 USEC_PER_MSEC, 433 5 * USEC_PER_MSEC, false, eth, 434 REG_MC_VLAN_CFG); 435 if (err) 436 return err; 437 } 438 } 439 440 return 0; 441 } 442 443 static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth) 444 { 445 /* CDM1_CRSN_QSEL */ 446 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_22 >> 2), 447 CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 448 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 449 CDM_CRSN_QSEL_Q1)); 450 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_08 >> 2), 451 CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 452 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 453 CDM_CRSN_QSEL_Q1)); 454 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_21 >> 2), 455 CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 456 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 457 CDM_CRSN_QSEL_Q1)); 458 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_24 >> 2), 459 CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 460 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 461 CDM_CRSN_QSEL_Q6)); 462 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_25 >> 2), 463 CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 464 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 465 CDM_CRSN_QSEL_Q1)); 466 /* CDM2_CRSN_QSEL */ 467 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_08 >> 2), 468 CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 469 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 470 CDM_CRSN_QSEL_Q1)); 471 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_21 >> 2), 472 CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 473 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 474 CDM_CRSN_QSEL_Q1)); 475 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_22 >> 2), 476 CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 477 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 478 CDM_CRSN_QSEL_Q1)); 479 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_24 >> 2), 480 CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 481 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 482 CDM_CRSN_QSEL_Q6)); 483 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_25 >> 2), 484 CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 485 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 486 CDM_CRSN_QSEL_Q1)); 487 } 488 489 static int airoha_fe_init(struct airoha_eth *eth) 490 { 491 airoha_fe_maccr_init(eth); 492 493 /* PSE IQ reserve */ 494 airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK, 495 FIELD_PREP(PSE_IQ_RES1_P2_MASK, 0x10)); 496 airoha_fe_rmw(eth, REG_PSE_IQ_REV2, 497 PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK, 498 FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) | 499 FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34)); 500 501 /* enable FE copy engine for KA/DPI */ 502 airoha_fe_wr(eth, REG_FE_PCE_CFG, PCE_DPI_EN_MASK | PCE_KA_EN_MASK); 503 /* set vip queue selection to ring 1 */ 504 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(1), CDM_VIP_QSEL_MASK, 505 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4)); 506 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_VIP_QSEL_MASK, 507 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4)); 508 /* set GDM4 source interface offset to 8 */ 509 airoha_fe_rmw(eth, REG_GDM_SRC_PORT_SET(4), 510 GDM_SPORT_OFF2_MASK | 511 GDM_SPORT_OFF1_MASK | 512 GDM_SPORT_OFF0_MASK, 513 FIELD_PREP(GDM_SPORT_OFF2_MASK, 8) | 514 FIELD_PREP(GDM_SPORT_OFF1_MASK, 8) | 515 FIELD_PREP(GDM_SPORT_OFF0_MASK, 8)); 516 517 /* set PSE Page as 128B */ 518 airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG, 519 FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK, 520 FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, 2) | 521 FE_DMA_GLO_PG_SZ_MASK); 522 airoha_fe_wr(eth, REG_FE_RST_GLO_CFG, 523 FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK | 524 FE_RST_GDM4_MBI_ARB_MASK); 525 usleep_range(1000, 2000); 526 527 /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1 528 * connect other rings to PSE Port0 OQ-0 529 */ 530 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4)); 531 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28)); 532 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4)); 533 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28)); 534 535 airoha_fe_vip_setup(eth); 536 airoha_fe_pse_ports_init(eth); 537 538 airoha_fe_set(eth, REG_GDM_MISC_CFG, 539 GDM2_RDM_ACK_WAIT_PREF_MASK | 540 GDM2_CHN_VLD_MODE_MASK); 541 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_OAM_QSEL_MASK, 542 FIELD_PREP(CDM_OAM_QSEL_MASK, 15)); 543 544 /* init fragment and assemble Force Port */ 545 /* NPU Core-3, NPU Bridge Channel-3 */ 546 airoha_fe_rmw(eth, REG_IP_FRAG_FP, 547 IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK, 548 FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) | 549 FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3)); 550 /* QDMA LAN, RX Ring-22 */ 551 airoha_fe_rmw(eth, REG_IP_FRAG_FP, 552 IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK, 553 FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) | 554 FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22)); 555 556 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM3_IDX), GDM_PAD_EN_MASK); 557 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM4_IDX), GDM_PAD_EN_MASK); 558 559 /* Enable split for MIB counters for GDM3 and GDM4 */ 560 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM3_IDX), 561 FE_GDM_TX_MIB_SPLIT_EN_MASK | 562 FE_GDM_RX_MIB_SPLIT_EN_MASK); 563 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM4_IDX), 564 FE_GDM_TX_MIB_SPLIT_EN_MASK | 565 FE_GDM_RX_MIB_SPLIT_EN_MASK); 566 567 airoha_fe_crsn_qsel_init(eth); 568 569 airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK); 570 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK); 571 572 /* default aging mode for mbi unlock issue */ 573 airoha_fe_rmw(eth, REG_GDM_CHN_RLS(2), 574 MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK, 575 FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) | 576 FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3)); 577 578 /* disable IFC by default */ 579 airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK); 580 581 /* enable 1:N vlan action, init vlan table */ 582 airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK); 583 584 return airoha_fe_mc_vlan_clear(eth); 585 } 586 587 static int airoha_qdma_fill_rx_queue(struct airoha_queue *q) 588 { 589 struct airoha_qdma *qdma = q->qdma; 590 int qid = q - &qdma->q_rx[0]; 591 int nframes = 0; 592 593 while (q->queued < q->ndesc - 1) { 594 struct airoha_queue_entry *e = &q->entry[q->head]; 595 struct airoha_qdma_desc *desc = &q->desc[q->head]; 596 struct page *page; 597 int offset; 598 u32 val; 599 600 page = page_pool_dev_alloc_frag(q->page_pool, &offset, 601 q->buf_size); 602 if (!page) 603 break; 604 605 q->head = (q->head + 1) % q->ndesc; 606 q->queued++; 607 nframes++; 608 609 offset += AIROHA_RX_HEADROOM; 610 e->buf = page_address(page) + offset; 611 e->dma_addr = page_pool_get_dma_addr(page) + offset; 612 e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size)); 613 614 val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len); 615 WRITE_ONCE(desc->ctrl, cpu_to_le32(val)); 616 WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr)); 617 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head); 618 WRITE_ONCE(desc->data, cpu_to_le32(val)); 619 WRITE_ONCE(desc->msg0, 0); 620 WRITE_ONCE(desc->msg1, 0); 621 WRITE_ONCE(desc->msg2, 0); 622 WRITE_ONCE(desc->msg3, 0); 623 } 624 625 if (nframes) 626 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), 627 RX_RING_CPU_IDX_MASK, 628 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); 629 630 return nframes; 631 } 632 633 static struct airoha_gdm_dev * 634 airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc *desc) 635 { 636 struct airoha_gdm_port *port; 637 u16 p, d; 638 639 if (eth->soc->ops.get_dev_from_sport(desc, &p, &d)) 640 return ERR_PTR(-ENODEV); 641 642 if (p >= ARRAY_SIZE(eth->ports)) 643 return ERR_PTR(-ENODEV); 644 645 port = eth->ports[p]; 646 if (!port) 647 return ERR_PTR(-ENODEV); 648 649 if (d >= ARRAY_SIZE(port->devs)) 650 return ERR_PTR(-ENODEV); 651 652 return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV); 653 } 654 655 static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) 656 { 657 enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool); 658 struct airoha_qdma *qdma = q->qdma; 659 struct airoha_eth *eth = qdma->eth; 660 int qid = q - &qdma->q_rx[0]; 661 int done = 0; 662 663 while (done < budget) { 664 struct airoha_queue_entry *e = &q->entry[q->tail]; 665 struct airoha_qdma_desc *desc = &q->desc[q->tail]; 666 u32 hash, reason, msg1, desc_ctrl; 667 struct airoha_gdm_dev *dev; 668 struct net_device *netdev; 669 int data_len, len; 670 struct page *page; 671 672 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); 673 if (!(desc_ctrl & QDMA_DESC_DONE_MASK)) 674 break; 675 676 dma_rmb(); 677 678 q->tail = (q->tail + 1) % q->ndesc; 679 q->queued--; 680 681 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, 682 dir); 683 684 page = virt_to_head_page(e->buf); 685 len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl); 686 data_len = q->skb ? AIROHA_RX_LEN(q->buf_size) : e->dma_len; 687 if (!len || data_len < len) 688 goto free_frag; 689 690 dev = airoha_qdma_get_gdm_dev(eth, desc); 691 if (IS_ERR(dev)) 692 goto free_frag; 693 694 netdev = netdev_from_priv(dev); 695 if (!q->skb) { /* first buffer */ 696 q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM, 697 q->buf_size); 698 if (!q->skb) 699 goto free_frag; 700 701 skb_reserve(q->skb, AIROHA_RX_HEADROOM); 702 __skb_put(q->skb, len); 703 skb_mark_for_recycle(q->skb); 704 q->skb->dev = netdev; 705 q->skb->protocol = eth_type_trans(q->skb, netdev); 706 q->skb->ip_summed = CHECKSUM_UNNECESSARY; 707 skb_record_rx_queue(q->skb, qid); 708 } else { /* scattered frame */ 709 struct skb_shared_info *shinfo = skb_shinfo(q->skb); 710 int nr_frags = shinfo->nr_frags; 711 712 if (nr_frags >= ARRAY_SIZE(shinfo->frags)) 713 goto free_frag; 714 715 skb_add_rx_frag(q->skb, nr_frags, page, 716 e->buf - page_address(page), len, 717 q->buf_size); 718 } 719 720 if (FIELD_GET(QDMA_DESC_MORE_MASK, desc_ctrl)) 721 continue; 722 723 if (netdev_uses_dsa(netdev)) { 724 struct airoha_gdm_port *port = dev->port; 725 726 /* PPE module requires untagged packets to work 727 * properly and it provides DSA port index via the 728 * DMA descriptor. Report DSA tag to the DSA stack 729 * via skb dst info. 730 */ 731 u32 msg0 = le32_to_cpu(READ_ONCE(desc->msg0)); 732 u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0); 733 734 if (sptag < ARRAY_SIZE(port->dsa_meta) && 735 port->dsa_meta[sptag]) 736 skb_dst_set_noref(q->skb, 737 &port->dsa_meta[sptag]->dst); 738 } 739 740 msg1 = le32_to_cpu(READ_ONCE(desc->msg1)); 741 hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1); 742 if (hash != AIROHA_RXD4_FOE_ENTRY) 743 skb_set_hash(q->skb, jhash_1word(hash, 0), 744 PKT_HASH_TYPE_L4); 745 746 reason = FIELD_GET(AIROHA_RXD4_PPE_CPU_REASON, msg1); 747 if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) 748 airoha_ppe_check_skb(ð->ppe->dev, q->skb, hash, 749 false); 750 751 done++; 752 napi_gro_receive(&q->napi, q->skb); 753 q->skb = NULL; 754 continue; 755 free_frag: 756 if (q->skb) { 757 dev_kfree_skb(q->skb); 758 q->skb = NULL; 759 } 760 page_pool_put_full_page(q->page_pool, page, true); 761 } 762 airoha_qdma_fill_rx_queue(q); 763 764 return done; 765 } 766 767 static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget) 768 { 769 struct airoha_queue *q = container_of(napi, struct airoha_queue, napi); 770 int cur, done = 0; 771 772 do { 773 cur = airoha_qdma_rx_process(q, budget - done); 774 done += cur; 775 } while (cur && done < budget); 776 777 if (done < budget && napi_complete(napi)) { 778 struct airoha_qdma *qdma = q->qdma; 779 int i, qid = q - &qdma->q_rx[0]; 780 int intr_reg = qid < RX_DONE_HIGH_OFFSET ? QDMA_INT_REG_IDX1 781 : QDMA_INT_REG_IDX2; 782 783 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 784 if (!(BIT(qid) & RX_IRQ_BANK_PIN_MASK(i))) 785 continue; 786 787 airoha_qdma_irq_enable(&qdma->irq_banks[i], intr_reg, 788 BIT(qid % RX_DONE_HIGH_OFFSET)); 789 } 790 } 791 792 return done; 793 } 794 795 static int airoha_qdma_init_rx_queue(struct airoha_queue *q, 796 struct airoha_qdma *qdma, int ndesc) 797 { 798 const struct page_pool_params pp_params = { 799 .order = 0, 800 .pool_size = 256, 801 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 802 .dma_dir = DMA_FROM_DEVICE, 803 .max_len = PAGE_SIZE, 804 .nid = NUMA_NO_NODE, 805 .dev = qdma->eth->dev, 806 .napi = &q->napi, 807 }; 808 struct airoha_eth *eth = qdma->eth; 809 int qid = q - &qdma->q_rx[0], thr; 810 dma_addr_t dma_addr; 811 812 q->buf_size = PAGE_SIZE / 2; 813 q->qdma = qdma; 814 815 q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry), 816 GFP_KERNEL); 817 if (!q->entry) 818 return -ENOMEM; 819 820 q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc), 821 &dma_addr, GFP_KERNEL); 822 if (!q->desc) 823 return -ENOMEM; 824 825 q->page_pool = page_pool_create(&pp_params); 826 if (IS_ERR(q->page_pool)) { 827 int err = PTR_ERR(q->page_pool); 828 829 q->page_pool = NULL; 830 return err; 831 } 832 833 q->ndesc = ndesc; 834 netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll); 835 836 airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr); 837 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), 838 RX_RING_SIZE_MASK, 839 FIELD_PREP(RX_RING_SIZE_MASK, ndesc)); 840 841 thr = clamp(ndesc >> 3, 1, 32); 842 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK, 843 FIELD_PREP(RX_RING_THR_MASK, thr)); 844 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, 845 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head)); 846 airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid), RX_RING_SG_EN_MASK); 847 848 airoha_qdma_fill_rx_queue(q); 849 850 return 0; 851 } 852 853 static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q) 854 { 855 struct airoha_qdma *qdma = q->qdma; 856 struct airoha_eth *eth = qdma->eth; 857 int qid = q - &qdma->q_rx[0]; 858 859 while (q->queued) { 860 struct airoha_queue_entry *e = &q->entry[q->tail]; 861 struct airoha_qdma_desc *desc = &q->desc[q->tail]; 862 struct page *page = virt_to_head_page(e->buf); 863 864 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, 865 page_pool_get_dma_dir(q->page_pool)); 866 page_pool_put_full_page(q->page_pool, page, false); 867 /* Reset DMA descriptor */ 868 WRITE_ONCE(desc->ctrl, 0); 869 WRITE_ONCE(desc->addr, 0); 870 WRITE_ONCE(desc->data, 0); 871 WRITE_ONCE(desc->msg0, 0); 872 WRITE_ONCE(desc->msg1, 0); 873 WRITE_ONCE(desc->msg2, 0); 874 WRITE_ONCE(desc->msg3, 0); 875 876 q->tail = (q->tail + 1) % q->ndesc; 877 q->queued--; 878 } 879 880 q->head = q->tail; 881 /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is 882 * empty. 883 */ 884 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK, 885 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); 886 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, 887 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail)); 888 } 889 890 static int airoha_qdma_init_rx(struct airoha_qdma *qdma) 891 { 892 int i; 893 894 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 895 int err; 896 897 if (!(RX_DONE_INT_MASK & BIT(i))) { 898 /* rx-queue not binded to irq */ 899 continue; 900 } 901 902 err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma, 903 RX_DSCP_NUM(i)); 904 if (err) 905 return err; 906 } 907 908 return 0; 909 } 910 911 static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q) 912 { 913 struct airoha_qdma *qdma = q->qdma; 914 struct airoha_eth *eth = qdma->eth; 915 int i, qid = q - &qdma->q_tx[0]; 916 917 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 918 struct airoha_gdm_port *port = eth->ports[i]; 919 int d; 920 921 if (!port) 922 continue; 923 924 for (d = 0; d < ARRAY_SIZE(port->devs); d++) { 925 struct airoha_gdm_dev *dev = port->devs[d]; 926 struct net_device *netdev; 927 int j; 928 929 if (!dev) 930 continue; 931 932 if (dev->qdma != qdma) 933 continue; 934 935 netdev = netdev_from_priv(dev); 936 for (j = 0; j < netdev->num_tx_queues; j++) { 937 if (airoha_qdma_get_txq(qdma, j) != qid) 938 continue; 939 940 netif_wake_subqueue(netdev, j); 941 } 942 } 943 } 944 q->txq_stopped = false; 945 } 946 947 static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget) 948 { 949 struct airoha_tx_irq_queue *irq_q; 950 int id, done = 0, irq_queued; 951 struct airoha_qdma *qdma; 952 struct airoha_eth *eth; 953 u32 status, head; 954 955 irq_q = container_of(napi, struct airoha_tx_irq_queue, napi); 956 qdma = irq_q->qdma; 957 id = irq_q - &qdma->q_tx_irq[0]; 958 eth = qdma->eth; 959 960 status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id)); 961 head = FIELD_GET(IRQ_HEAD_IDX_MASK, status); 962 head = head % irq_q->size; 963 irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status); 964 965 while (irq_queued > 0 && done < budget) { 966 u32 qid, val = irq_q->q[head]; 967 struct airoha_qdma_desc *desc; 968 struct airoha_queue_entry *e; 969 struct airoha_queue *q; 970 u32 index, desc_ctrl; 971 struct sk_buff *skb; 972 973 if (val == 0xff) 974 break; 975 976 irq_q->q[head] = 0xff; /* mark as done */ 977 head = (head + 1) % irq_q->size; 978 irq_queued--; 979 done++; 980 981 qid = FIELD_GET(IRQ_RING_IDX_MASK, val); 982 if (qid >= ARRAY_SIZE(qdma->q_tx)) 983 continue; 984 985 q = &qdma->q_tx[qid]; 986 if (!q->ndesc) 987 continue; 988 989 index = FIELD_GET(IRQ_DESC_IDX_MASK, val); 990 if (index >= q->ndesc) 991 continue; 992 993 spin_lock_bh(&q->lock); 994 995 if (!q->queued) 996 goto unlock; 997 998 desc = &q->desc[index]; 999 desc_ctrl = le32_to_cpu(desc->ctrl); 1000 1001 if (!(desc_ctrl & QDMA_DESC_DONE_MASK) && 1002 !(desc_ctrl & QDMA_DESC_DROP_MASK)) 1003 goto unlock; 1004 1005 e = &q->entry[index]; 1006 skb = e->skb; 1007 1008 dma_unmap_single(eth->dev, e->dma_addr, e->dma_len, 1009 DMA_TO_DEVICE); 1010 e->dma_addr = 0; 1011 list_add_tail(&e->list, &q->tx_list); 1012 1013 WRITE_ONCE(desc->msg0, 0); 1014 WRITE_ONCE(desc->msg1, 0); 1015 q->queued--; 1016 1017 if (skb) { 1018 struct netdev_queue *txq; 1019 1020 txq = skb_get_tx_queue(skb->dev, skb); 1021 netdev_tx_completed_queue(txq, 1, skb->len); 1022 dev_kfree_skb_any(skb); 1023 } 1024 1025 if (q->txq_stopped && q->ndesc - q->queued >= q->free_thr) { 1026 /* Since multiple net_device TX queues can share the 1027 * same hw QDMA TX queue, there is no guarantee we have 1028 * inflight packets queued in hw belonging to a 1029 * net_device TX queue stopped in the xmit path. 1030 * In order to avoid any potential net_device TX queue 1031 * stall, we need to wake all the net_device TX queues 1032 * feeding the same hw QDMA TX queue. 1033 */ 1034 airoha_qdma_wake_netdev_txqs(q); 1035 } 1036 1037 unlock: 1038 spin_unlock_bh(&q->lock); 1039 } 1040 1041 if (done) { 1042 int i, len = done >> 7; 1043 1044 for (i = 0; i < len; i++) 1045 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id), 1046 IRQ_CLEAR_LEN_MASK, 0x80); 1047 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id), 1048 IRQ_CLEAR_LEN_MASK, (done & 0x7f)); 1049 } 1050 1051 if (done < budget && napi_complete(napi)) 1052 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0, 1053 TX_DONE_INT_MASK(id)); 1054 1055 return done; 1056 } 1057 1058 static int airoha_qdma_init_tx_queue(struct airoha_queue *q, 1059 struct airoha_qdma *qdma, int size) 1060 { 1061 struct airoha_eth *eth = qdma->eth; 1062 int i, qid = q - &qdma->q_tx[0]; 1063 dma_addr_t dma_addr; 1064 1065 spin_lock_init(&q->lock); 1066 q->qdma = qdma; 1067 q->free_thr = 1 + MAX_SKB_FRAGS; 1068 INIT_LIST_HEAD(&q->tx_list); 1069 1070 q->entry = devm_kzalloc(eth->dev, size * sizeof(*q->entry), 1071 GFP_KERNEL); 1072 if (!q->entry) 1073 return -ENOMEM; 1074 1075 q->desc = dmam_alloc_coherent(eth->dev, size * sizeof(*q->desc), 1076 &dma_addr, GFP_KERNEL); 1077 if (!q->desc) 1078 return -ENOMEM; 1079 1080 for (i = 0; i < size; i++) { 1081 u32 val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1); 1082 1083 list_add_tail(&q->entry[i].list, &q->tx_list); 1084 WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val)); 1085 } 1086 q->ndesc = size; 1087 1088 /* xmit ring drop default setting */ 1089 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(qid), 1090 TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK); 1091 1092 airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr); 1093 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK, 1094 FIELD_PREP(TX_RING_CPU_IDX_MASK, 0)); 1095 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK, 1096 FIELD_PREP(TX_RING_DMA_IDX_MASK, 0)); 1097 1098 return 0; 1099 } 1100 1101 static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q, 1102 struct airoha_qdma *qdma, int size) 1103 { 1104 int id = irq_q - &qdma->q_tx_irq[0]; 1105 struct airoha_eth *eth = qdma->eth; 1106 dma_addr_t dma_addr; 1107 1108 irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32), 1109 &dma_addr, GFP_KERNEL); 1110 if (!irq_q->q) 1111 return -ENOMEM; 1112 1113 memset(irq_q->q, 0xff, size * sizeof(u32)); 1114 irq_q->size = size; 1115 irq_q->qdma = qdma; 1116 1117 netif_napi_add_tx(eth->napi_dev, &irq_q->napi, 1118 airoha_qdma_tx_napi_poll); 1119 1120 airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr); 1121 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK, 1122 FIELD_PREP(TX_IRQ_DEPTH_MASK, size)); 1123 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK, 1124 FIELD_PREP(TX_IRQ_THR_MASK, 1)); 1125 1126 return 0; 1127 } 1128 1129 static int airoha_qdma_init_tx(struct airoha_qdma *qdma) 1130 { 1131 int i, err; 1132 1133 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1134 err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma, 1135 IRQ_QUEUE_LEN(i)); 1136 if (err) 1137 return err; 1138 } 1139 1140 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1141 err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma, 1142 TX_DSCP_NUM); 1143 if (err) 1144 return err; 1145 } 1146 1147 return 0; 1148 } 1149 1150 static void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q) 1151 { 1152 struct airoha_qdma *qdma = q->qdma; 1153 struct airoha_eth *eth = qdma->eth; 1154 int i, qid = q - &qdma->q_tx[0]; 1155 u16 index = 0; 1156 1157 spin_lock_bh(&q->lock); 1158 for (i = 0; i < q->ndesc; i++) { 1159 struct airoha_queue_entry *e = &q->entry[i]; 1160 struct airoha_qdma_desc *desc = &q->desc[i]; 1161 1162 if (!e->dma_addr) 1163 continue; 1164 1165 dma_unmap_single(eth->dev, e->dma_addr, e->dma_len, 1166 DMA_TO_DEVICE); 1167 dev_kfree_skb_any(e->skb); 1168 e->dma_addr = 0; 1169 e->skb = NULL; 1170 list_add_tail(&e->list, &q->tx_list); 1171 1172 /* Reset DMA descriptor */ 1173 WRITE_ONCE(desc->ctrl, 0); 1174 WRITE_ONCE(desc->addr, 0); 1175 WRITE_ONCE(desc->data, 0); 1176 WRITE_ONCE(desc->msg0, 0); 1177 WRITE_ONCE(desc->msg1, 0); 1178 WRITE_ONCE(desc->msg2, 0); 1179 1180 q->queued--; 1181 } 1182 1183 if (!list_empty(&q->tx_list)) { 1184 struct airoha_queue_entry *e; 1185 1186 e = list_first_entry(&q->tx_list, struct airoha_queue_entry, 1187 list); 1188 index = e - q->entry; 1189 } 1190 /* Set TX_DMA_IDX to TX_CPU_IDX to notify the hw the QDMA TX ring is 1191 * empty. 1192 */ 1193 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK, 1194 FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); 1195 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK, 1196 FIELD_PREP(TX_RING_DMA_IDX_MASK, index)); 1197 1198 spin_unlock_bh(&q->lock); 1199 } 1200 1201 static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma) 1202 { 1203 int size, index, num_desc = HW_DSCP_NUM; 1204 struct airoha_eth *eth = qdma->eth; 1205 int id = qdma - ð->qdma[0]; 1206 u32 status, buf_size; 1207 dma_addr_t dma_addr; 1208 const char *name; 1209 1210 name = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d-buf", id); 1211 if (!name) 1212 return -ENOMEM; 1213 1214 buf_size = id ? AIROHA_MAX_PACKET_SIZE / 2 : AIROHA_MAX_PACKET_SIZE; 1215 index = of_property_match_string(eth->dev->of_node, 1216 "memory-region-names", name); 1217 if (index >= 0) { 1218 struct reserved_mem *rmem; 1219 struct device_node *np; 1220 1221 /* Consume reserved memory for hw forwarding buffers queue if 1222 * available in the DTS 1223 */ 1224 np = of_parse_phandle(eth->dev->of_node, "memory-region", 1225 index); 1226 if (!np) 1227 return -ENODEV; 1228 1229 rmem = of_reserved_mem_lookup(np); 1230 of_node_put(np); 1231 if (!rmem) 1232 return -ENODEV; 1233 1234 dma_addr = rmem->base; 1235 /* Compute the number of hw descriptors according to the 1236 * reserved memory size and the payload buffer size 1237 */ 1238 num_desc = div_u64(rmem->size, buf_size); 1239 } else { 1240 size = buf_size * num_desc; 1241 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, 1242 GFP_KERNEL)) 1243 return -ENOMEM; 1244 } 1245 1246 airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr); 1247 1248 size = num_desc * sizeof(struct airoha_qdma_fwd_desc); 1249 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, GFP_KERNEL)) 1250 return -ENOMEM; 1251 1252 airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr); 1253 /* QDMA0: 2KB. QDMA1: 1KB */ 1254 airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG, 1255 HW_FWD_DSCP_PAYLOAD_SIZE_MASK, 1256 FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK, !!id)); 1257 airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK, 1258 FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128)); 1259 airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG, 1260 LMGR_INIT_START | LMGR_SRAM_MODE_MASK | 1261 HW_FWD_DESC_NUM_MASK, 1262 FIELD_PREP(HW_FWD_DESC_NUM_MASK, num_desc) | 1263 LMGR_INIT_START | LMGR_SRAM_MODE_MASK); 1264 1265 return read_poll_timeout(airoha_qdma_rr, status, 1266 !(status & LMGR_INIT_START), USEC_PER_MSEC, 1267 30 * USEC_PER_MSEC, true, qdma, 1268 REG_LMGR_INIT_CFG); 1269 } 1270 1271 static void airoha_qdma_init_qos(struct airoha_qdma *qdma) 1272 { 1273 airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK); 1274 airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK); 1275 1276 airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG, 1277 PSE_BUF_ESTIMATE_EN_MASK); 1278 1279 airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG, 1280 EGRESS_RATE_METER_EN_MASK | 1281 EGRESS_RATE_METER_EQ_RATE_EN_MASK); 1282 /* 2047us x 31 = 63.457ms */ 1283 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG, 1284 EGRESS_RATE_METER_WINDOW_SZ_MASK, 1285 FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK, 0x1f)); 1286 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG, 1287 EGRESS_RATE_METER_TIMESLICE_MASK, 1288 FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK, 0x7ff)); 1289 1290 /* ratelimit init */ 1291 airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK); 1292 /* fast-tick 25us */ 1293 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK, 1294 FIELD_PREP(GLB_FAST_TICK_MASK, 25)); 1295 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK, 1296 FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40)); 1297 1298 airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK); 1299 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK, 1300 FIELD_PREP(EGRESS_FAST_TICK_MASK, 25)); 1301 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, 1302 EGRESS_SLOW_TICK_RATIO_MASK, 1303 FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40)); 1304 1305 airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK); 1306 airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG, 1307 INGRESS_TRTCM_MODE_MASK); 1308 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK, 1309 FIELD_PREP(INGRESS_FAST_TICK_MASK, 125)); 1310 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, 1311 INGRESS_SLOW_TICK_RATIO_MASK, 1312 FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8)); 1313 1314 airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK); 1315 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK, 1316 FIELD_PREP(SLA_FAST_TICK_MASK, 25)); 1317 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK, 1318 FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40)); 1319 } 1320 1321 static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma) 1322 { 1323 int i; 1324 1325 for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) { 1326 /* Tx-cpu transferred count */ 1327 airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0); 1328 airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1), 1329 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK | 1330 CNTR_ALL_DSCP_RING_EN_MASK | 1331 FIELD_PREP(CNTR_CHAN_MASK, i)); 1332 /* Tx-fwd transferred count */ 1333 airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0); 1334 airoha_qdma_wr(qdma, REG_CNTR_CFG((i << 1) + 1), 1335 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK | 1336 CNTR_ALL_DSCP_RING_EN_MASK | 1337 FIELD_PREP(CNTR_SRC_MASK, 1) | 1338 FIELD_PREP(CNTR_CHAN_MASK, i)); 1339 } 1340 } 1341 1342 static int airoha_qdma_hw_init(struct airoha_qdma *qdma) 1343 { 1344 int i; 1345 1346 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 1347 /* clear pending irqs */ 1348 airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff); 1349 /* setup rx irqs */ 1350 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX0, 1351 INT_RX0_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1352 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX1, 1353 INT_RX1_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1354 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX2, 1355 INT_RX2_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1356 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX3, 1357 INT_RX3_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1358 } 1359 /* setup tx irqs */ 1360 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0, 1361 TX_COHERENT_LOW_INT_MASK | INT_TX_MASK); 1362 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX4, 1363 TX_COHERENT_HIGH_INT_MASK); 1364 1365 /* setup irq binding */ 1366 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1367 if (!qdma->q_tx[i].ndesc) 1368 continue; 1369 1370 if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i)) 1371 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i), 1372 TX_RING_IRQ_BLOCKING_CFG_MASK); 1373 else 1374 airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i), 1375 TX_RING_IRQ_BLOCKING_CFG_MASK); 1376 } 1377 1378 airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG, 1379 FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) | 1380 GLOBAL_CFG_CPU_TXR_RR_MASK | 1381 GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK | 1382 GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK | 1383 GLOBAL_CFG_MULTICAST_EN_MASK | 1384 GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK | 1385 GLOBAL_CFG_TX_WB_DONE_MASK | 1386 FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2)); 1387 1388 airoha_qdma_init_qos(qdma); 1389 1390 /* disable qdma rx delay interrupt */ 1391 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1392 if (!qdma->q_rx[i].ndesc) 1393 continue; 1394 1395 airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i), 1396 RX_DELAY_INT_MASK); 1397 } 1398 1399 airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG, 1400 TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN); 1401 airoha_qdma_init_qos_stats(qdma); 1402 1403 return 0; 1404 } 1405 1406 static irqreturn_t airoha_irq_handler(int irq, void *dev_instance) 1407 { 1408 struct airoha_irq_bank *irq_bank = dev_instance; 1409 struct airoha_qdma *qdma = irq_bank->qdma; 1410 u32 rx_intr_mask = 0, rx_intr1, rx_intr2; 1411 u32 intr[ARRAY_SIZE(irq_bank->irqmask)]; 1412 int i; 1413 1414 for (i = 0; i < ARRAY_SIZE(intr); i++) { 1415 intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i)); 1416 intr[i] &= irq_bank->irqmask[i]; 1417 airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]); 1418 } 1419 1420 if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state)) 1421 return IRQ_NONE; 1422 1423 rx_intr1 = intr[1] & RX_DONE_LOW_INT_MASK; 1424 if (rx_intr1) { 1425 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX1, rx_intr1); 1426 rx_intr_mask |= rx_intr1; 1427 } 1428 1429 rx_intr2 = intr[2] & RX_DONE_HIGH_INT_MASK; 1430 if (rx_intr2) { 1431 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX2, rx_intr2); 1432 rx_intr_mask |= (rx_intr2 << 16); 1433 } 1434 1435 for (i = 0; rx_intr_mask && i < ARRAY_SIZE(qdma->q_rx); i++) { 1436 if (!qdma->q_rx[i].ndesc) 1437 continue; 1438 1439 if (rx_intr_mask & BIT(i)) 1440 napi_schedule(&qdma->q_rx[i].napi); 1441 } 1442 1443 if (intr[0] & INT_TX_MASK) { 1444 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1445 if (!(intr[0] & TX_DONE_INT_MASK(i))) 1446 continue; 1447 1448 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX0, 1449 TX_DONE_INT_MASK(i)); 1450 napi_schedule(&qdma->q_tx_irq[i].napi); 1451 } 1452 } 1453 1454 return IRQ_HANDLED; 1455 } 1456 1457 static int airoha_qdma_init_irq_banks(struct platform_device *pdev, 1458 struct airoha_qdma *qdma) 1459 { 1460 struct airoha_eth *eth = qdma->eth; 1461 int i, id = qdma - ð->qdma[0]; 1462 1463 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 1464 struct airoha_irq_bank *irq_bank = &qdma->irq_banks[i]; 1465 int err, irq_index = 4 * id + i; 1466 const char *name; 1467 1468 spin_lock_init(&irq_bank->irq_lock); 1469 irq_bank->qdma = qdma; 1470 1471 irq_bank->irq = platform_get_irq(pdev, irq_index); 1472 if (irq_bank->irq < 0) 1473 return irq_bank->irq; 1474 1475 name = devm_kasprintf(eth->dev, GFP_KERNEL, 1476 KBUILD_MODNAME ".%d", irq_index); 1477 if (!name) 1478 return -ENOMEM; 1479 1480 err = devm_request_irq(eth->dev, irq_bank->irq, 1481 airoha_irq_handler, IRQF_SHARED, name, 1482 irq_bank); 1483 if (err) 1484 return err; 1485 } 1486 1487 return 0; 1488 } 1489 1490 static int airoha_qdma_init(struct platform_device *pdev, 1491 struct airoha_eth *eth, 1492 struct airoha_qdma *qdma) 1493 { 1494 int err, id = qdma - ð->qdma[0]; 1495 const char *res; 1496 1497 qdma->eth = eth; 1498 res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id); 1499 if (!res) 1500 return -ENOMEM; 1501 1502 qdma->regs = devm_platform_ioremap_resource_byname(pdev, res); 1503 if (IS_ERR(qdma->regs)) 1504 return dev_err_probe(eth->dev, PTR_ERR(qdma->regs), 1505 "failed to iomap qdma%d regs\n", id); 1506 1507 err = airoha_qdma_init_irq_banks(pdev, qdma); 1508 if (err) 1509 return err; 1510 1511 err = airoha_qdma_init_rx(qdma); 1512 if (err) 1513 return err; 1514 1515 err = airoha_qdma_init_tx(qdma); 1516 if (err) 1517 return err; 1518 1519 err = airoha_qdma_init_hfwd_queues(qdma); 1520 if (err) 1521 return err; 1522 1523 return airoha_qdma_hw_init(qdma); 1524 } 1525 1526 static void airoha_qdma_cleanup(struct airoha_qdma *qdma) 1527 { 1528 int i; 1529 1530 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1531 if (!qdma->q_rx[i].ndesc) 1532 continue; 1533 1534 netif_napi_del(&qdma->q_rx[i].napi); 1535 airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]); 1536 if (qdma->q_rx[i].page_pool) { 1537 page_pool_destroy(qdma->q_rx[i].page_pool); 1538 qdma->q_rx[i].page_pool = NULL; 1539 } 1540 } 1541 1542 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1543 if (!qdma->q_tx_irq[i].size) 1544 continue; 1545 1546 netif_napi_del(&qdma->q_tx_irq[i].napi); 1547 } 1548 1549 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1550 if (!qdma->q_tx[i].ndesc) 1551 continue; 1552 1553 airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]); 1554 } 1555 } 1556 1557 static int airoha_hw_init(struct platform_device *pdev, 1558 struct airoha_eth *eth) 1559 { 1560 int err, i; 1561 1562 /* disable xsi */ 1563 err = reset_control_bulk_assert(eth->soc->num_xsi_rsts, eth->xsi_rsts); 1564 if (err) 1565 return err; 1566 1567 err = reset_control_bulk_assert(ARRAY_SIZE(eth->rsts), eth->rsts); 1568 if (err) 1569 return err; 1570 1571 msleep(20); 1572 err = reset_control_bulk_deassert(ARRAY_SIZE(eth->rsts), eth->rsts); 1573 if (err) 1574 return err; 1575 1576 msleep(20); 1577 err = airoha_fe_init(eth); 1578 if (err) 1579 return err; 1580 1581 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 1582 err = airoha_qdma_init(pdev, eth, ð->qdma[i]); 1583 if (err) 1584 goto error; 1585 } 1586 1587 err = airoha_ppe_init(eth); 1588 if (err) 1589 goto error; 1590 1591 set_bit(DEV_STATE_INITIALIZED, ð->state); 1592 1593 return 0; 1594 error: 1595 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 1596 airoha_qdma_cleanup(ð->qdma[i]); 1597 1598 return err; 1599 } 1600 1601 static void airoha_hw_cleanup(struct airoha_eth *eth) 1602 { 1603 int i; 1604 1605 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 1606 airoha_qdma_cleanup(ð->qdma[i]); 1607 airoha_ppe_deinit(eth); 1608 } 1609 1610 static void airoha_qdma_start_napi(struct airoha_qdma *qdma) 1611 { 1612 int i; 1613 1614 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) 1615 napi_enable(&qdma->q_tx_irq[i].napi); 1616 1617 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1618 if (!qdma->q_rx[i].ndesc) 1619 continue; 1620 1621 napi_enable(&qdma->q_rx[i].napi); 1622 } 1623 } 1624 1625 static void airoha_qdma_stop_napi(struct airoha_qdma *qdma) 1626 { 1627 int i; 1628 1629 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) 1630 napi_disable(&qdma->q_tx_irq[i].napi); 1631 1632 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1633 if (!qdma->q_rx[i].ndesc) 1634 continue; 1635 1636 napi_disable(&qdma->q_rx[i].napi); 1637 } 1638 } 1639 1640 static void airoha_dev_get_hw_stats(struct airoha_gdm_dev *dev) 1641 { 1642 struct airoha_gdm_port *port = dev->port; 1643 struct airoha_eth *eth = dev->eth; 1644 u32 val, i = 0; 1645 1646 /* Read relevant MIB for GDM with multiple port attached */ 1647 if (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX) 1648 airoha_fe_rmw(eth, REG_FE_GDM_MIB_CFG(port->id), 1649 FE_TX_MIB_ID_MASK | FE_RX_MIB_ID_MASK, 1650 FIELD_PREP(FE_TX_MIB_ID_MASK, dev->nbq) | 1651 FIELD_PREP(FE_RX_MIB_ID_MASK, dev->nbq)); 1652 1653 u64_stats_update_begin(&dev->stats.syncp); 1654 1655 /* TX */ 1656 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_H(port->id)); 1657 dev->stats.tx_ok_pkts += ((u64)val << 32); 1658 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_L(port->id)); 1659 dev->stats.tx_ok_pkts += val; 1660 1661 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_H(port->id)); 1662 dev->stats.tx_ok_bytes += ((u64)val << 32); 1663 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_L(port->id)); 1664 dev->stats.tx_ok_bytes += val; 1665 1666 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_DROP_CNT(port->id)); 1667 dev->stats.tx_drops += val; 1668 1669 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_BC_CNT(port->id)); 1670 dev->stats.tx_broadcast += val; 1671 1672 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_MC_CNT(port->id)); 1673 dev->stats.tx_multicast += val; 1674 1675 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_RUNT_CNT(port->id)); 1676 dev->stats.tx_len[i] += val; 1677 1678 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_H(port->id)); 1679 dev->stats.tx_len[i] += ((u64)val << 32); 1680 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_L(port->id)); 1681 dev->stats.tx_len[i++] += val; 1682 1683 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_H(port->id)); 1684 dev->stats.tx_len[i] += ((u64)val << 32); 1685 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_L(port->id)); 1686 dev->stats.tx_len[i++] += val; 1687 1688 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_H(port->id)); 1689 dev->stats.tx_len[i] += ((u64)val << 32); 1690 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_L(port->id)); 1691 dev->stats.tx_len[i++] += val; 1692 1693 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_H(port->id)); 1694 dev->stats.tx_len[i] += ((u64)val << 32); 1695 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_L(port->id)); 1696 dev->stats.tx_len[i++] += val; 1697 1698 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_H(port->id)); 1699 dev->stats.tx_len[i] += ((u64)val << 32); 1700 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_L(port->id)); 1701 dev->stats.tx_len[i++] += val; 1702 1703 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_H(port->id)); 1704 dev->stats.tx_len[i] += ((u64)val << 32); 1705 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_L(port->id)); 1706 dev->stats.tx_len[i++] += val; 1707 1708 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_LONG_CNT(port->id)); 1709 dev->stats.tx_len[i++] += val; 1710 1711 /* RX */ 1712 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_H(port->id)); 1713 dev->stats.rx_ok_pkts += ((u64)val << 32); 1714 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_L(port->id)); 1715 dev->stats.rx_ok_pkts += val; 1716 1717 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_H(port->id)); 1718 dev->stats.rx_ok_bytes += ((u64)val << 32); 1719 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_L(port->id)); 1720 dev->stats.rx_ok_bytes += val; 1721 1722 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_DROP_CNT(port->id)); 1723 dev->stats.rx_drops += val; 1724 1725 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_BC_CNT(port->id)); 1726 dev->stats.rx_broadcast += val; 1727 1728 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_MC_CNT(port->id)); 1729 dev->stats.rx_multicast += val; 1730 1731 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ERROR_DROP_CNT(port->id)); 1732 dev->stats.rx_errors += val; 1733 1734 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_CRC_ERR_CNT(port->id)); 1735 dev->stats.rx_crc_error += val; 1736 1737 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OVERFLOW_DROP_CNT(port->id)); 1738 dev->stats.rx_over_errors += val; 1739 1740 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_FRAG_CNT(port->id)); 1741 dev->stats.rx_fragment += val; 1742 1743 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_JABBER_CNT(port->id)); 1744 dev->stats.rx_jabber += val; 1745 1746 i = 0; 1747 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_RUNT_CNT(port->id)); 1748 dev->stats.rx_len[i] += val; 1749 1750 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_H(port->id)); 1751 dev->stats.rx_len[i] += ((u64)val << 32); 1752 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_L(port->id)); 1753 dev->stats.rx_len[i++] += val; 1754 1755 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_H(port->id)); 1756 dev->stats.rx_len[i] += ((u64)val << 32); 1757 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_L(port->id)); 1758 dev->stats.rx_len[i++] += val; 1759 1760 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_H(port->id)); 1761 dev->stats.rx_len[i] += ((u64)val << 32); 1762 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_L(port->id)); 1763 dev->stats.rx_len[i++] += val; 1764 1765 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_H(port->id)); 1766 dev->stats.rx_len[i] += ((u64)val << 32); 1767 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_L(port->id)); 1768 dev->stats.rx_len[i++] += val; 1769 1770 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_H(port->id)); 1771 dev->stats.rx_len[i] += ((u64)val << 32); 1772 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_L(port->id)); 1773 dev->stats.rx_len[i++] += val; 1774 1775 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_H(port->id)); 1776 dev->stats.rx_len[i] += ((u64)val << 32); 1777 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_L(port->id)); 1778 dev->stats.rx_len[i++] += val; 1779 1780 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_LONG_CNT(port->id)); 1781 dev->stats.rx_len[i++] += val; 1782 1783 u64_stats_update_end(&dev->stats.syncp); 1784 } 1785 1786 static void airoha_update_hw_stats(struct airoha_gdm_dev *dev) 1787 { 1788 struct airoha_gdm_port *port = dev->port; 1789 int i; 1790 1791 spin_lock(&port->stats_lock); 1792 1793 for (i = 0; i < ARRAY_SIZE(port->devs); i++) { 1794 if (port->devs[i]) 1795 airoha_dev_get_hw_stats(port->devs[i]); 1796 } 1797 1798 /* Reset MIB counters */ 1799 airoha_fe_set(dev->eth, REG_FE_GDM_MIB_CLEAR(port->id), 1800 FE_GDM_MIB_RX_CLEAR_MASK | FE_GDM_MIB_TX_CLEAR_MASK); 1801 1802 spin_unlock(&port->stats_lock); 1803 } 1804 1805 static int airoha_dev_open(struct net_device *netdev) 1806 { 1807 int err, len = ETH_HLEN + netdev->mtu + ETH_FCS_LEN; 1808 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1809 struct airoha_gdm_port *port = dev->port; 1810 u32 cur_len, pse_port = FE_PSE_PORT_PPE1; 1811 struct airoha_qdma *qdma = dev->qdma; 1812 1813 netif_tx_start_all_queues(netdev); 1814 err = airoha_set_vip_for_gdm_port(dev, true); 1815 if (err) 1816 return err; 1817 1818 if (netdev_uses_dsa(netdev)) 1819 airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id), 1820 GDM_STAG_EN_MASK); 1821 else 1822 airoha_fe_clear(qdma->eth, REG_GDM_INGRESS_CFG(port->id), 1823 GDM_STAG_EN_MASK); 1824 1825 cur_len = airoha_fe_get(qdma->eth, REG_GDM_LEN_CFG(port->id), 1826 GDM_LONG_LEN_MASK); 1827 if (!port->users || len > cur_len) { 1828 /* Opening a sibling net_device with a larger MTU updates the 1829 * MTU of already running devices. This is required to allow 1830 * multiple net_devices with different MTUs to share the same 1831 * GDM port. 1832 */ 1833 airoha_fe_rmw(qdma->eth, REG_GDM_LEN_CFG(port->id), 1834 GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK, 1835 FIELD_PREP(GDM_SHORT_LEN_MASK, 60) | 1836 FIELD_PREP(GDM_LONG_LEN_MASK, len)); 1837 } 1838 port->users++; 1839 1840 airoha_qdma_set(qdma, REG_QDMA_GLOBAL_CFG, 1841 GLOBAL_CFG_TX_DMA_EN_MASK | 1842 GLOBAL_CFG_RX_DMA_EN_MASK); 1843 qdma->users++; 1844 1845 if (!airoha_is_lan_gdm_dev(dev) && 1846 airoha_ppe_is_enabled(qdma->eth, 1)) 1847 pse_port = FE_PSE_PORT_PPE2; 1848 airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id), 1849 pse_port); 1850 1851 return 0; 1852 } 1853 1854 static void airoha_set_port_mtu(struct airoha_eth *eth, 1855 struct airoha_gdm_port *port) 1856 { 1857 u32 len = 0; 1858 int i; 1859 1860 for (i = 0; i < ARRAY_SIZE(port->devs); i++) { 1861 struct airoha_gdm_dev *dev = port->devs[i]; 1862 struct net_device *netdev; 1863 1864 if (!dev) 1865 continue; 1866 1867 netdev = netdev_from_priv(dev); 1868 if (netif_running(netdev)) 1869 len = max_t(u32, len, netdev->mtu); 1870 } 1871 len += ETH_HLEN + ETH_FCS_LEN; 1872 1873 airoha_fe_rmw(eth, REG_GDM_LEN_CFG(port->id), 1874 GDM_LONG_LEN_MASK, 1875 FIELD_PREP(GDM_LONG_LEN_MASK, len)); 1876 } 1877 1878 static int airoha_dev_stop(struct net_device *netdev) 1879 { 1880 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1881 struct airoha_gdm_port *port = dev->port; 1882 struct airoha_qdma *qdma = dev->qdma; 1883 int i; 1884 1885 netif_tx_disable(netdev); 1886 airoha_set_vip_for_gdm_port(dev, false); 1887 for (i = 0; i < netdev->num_tx_queues; i++) 1888 netdev_tx_reset_subqueue(netdev, i); 1889 1890 if (--port->users) 1891 airoha_set_port_mtu(dev->eth, port); 1892 else 1893 airoha_set_gdm_port_fwd_cfg(qdma->eth, 1894 REG_GDM_FWD_CFG(port->id), 1895 FE_PSE_PORT_DROP); 1896 1897 if (!--qdma->users) { 1898 airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG, 1899 GLOBAL_CFG_TX_DMA_EN_MASK | 1900 GLOBAL_CFG_RX_DMA_EN_MASK); 1901 1902 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1903 if (!qdma->q_tx[i].ndesc) 1904 continue; 1905 1906 airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]); 1907 } 1908 } 1909 1910 return 0; 1911 } 1912 1913 static int airoha_dev_set_macaddr(struct net_device *netdev, void *p) 1914 { 1915 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1916 struct sockaddr *addr = p; 1917 int err; 1918 1919 err = eth_prepare_mac_addr_change(netdev, p); 1920 if (err) 1921 return err; 1922 1923 err = airoha_set_macaddr(dev, addr->sa_data); 1924 if (err) 1925 return err; 1926 1927 eth_commit_mac_addr_change(netdev, p); 1928 1929 return 0; 1930 } 1931 1932 static int airoha_enable_gdm2_loopback(struct airoha_gdm_dev *dev) 1933 { 1934 struct airoha_gdm_port *port = dev->port; 1935 struct airoha_eth *eth = dev->eth; 1936 u32 val, pse_port, chan; 1937 int i, src_port; 1938 1939 src_port = eth->soc->ops.get_sport(port, dev->nbq); 1940 if (src_port < 0) 1941 return src_port; 1942 1943 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1944 FE_PSE_PORT_DROP); 1945 airoha_fe_clear(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1946 GDM_STRIP_CRC_MASK); 1947 1948 /* Enable GDM2 loopback */ 1949 airoha_fe_wr(eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), 0xffffffff); 1950 airoha_fe_wr(eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX), 0xffff); 1951 1952 chan = port->id == AIROHA_GDM3_IDX ? airoha_is_7581(eth) ? 4 : 3 : 0; 1953 airoha_fe_rmw(eth, REG_GDM_LPBK_CFG(AIROHA_GDM2_IDX), 1954 LPBK_CHAN_MASK | LPBK_MODE_MASK | LPBK_EN_MASK, 1955 FIELD_PREP(LPBK_CHAN_MASK, chan) | 1956 LBK_GAP_MODE_MASK | LBK_LEN_MODE_MASK | 1957 LBK_CHAN_MODE_MASK | LPBK_EN_MASK); 1958 airoha_fe_rmw(eth, REG_GDM_LEN_CFG(AIROHA_GDM2_IDX), 1959 GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK, 1960 FIELD_PREP(GDM_SHORT_LEN_MASK, 60) | 1961 FIELD_PREP(GDM_LONG_LEN_MASK, AIROHA_MAX_MTU)); 1962 /* Forward the traffic to the proper GDM port */ 1963 pse_port = port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3 1964 : FE_PSE_PORT_GDM4; 1965 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1966 pse_port); 1967 1968 /* Disable VIP and IFC for GDM2 */ 1969 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX)); 1970 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX)); 1971 1972 airoha_fe_rmw(eth, REG_FE_WAN_PORT, 1973 WAN1_EN_MASK | WAN1_MASK | WAN0_MASK, 1974 FIELD_PREP(WAN0_MASK, src_port)); 1975 val = src_port & SP_CPORT_DFT_MASK; 1976 airoha_fe_rmw(eth, 1977 REG_SP_DFT_CPORT(src_port >> fls(SP_CPORT_DFT_MASK)), 1978 SP_CPORT_MASK(val), 1979 __field_prep(SP_CPORT_MASK(val), FE_PSE_PORT_CDM2)); 1980 1981 for (i = 0; i < eth->soc->num_ppe; i++) 1982 airoha_ppe_set_cpu_port(dev, i, AIROHA_GDM2_IDX); 1983 1984 if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) { 1985 u32 mask = FC_ID_OF_SRC_PORT_MASK(dev->nbq); 1986 1987 airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask, 1988 __field_prep(mask, AIROHA_GDM2_IDX)); 1989 } 1990 1991 return 0; 1992 } 1993 1994 static struct airoha_gdm_dev * 1995 airoha_get_wan_gdm_dev(struct airoha_eth *eth) 1996 { 1997 int i; 1998 1999 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 2000 struct airoha_gdm_port *port = eth->ports[i]; 2001 int j; 2002 2003 if (!port) 2004 continue; 2005 2006 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 2007 struct airoha_gdm_dev *dev = port->devs[j]; 2008 2009 if (dev && !airoha_is_lan_gdm_dev(dev)) 2010 return dev; 2011 } 2012 } 2013 2014 return NULL; 2015 } 2016 2017 static void airoha_dev_set_qdma(struct airoha_gdm_dev *dev) 2018 { 2019 struct net_device *netdev = netdev_from_priv(dev); 2020 struct airoha_eth *eth = dev->eth; 2021 int ppe_id; 2022 2023 /* QDMA0 is used for lan ports while QDMA1 is used for WAN ports */ 2024 dev->qdma = ð->qdma[!airoha_is_lan_gdm_dev(dev)]; 2025 netdev->irq = dev->qdma->irq_banks[0].irq; 2026 2027 ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1); 2028 airoha_ppe_set_cpu_port(dev, ppe_id, airoha_get_fe_port(dev)); 2029 } 2030 2031 static int airoha_dev_init(struct net_device *netdev) 2032 { 2033 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2034 struct airoha_gdm_port *port = dev->port; 2035 2036 switch (port->id) { 2037 case AIROHA_GDM3_IDX: 2038 case AIROHA_GDM4_IDX: 2039 if (airoha_get_wan_gdm_dev(dev->eth)) 2040 break; 2041 fallthrough; 2042 case AIROHA_GDM2_IDX: 2043 /* GDM2 is always used as wan */ 2044 dev->flags |= AIROHA_PRIV_F_WAN; 2045 break; 2046 default: 2047 break; 2048 } 2049 2050 airoha_dev_set_qdma(dev); 2051 airoha_set_macaddr(dev, netdev->dev_addr); 2052 2053 if (!airoha_is_lan_gdm_dev(dev) && 2054 (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX)) { 2055 int err; 2056 2057 err = airoha_enable_gdm2_loopback(dev); 2058 if (err) 2059 return err; 2060 } 2061 2062 return 0; 2063 } 2064 2065 static void airoha_dev_get_stats64(struct net_device *netdev, 2066 struct rtnl_link_stats64 *storage) 2067 { 2068 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2069 unsigned int start; 2070 2071 airoha_update_hw_stats(dev); 2072 do { 2073 start = u64_stats_fetch_begin(&dev->stats.syncp); 2074 storage->rx_packets = dev->stats.rx_ok_pkts; 2075 storage->tx_packets = dev->stats.tx_ok_pkts; 2076 storage->rx_bytes = dev->stats.rx_ok_bytes; 2077 storage->tx_bytes = dev->stats.tx_ok_bytes; 2078 storage->multicast = dev->stats.rx_multicast; 2079 storage->rx_errors = dev->stats.rx_errors; 2080 storage->rx_dropped = dev->stats.rx_drops; 2081 storage->tx_dropped = dev->stats.tx_drops; 2082 storage->rx_crc_errors = dev->stats.rx_crc_error; 2083 storage->rx_over_errors = dev->stats.rx_over_errors; 2084 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2085 } 2086 2087 static int airoha_dev_change_mtu(struct net_device *netdev, int mtu) 2088 { 2089 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2090 struct airoha_gdm_port *port = dev->port; 2091 2092 WRITE_ONCE(netdev->mtu, mtu); 2093 if (port->users) 2094 airoha_set_port_mtu(dev->eth, port); 2095 2096 return 0; 2097 } 2098 2099 static u16 airoha_dev_select_queue(struct net_device *netdev, 2100 struct sk_buff *skb, 2101 struct net_device *sb_dev) 2102 { 2103 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2104 struct airoha_gdm_port *port = dev->port; 2105 int queue, channel; 2106 2107 /* For dsa device select QoS channel according to the dsa user port 2108 * index, rely on port id otherwise. Select QoS queue based on the 2109 * skb priority. 2110 */ 2111 channel = netdev_uses_dsa(netdev) ? skb_get_queue_mapping(skb) : port->id; 2112 channel = channel % AIROHA_NUM_QOS_CHANNELS; 2113 queue = (skb->priority - 1) % AIROHA_NUM_QOS_QUEUES; /* QoS queue */ 2114 queue = channel * AIROHA_NUM_QOS_QUEUES + queue; 2115 2116 return queue < netdev->num_tx_queues ? queue : 0; 2117 } 2118 2119 static u32 airoha_get_dsa_tag(struct sk_buff *skb, struct net_device *dev) 2120 { 2121 #if IS_ENABLED(CONFIG_NET_DSA) 2122 struct ethhdr *ehdr; 2123 u8 xmit_tpid; 2124 u16 tag; 2125 2126 if (!netdev_uses_dsa(dev)) 2127 return 0; 2128 2129 if (dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 2130 return 0; 2131 2132 if (skb_cow_head(skb, 0)) 2133 return 0; 2134 2135 ehdr = (struct ethhdr *)skb->data; 2136 tag = be16_to_cpu(ehdr->h_proto); 2137 xmit_tpid = tag >> 8; 2138 2139 switch (xmit_tpid) { 2140 case MTK_HDR_XMIT_TAGGED_TPID_8100: 2141 ehdr->h_proto = cpu_to_be16(ETH_P_8021Q); 2142 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_8100 << 8); 2143 break; 2144 case MTK_HDR_XMIT_TAGGED_TPID_88A8: 2145 ehdr->h_proto = cpu_to_be16(ETH_P_8021AD); 2146 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_88A8 << 8); 2147 break; 2148 default: 2149 /* PPE module requires untagged DSA packets to work properly, 2150 * so move DSA tag to DMA descriptor. 2151 */ 2152 memmove(skb->data + MTK_HDR_LEN, skb->data, 2 * ETH_ALEN); 2153 __skb_pull(skb, MTK_HDR_LEN); 2154 break; 2155 } 2156 2157 return tag; 2158 #else 2159 return 0; 2160 #endif 2161 } 2162 2163 int airoha_get_fe_port(struct airoha_gdm_dev *dev) 2164 { 2165 struct airoha_gdm_port *port = dev->port; 2166 struct airoha_eth *eth = dev->eth; 2167 2168 switch (eth->soc->version) { 2169 case 0x7583: 2170 return port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3 2171 : port->id; 2172 case 0x7581: 2173 default: 2174 return port->id == AIROHA_GDM4_IDX ? FE_PSE_PORT_GDM4 2175 : port->id; 2176 } 2177 } 2178 2179 static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, 2180 struct net_device *netdev) 2181 { 2182 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2183 struct airoha_qdma *qdma = dev->qdma; 2184 u32 nr_frags, tag, msg0, msg1, len; 2185 struct airoha_queue_entry *e; 2186 struct netdev_queue *txq; 2187 struct airoha_queue *q; 2188 LIST_HEAD(tx_list); 2189 int i = 0, qid; 2190 void *data; 2191 u16 index; 2192 u8 fport; 2193 2194 qid = airoha_qdma_get_txq(qdma, skb_get_queue_mapping(skb)); 2195 tag = airoha_get_dsa_tag(skb, netdev); 2196 2197 msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK, 2198 qid / AIROHA_NUM_QOS_QUEUES) | 2199 FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK, 2200 qid % AIROHA_NUM_QOS_QUEUES) | 2201 FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK, tag); 2202 if (skb->ip_summed == CHECKSUM_PARTIAL) 2203 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TCO_MASK, 1) | 2204 FIELD_PREP(QDMA_ETH_TXMSG_UCO_MASK, 1) | 2205 FIELD_PREP(QDMA_ETH_TXMSG_ICO_MASK, 1); 2206 2207 /* TSO: fill MSS info in tcp checksum field */ 2208 if (skb_is_gso(skb)) { 2209 if (skb_cow_head(skb, 0)) 2210 goto error; 2211 2212 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | 2213 SKB_GSO_TCPV6)) { 2214 __be16 csum = cpu_to_be16(skb_shinfo(skb)->gso_size); 2215 2216 tcp_hdr(skb)->check = (__force __sum16)csum; 2217 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TSO_MASK, 1); 2218 } 2219 } 2220 2221 fport = airoha_get_fe_port(dev); 2222 msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) | 2223 FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) | 2224 FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f); 2225 2226 q = &qdma->q_tx[qid]; 2227 if (WARN_ON_ONCE(!q->ndesc)) 2228 goto error; 2229 2230 spin_lock_bh(&q->lock); 2231 2232 txq = skb_get_tx_queue(netdev, skb); 2233 nr_frags = 1 + skb_shinfo(skb)->nr_frags; 2234 2235 if (q->queued + nr_frags >= q->ndesc) { 2236 /* not enough space in the queue */ 2237 netif_tx_stop_queue(txq); 2238 q->txq_stopped = true; 2239 spin_unlock_bh(&q->lock); 2240 return NETDEV_TX_BUSY; 2241 } 2242 2243 len = skb_headlen(skb); 2244 data = skb->data; 2245 2246 e = list_first_entry(&q->tx_list, struct airoha_queue_entry, 2247 list); 2248 index = e - q->entry; 2249 2250 while (true) { 2251 struct airoha_qdma_desc *desc = &q->desc[index]; 2252 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2253 dma_addr_t addr; 2254 u32 val; 2255 2256 addr = dma_map_single(netdev->dev.parent, data, len, 2257 DMA_TO_DEVICE); 2258 if (unlikely(dma_mapping_error(netdev->dev.parent, addr))) 2259 goto error_unmap; 2260 2261 list_move_tail(&e->list, &tx_list); 2262 e->skb = i == nr_frags - 1 ? skb : NULL; 2263 e->dma_addr = addr; 2264 e->dma_len = len; 2265 2266 e = list_first_entry(&q->tx_list, struct airoha_queue_entry, 2267 list); 2268 index = e - q->entry; 2269 2270 val = FIELD_PREP(QDMA_DESC_LEN_MASK, len); 2271 if (i < nr_frags - 1) 2272 val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1); 2273 WRITE_ONCE(desc->ctrl, cpu_to_le32(val)); 2274 WRITE_ONCE(desc->addr, cpu_to_le32(addr)); 2275 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, index); 2276 WRITE_ONCE(desc->data, cpu_to_le32(val)); 2277 WRITE_ONCE(desc->msg0, cpu_to_le32(msg0)); 2278 WRITE_ONCE(desc->msg1, cpu_to_le32(msg1)); 2279 WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff)); 2280 2281 if (++i == nr_frags) 2282 break; 2283 2284 data = skb_frag_address(frag); 2285 len = skb_frag_size(frag); 2286 } 2287 q->queued += i; 2288 2289 skb_tx_timestamp(skb); 2290 netdev_tx_sent_queue(txq, skb->len); 2291 if (q->ndesc - q->queued < q->free_thr) { 2292 netif_tx_stop_queue(txq); 2293 q->txq_stopped = true; 2294 } 2295 2296 if (netif_xmit_stopped(txq) || !netdev_xmit_more()) 2297 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), 2298 TX_RING_CPU_IDX_MASK, 2299 FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); 2300 2301 spin_unlock_bh(&q->lock); 2302 2303 return NETDEV_TX_OK; 2304 2305 error_unmap: 2306 list_for_each_entry(e, &tx_list, list) { 2307 dma_unmap_single(netdev->dev.parent, e->dma_addr, e->dma_len, 2308 DMA_TO_DEVICE); 2309 e->dma_addr = 0; 2310 } 2311 list_splice(&tx_list, &q->tx_list); 2312 2313 spin_unlock_bh(&q->lock); 2314 error: 2315 dev_kfree_skb_any(skb); 2316 netdev->stats.tx_dropped++; 2317 2318 return NETDEV_TX_OK; 2319 } 2320 2321 static void airoha_ethtool_get_drvinfo(struct net_device *netdev, 2322 struct ethtool_drvinfo *info) 2323 { 2324 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2325 struct airoha_eth *eth = dev->eth; 2326 2327 strscpy(info->driver, eth->dev->driver->name, sizeof(info->driver)); 2328 strscpy(info->bus_info, dev_name(eth->dev), sizeof(info->bus_info)); 2329 } 2330 2331 static void airoha_ethtool_get_mac_stats(struct net_device *netdev, 2332 struct ethtool_eth_mac_stats *stats) 2333 { 2334 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2335 unsigned int start; 2336 2337 airoha_update_hw_stats(dev); 2338 do { 2339 start = u64_stats_fetch_begin(&dev->stats.syncp); 2340 stats->FramesTransmittedOK = dev->stats.tx_ok_pkts; 2341 stats->OctetsTransmittedOK = dev->stats.tx_ok_bytes; 2342 stats->MulticastFramesXmittedOK = dev->stats.tx_multicast; 2343 stats->BroadcastFramesXmittedOK = dev->stats.tx_broadcast; 2344 stats->FramesReceivedOK = dev->stats.rx_ok_pkts; 2345 stats->OctetsReceivedOK = dev->stats.rx_ok_bytes; 2346 stats->BroadcastFramesReceivedOK = dev->stats.rx_broadcast; 2347 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2348 } 2349 2350 static const struct ethtool_rmon_hist_range airoha_ethtool_rmon_ranges[] = { 2351 { 0, 64 }, 2352 { 65, 127 }, 2353 { 128, 255 }, 2354 { 256, 511 }, 2355 { 512, 1023 }, 2356 { 1024, 1518 }, 2357 { 1519, 10239 }, 2358 {}, 2359 }; 2360 2361 static void 2362 airoha_ethtool_get_rmon_stats(struct net_device *netdev, 2363 struct ethtool_rmon_stats *stats, 2364 const struct ethtool_rmon_hist_range **ranges) 2365 { 2366 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2367 struct airoha_hw_stats *hw_stats = &dev->stats; 2368 unsigned int start; 2369 2370 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) != 2371 ARRAY_SIZE(hw_stats->tx_len) + 1); 2372 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) != 2373 ARRAY_SIZE(hw_stats->rx_len) + 1); 2374 2375 *ranges = airoha_ethtool_rmon_ranges; 2376 airoha_update_hw_stats(dev); 2377 do { 2378 int i; 2379 2380 start = u64_stats_fetch_begin(&dev->stats.syncp); 2381 stats->fragments = hw_stats->rx_fragment; 2382 stats->jabbers = hw_stats->rx_jabber; 2383 for (i = 0; i < ARRAY_SIZE(airoha_ethtool_rmon_ranges) - 1; 2384 i++) { 2385 stats->hist[i] = hw_stats->rx_len[i]; 2386 stats->hist_tx[i] = hw_stats->tx_len[i]; 2387 } 2388 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2389 } 2390 2391 static int airoha_qdma_set_chan_tx_sched(struct net_device *netdev, 2392 int channel, enum tx_sched_mode mode, 2393 const u16 *weights, u8 n_weights) 2394 { 2395 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2396 int i; 2397 2398 for (i = 0; i < AIROHA_NUM_TX_RING; i++) 2399 airoha_qdma_clear(dev->qdma, REG_QUEUE_CLOSE_CFG(channel), 2400 TXQ_DISABLE_CHAN_QUEUE_MASK(channel, i)); 2401 2402 for (i = 0; i < n_weights; i++) { 2403 u32 status; 2404 int err; 2405 2406 airoha_qdma_wr(dev->qdma, REG_TXWRR_WEIGHT_CFG, 2407 TWRR_RW_CMD_MASK | 2408 FIELD_PREP(TWRR_CHAN_IDX_MASK, channel) | 2409 FIELD_PREP(TWRR_QUEUE_IDX_MASK, i) | 2410 FIELD_PREP(TWRR_VALUE_MASK, weights[i])); 2411 err = read_poll_timeout(airoha_qdma_rr, status, 2412 status & TWRR_RW_CMD_DONE, 2413 USEC_PER_MSEC, 10 * USEC_PER_MSEC, 2414 true, dev->qdma, REG_TXWRR_WEIGHT_CFG); 2415 if (err) 2416 return err; 2417 } 2418 2419 airoha_qdma_rmw(dev->qdma, REG_CHAN_QOS_MODE(channel >> 3), 2420 CHAN_QOS_MODE_MASK(channel), 2421 __field_prep(CHAN_QOS_MODE_MASK(channel), mode)); 2422 2423 return 0; 2424 } 2425 2426 static int airoha_qdma_set_tx_prio_sched(struct net_device *dev, int channel) 2427 { 2428 static const u16 w[AIROHA_NUM_QOS_QUEUES] = {}; 2429 2430 return airoha_qdma_set_chan_tx_sched(dev, channel, TC_SCH_SP, w, 2431 ARRAY_SIZE(w)); 2432 } 2433 2434 static int airoha_qdma_set_tx_ets_sched(struct net_device *dev, int channel, 2435 struct tc_ets_qopt_offload *opt) 2436 { 2437 struct tc_ets_qopt_offload_replace_params *p = &opt->replace_params; 2438 enum tx_sched_mode mode = TC_SCH_SP; 2439 u16 w[AIROHA_NUM_QOS_QUEUES] = {}; 2440 int i, nstrict = 0; 2441 2442 if (p->bands > AIROHA_NUM_QOS_QUEUES) 2443 return -EINVAL; 2444 2445 for (i = 0; i < p->bands; i++) { 2446 if (!p->quanta[i]) 2447 nstrict++; 2448 } 2449 2450 /* this configuration is not supported by the hw */ 2451 if (nstrict == AIROHA_NUM_QOS_QUEUES - 1) 2452 return -EINVAL; 2453 2454 /* EN7581 SoC supports fixed QoS band priority where WRR queues have 2455 * lowest priorities with respect to SP ones. 2456 * e.g: WRR0, WRR1, .., WRRm, SP0, SP1, .., SPn 2457 */ 2458 for (i = 0; i < nstrict; i++) { 2459 if (p->priomap[p->bands - i - 1] != i) 2460 return -EINVAL; 2461 } 2462 2463 for (i = 0; i < p->bands - nstrict; i++) { 2464 if (p->priomap[i] != nstrict + i) 2465 return -EINVAL; 2466 2467 w[i] = p->weights[nstrict + i]; 2468 } 2469 2470 if (!nstrict) 2471 mode = TC_SCH_WRR8; 2472 else if (nstrict < AIROHA_NUM_QOS_QUEUES - 1) 2473 mode = nstrict + 1; 2474 2475 return airoha_qdma_set_chan_tx_sched(dev, channel, mode, w, 2476 ARRAY_SIZE(w)); 2477 } 2478 2479 static int airoha_qdma_get_tx_ets_stats(struct net_device *netdev, int channel, 2480 struct tc_ets_qopt_offload *opt) 2481 { 2482 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2483 struct airoha_qdma *qdma = dev->qdma; 2484 2485 u64 cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1)); 2486 u64 fwd_tx_packets = airoha_qdma_rr(qdma, 2487 REG_CNTR_VAL((channel << 1) + 1)); 2488 u64 tx_packets = (cpu_tx_packets - dev->cpu_tx_packets) + 2489 (fwd_tx_packets - dev->fwd_tx_packets); 2490 2491 _bstats_update(opt->stats.bstats, 0, tx_packets); 2492 dev->cpu_tx_packets = cpu_tx_packets; 2493 dev->fwd_tx_packets = fwd_tx_packets; 2494 2495 return 0; 2496 } 2497 2498 static int airoha_tc_setup_qdisc_ets(struct net_device *dev, 2499 struct tc_ets_qopt_offload *opt) 2500 { 2501 int channel; 2502 2503 if (opt->parent == TC_H_ROOT) 2504 return -EINVAL; 2505 2506 channel = TC_H_MAJ(opt->handle) >> 16; 2507 channel = channel % AIROHA_NUM_QOS_CHANNELS; 2508 2509 switch (opt->command) { 2510 case TC_ETS_REPLACE: 2511 return airoha_qdma_set_tx_ets_sched(dev, channel, opt); 2512 case TC_ETS_DESTROY: 2513 /* PRIO is default qdisc scheduler */ 2514 return airoha_qdma_set_tx_prio_sched(dev, channel); 2515 case TC_ETS_STATS: 2516 return airoha_qdma_get_tx_ets_stats(dev, channel, opt); 2517 default: 2518 return -EOPNOTSUPP; 2519 } 2520 } 2521 2522 static int airoha_qdma_get_rl_param(struct airoha_qdma *qdma, int queue_id, 2523 u32 addr, enum trtcm_param_type param, 2524 u32 *val_low, u32 *val_high) 2525 { 2526 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id); 2527 u32 val, config = FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) | 2528 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) | 2529 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx); 2530 2531 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2532 if (read_poll_timeout(airoha_qdma_rr, val, 2533 val & RATE_LIMIT_PARAM_RW_DONE_MASK, 2534 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, qdma, 2535 REG_TRTCM_CFG_PARAM(addr))) 2536 return -ETIMEDOUT; 2537 2538 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr)); 2539 if (val_high) 2540 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr)); 2541 2542 return 0; 2543 } 2544 2545 static int airoha_qdma_set_rl_param(struct airoha_qdma *qdma, int queue_id, 2546 u32 addr, enum trtcm_param_type param, 2547 u32 val) 2548 { 2549 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id); 2550 u32 config = RATE_LIMIT_PARAM_RW_MASK | 2551 FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) | 2552 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) | 2553 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx); 2554 2555 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); 2556 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2557 2558 return read_poll_timeout(airoha_qdma_rr, val, 2559 val & RATE_LIMIT_PARAM_RW_DONE_MASK, 2560 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2561 qdma, REG_TRTCM_CFG_PARAM(addr)); 2562 } 2563 2564 static int airoha_qdma_set_rl_config(struct airoha_qdma *qdma, int queue_id, 2565 u32 addr, bool enable, u32 enable_mask) 2566 { 2567 u32 val; 2568 int err; 2569 2570 err = airoha_qdma_get_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE, 2571 &val, NULL); 2572 if (err) 2573 return err; 2574 2575 val = enable ? val | enable_mask : val & ~enable_mask; 2576 2577 return airoha_qdma_set_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE, 2578 val); 2579 } 2580 2581 static int airoha_qdma_set_rl_token_bucket(struct airoha_qdma *qdma, 2582 int queue_id, u32 rate_val, 2583 u32 bucket_size) 2584 { 2585 u32 val, config, tick, unit, rate, rate_frac; 2586 int err; 2587 2588 err = airoha_qdma_get_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2589 TRTCM_MISC_MODE, &config, NULL); 2590 if (err) 2591 return err; 2592 2593 val = airoha_qdma_rr(qdma, REG_INGRESS_TRTCM_CFG); 2594 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val); 2595 if (config & TRTCM_TICK_SEL) 2596 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val); 2597 if (!tick) 2598 return -EINVAL; 2599 2600 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick; 2601 if (!unit) 2602 return -EINVAL; 2603 2604 rate = rate_val / unit; 2605 rate_frac = rate_val % unit; 2606 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit; 2607 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) | 2608 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac); 2609 2610 err = airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2611 TRTCM_TOKEN_RATE_MODE, rate); 2612 if (err) 2613 return err; 2614 2615 val = bucket_size; 2616 if (!(config & TRTCM_PKT_MODE)) 2617 val = max_t(u32, val, MIN_TOKEN_SIZE); 2618 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET); 2619 2620 return airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2621 TRTCM_BUCKETSIZE_SHIFT_MODE, val); 2622 } 2623 2624 static int airoha_qdma_init_rl_config(struct airoha_qdma *qdma, int queue_id, 2625 bool enable, enum trtcm_unit_type unit) 2626 { 2627 bool tick_sel = queue_id == 0 || queue_id == 2 || queue_id == 8; 2628 enum trtcm_param mode = TRTCM_METER_MODE; 2629 int err; 2630 2631 mode |= unit == TRTCM_PACKET_UNIT ? TRTCM_PKT_MODE : 0; 2632 err = airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2633 enable, mode); 2634 if (err) 2635 return err; 2636 2637 return airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2638 tick_sel, TRTCM_TICK_SEL); 2639 } 2640 2641 static int airoha_qdma_get_trtcm_param(struct airoha_qdma *qdma, int channel, 2642 u32 addr, enum trtcm_param_type param, 2643 enum trtcm_mode_type mode, 2644 u32 *val_low, u32 *val_high) 2645 { 2646 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel); 2647 u32 val, config = FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) | 2648 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) | 2649 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) | 2650 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode); 2651 2652 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2653 if (read_poll_timeout(airoha_qdma_rr, val, 2654 val & TRTCM_PARAM_RW_DONE_MASK, 2655 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2656 qdma, REG_TRTCM_CFG_PARAM(addr))) 2657 return -ETIMEDOUT; 2658 2659 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr)); 2660 if (val_high) 2661 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr)); 2662 2663 return 0; 2664 } 2665 2666 static int airoha_qdma_set_trtcm_param(struct airoha_qdma *qdma, int channel, 2667 u32 addr, enum trtcm_param_type param, 2668 enum trtcm_mode_type mode, u32 val) 2669 { 2670 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel); 2671 u32 config = TRTCM_PARAM_RW_MASK | 2672 FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) | 2673 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) | 2674 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) | 2675 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode); 2676 2677 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); 2678 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2679 2680 return read_poll_timeout(airoha_qdma_rr, val, 2681 val & TRTCM_PARAM_RW_DONE_MASK, 2682 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2683 qdma, REG_TRTCM_CFG_PARAM(addr)); 2684 } 2685 2686 static int airoha_qdma_set_trtcm_config(struct airoha_qdma *qdma, int channel, 2687 u32 addr, enum trtcm_mode_type mode, 2688 bool enable, u32 enable_mask) 2689 { 2690 u32 val; 2691 2692 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2693 mode, &val, NULL)) 2694 return -EINVAL; 2695 2696 val = enable ? val | enable_mask : val & ~enable_mask; 2697 2698 return airoha_qdma_set_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2699 mode, val); 2700 } 2701 2702 static int airoha_qdma_set_trtcm_token_bucket(struct airoha_qdma *qdma, 2703 int channel, u32 addr, 2704 enum trtcm_mode_type mode, 2705 u32 rate_val, u32 bucket_size) 2706 { 2707 u32 val, config, tick, unit, rate, rate_frac; 2708 int err; 2709 2710 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2711 mode, &config, NULL)) 2712 return -EINVAL; 2713 2714 val = airoha_qdma_rr(qdma, addr); 2715 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val); 2716 if (config & TRTCM_TICK_SEL) 2717 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val); 2718 if (!tick) 2719 return -EINVAL; 2720 2721 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick; 2722 if (!unit) 2723 return -EINVAL; 2724 2725 rate = rate_val / unit; 2726 rate_frac = rate_val % unit; 2727 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit; 2728 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) | 2729 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac); 2730 2731 err = airoha_qdma_set_trtcm_param(qdma, channel, addr, 2732 TRTCM_TOKEN_RATE_MODE, mode, rate); 2733 if (err) 2734 return err; 2735 2736 val = max_t(u32, bucket_size, MIN_TOKEN_SIZE); 2737 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET); 2738 2739 return airoha_qdma_set_trtcm_param(qdma, channel, addr, 2740 TRTCM_BUCKETSIZE_SHIFT_MODE, 2741 mode, val); 2742 } 2743 2744 static int airoha_qdma_set_tx_rate_limit(struct net_device *netdev, 2745 int channel, u32 rate, 2746 u32 bucket_size) 2747 { 2748 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2749 int i, err; 2750 2751 for (i = 0; i <= TRTCM_PEAK_MODE; i++) { 2752 err = airoha_qdma_set_trtcm_config(dev->qdma, channel, 2753 REG_EGRESS_TRTCM_CFG, i, 2754 !!rate, TRTCM_METER_MODE); 2755 if (err) 2756 return err; 2757 2758 err = airoha_qdma_set_trtcm_token_bucket(dev->qdma, channel, 2759 REG_EGRESS_TRTCM_CFG, 2760 i, rate, bucket_size); 2761 if (err) 2762 return err; 2763 } 2764 2765 return 0; 2766 } 2767 2768 static int airoha_tc_htb_modify_queue(struct net_device *dev, 2769 struct tc_htb_qopt_offload *opt) 2770 { 2771 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 2772 u32 rate = div_u64(opt->rate, 1000) << 3; /* kbps */ 2773 int err; 2774 2775 if (opt->parent_classid != TC_HTB_CLASSID_ROOT) { 2776 NL_SET_ERR_MSG_MOD(opt->extack, "invalid parent classid"); 2777 return -EINVAL; 2778 } 2779 2780 err = airoha_qdma_set_tx_rate_limit(dev, channel, rate, opt->quantum); 2781 if (err) 2782 NL_SET_ERR_MSG_MOD(opt->extack, 2783 "failed configuring htb offload"); 2784 2785 return err; 2786 } 2787 2788 static int airoha_tc_htb_alloc_leaf_queue(struct net_device *netdev, 2789 struct tc_htb_qopt_offload *opt) 2790 { 2791 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 2792 int err, num_tx_queues = netdev->real_num_tx_queues; 2793 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2794 struct airoha_qdma *qdma = dev->qdma; 2795 2796 /* Here we need to check the requested QDMA channel is not already 2797 * in use by another net_device running on the same QDMA block. 2798 */ 2799 if (test_and_set_bit(channel, qdma->qos_channel_map)) { 2800 NL_SET_ERR_MSG_MOD(opt->extack, 2801 "qdma qos channel already in use"); 2802 return -EBUSY; 2803 } 2804 2805 err = airoha_tc_htb_modify_queue(netdev, opt); 2806 if (err) 2807 goto error; 2808 2809 err = netif_set_real_num_tx_queues(netdev, num_tx_queues + 1); 2810 if (err) { 2811 airoha_qdma_set_tx_rate_limit(netdev, channel, 0, 2812 opt->quantum); 2813 NL_SET_ERR_MSG_MOD(opt->extack, 2814 "failed setting real_num_tx_queues"); 2815 goto error; 2816 } 2817 2818 set_bit(channel, dev->qos_sq_bmap); 2819 opt->qid = AIROHA_NUM_TX_RING + channel; 2820 2821 return 0; 2822 error: 2823 clear_bit(channel, qdma->qos_channel_map); 2824 2825 return err; 2826 } 2827 2828 static int airoha_qdma_set_rx_meter(struct airoha_gdm_dev *dev, 2829 u32 rate, u32 bucket_size, 2830 enum trtcm_unit_type unit_type) 2831 { 2832 struct airoha_qdma *qdma = dev->qdma; 2833 int i; 2834 2835 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 2836 int err; 2837 2838 if (!qdma->q_rx[i].ndesc) 2839 continue; 2840 2841 err = airoha_qdma_init_rl_config(qdma, i, !!rate, unit_type); 2842 if (err) 2843 return err; 2844 2845 err = airoha_qdma_set_rl_token_bucket(qdma, i, rate, 2846 bucket_size); 2847 if (err) 2848 return err; 2849 } 2850 2851 return 0; 2852 } 2853 2854 static int airoha_tc_matchall_act_validate(struct tc_cls_matchall_offload *f) 2855 { 2856 const struct flow_action *actions = &f->rule->action; 2857 const struct flow_action_entry *act; 2858 2859 if (!flow_action_has_entries(actions)) { 2860 NL_SET_ERR_MSG_MOD(f->common.extack, 2861 "filter run with no actions"); 2862 return -EINVAL; 2863 } 2864 2865 if (!flow_offload_has_one_action(actions)) { 2866 NL_SET_ERR_MSG_MOD(f->common.extack, 2867 "only once action per filter is supported"); 2868 return -EOPNOTSUPP; 2869 } 2870 2871 act = &actions->entries[0]; 2872 if (act->id != FLOW_ACTION_POLICE) { 2873 NL_SET_ERR_MSG_MOD(f->common.extack, "unsupported action"); 2874 return -EOPNOTSUPP; 2875 } 2876 2877 if (act->police.exceed.act_id != FLOW_ACTION_DROP) { 2878 NL_SET_ERR_MSG_MOD(f->common.extack, 2879 "invalid exceed action id"); 2880 return -EOPNOTSUPP; 2881 } 2882 2883 if (act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { 2884 NL_SET_ERR_MSG_MOD(f->common.extack, 2885 "invalid notexceed action id"); 2886 return -EOPNOTSUPP; 2887 } 2888 2889 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && 2890 !flow_action_is_last_entry(actions, act)) { 2891 NL_SET_ERR_MSG_MOD(f->common.extack, 2892 "action accept must be last"); 2893 return -EOPNOTSUPP; 2894 } 2895 2896 if (act->police.peakrate_bytes_ps || act->police.avrate || 2897 act->police.overhead || act->police.mtu) { 2898 NL_SET_ERR_MSG_MOD(f->common.extack, 2899 "peakrate/avrate/overhead/mtu unsupported"); 2900 return -EOPNOTSUPP; 2901 } 2902 2903 return 0; 2904 } 2905 2906 static int airoha_dev_tc_matchall(struct net_device *netdev, 2907 struct tc_cls_matchall_offload *f) 2908 { 2909 enum trtcm_unit_type unit_type = TRTCM_BYTE_UNIT; 2910 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2911 u32 rate = 0, bucket_size = 0; 2912 2913 switch (f->command) { 2914 case TC_CLSMATCHALL_REPLACE: { 2915 const struct flow_action_entry *act; 2916 int err; 2917 2918 err = airoha_tc_matchall_act_validate(f); 2919 if (err) 2920 return err; 2921 2922 act = &f->rule->action.entries[0]; 2923 if (act->police.rate_pkt_ps) { 2924 rate = act->police.rate_pkt_ps; 2925 bucket_size = act->police.burst_pkt; 2926 unit_type = TRTCM_PACKET_UNIT; 2927 } else { 2928 rate = div_u64(act->police.rate_bytes_ps, 1000); 2929 rate = rate << 3; /* Kbps */ 2930 bucket_size = act->police.burst; 2931 } 2932 fallthrough; 2933 } 2934 case TC_CLSMATCHALL_DESTROY: 2935 return airoha_qdma_set_rx_meter(dev, rate, bucket_size, 2936 unit_type); 2937 default: 2938 return -EOPNOTSUPP; 2939 } 2940 } 2941 2942 static int airoha_dev_setup_tc_block_cb(enum tc_setup_type type, 2943 void *type_data, void *cb_priv) 2944 { 2945 struct net_device *netdev = cb_priv; 2946 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2947 struct airoha_eth *eth = dev->eth; 2948 2949 if (!tc_can_offload(netdev)) 2950 return -EOPNOTSUPP; 2951 2952 switch (type) { 2953 case TC_SETUP_CLSFLOWER: 2954 return airoha_ppe_setup_tc_block_cb(ð->ppe->dev, type_data); 2955 case TC_SETUP_CLSMATCHALL: 2956 return airoha_dev_tc_matchall(netdev, type_data); 2957 default: 2958 return -EOPNOTSUPP; 2959 } 2960 } 2961 2962 static int airoha_dev_setup_tc_block(struct net_device *dev, 2963 struct flow_block_offload *f) 2964 { 2965 flow_setup_cb_t *cb = airoha_dev_setup_tc_block_cb; 2966 static LIST_HEAD(block_cb_list); 2967 struct flow_block_cb *block_cb; 2968 2969 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 2970 return -EOPNOTSUPP; 2971 2972 f->driver_block_list = &block_cb_list; 2973 switch (f->command) { 2974 case FLOW_BLOCK_BIND: 2975 block_cb = flow_block_cb_lookup(f->block, cb, dev); 2976 if (block_cb) { 2977 flow_block_cb_incref(block_cb); 2978 return 0; 2979 } 2980 block_cb = flow_block_cb_alloc(cb, dev, dev, NULL); 2981 if (IS_ERR(block_cb)) 2982 return PTR_ERR(block_cb); 2983 2984 flow_block_cb_incref(block_cb); 2985 flow_block_cb_add(block_cb, f); 2986 list_add_tail(&block_cb->driver_list, &block_cb_list); 2987 return 0; 2988 case FLOW_BLOCK_UNBIND: 2989 block_cb = flow_block_cb_lookup(f->block, cb, dev); 2990 if (!block_cb) 2991 return -ENOENT; 2992 2993 if (!flow_block_cb_decref(block_cb)) { 2994 flow_block_cb_remove(block_cb, f); 2995 list_del(&block_cb->driver_list); 2996 } 2997 return 0; 2998 default: 2999 return -EOPNOTSUPP; 3000 } 3001 } 3002 3003 static void airoha_tc_remove_htb_queue(struct net_device *netdev, int queue) 3004 { 3005 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3006 struct airoha_qdma *qdma = dev->qdma; 3007 3008 netif_set_real_num_tx_queues(netdev, netdev->real_num_tx_queues - 1); 3009 airoha_qdma_set_tx_rate_limit(netdev, queue + 1, 0, 0); 3010 3011 clear_bit(queue, qdma->qos_channel_map); 3012 clear_bit(queue, dev->qos_sq_bmap); 3013 } 3014 3015 static int airoha_tc_htb_delete_leaf_queue(struct net_device *netdev, 3016 struct tc_htb_qopt_offload *opt) 3017 { 3018 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 3019 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3020 3021 if (!test_bit(channel, dev->qos_sq_bmap)) { 3022 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); 3023 return -EINVAL; 3024 } 3025 3026 airoha_tc_remove_htb_queue(netdev, channel); 3027 3028 return 0; 3029 } 3030 3031 static int airoha_tc_htb_destroy(struct net_device *netdev) 3032 { 3033 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3034 int q; 3035 3036 for_each_set_bit(q, dev->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS) 3037 airoha_tc_remove_htb_queue(netdev, q); 3038 3039 return 0; 3040 } 3041 3042 static int airoha_tc_get_htb_get_leaf_queue(struct net_device *netdev, 3043 struct tc_htb_qopt_offload *opt) 3044 { 3045 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 3046 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3047 3048 if (!test_bit(channel, dev->qos_sq_bmap)) { 3049 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); 3050 return -EINVAL; 3051 } 3052 3053 opt->qid = AIROHA_NUM_TX_RING + channel; 3054 3055 return 0; 3056 } 3057 3058 static int airoha_tc_setup_qdisc_htb(struct net_device *dev, 3059 struct tc_htb_qopt_offload *opt) 3060 { 3061 switch (opt->command) { 3062 case TC_HTB_CREATE: 3063 break; 3064 case TC_HTB_DESTROY: 3065 return airoha_tc_htb_destroy(dev); 3066 case TC_HTB_NODE_MODIFY: 3067 return airoha_tc_htb_modify_queue(dev, opt); 3068 case TC_HTB_LEAF_ALLOC_QUEUE: 3069 return airoha_tc_htb_alloc_leaf_queue(dev, opt); 3070 case TC_HTB_LEAF_DEL: 3071 case TC_HTB_LEAF_DEL_LAST: 3072 case TC_HTB_LEAF_DEL_LAST_FORCE: 3073 return airoha_tc_htb_delete_leaf_queue(dev, opt); 3074 case TC_HTB_LEAF_QUERY_QUEUE: 3075 return airoha_tc_get_htb_get_leaf_queue(dev, opt); 3076 default: 3077 return -EOPNOTSUPP; 3078 } 3079 3080 return 0; 3081 } 3082 3083 static int airoha_dev_tc_setup(struct net_device *dev, 3084 enum tc_setup_type type, void *type_data) 3085 { 3086 switch (type) { 3087 case TC_SETUP_QDISC_ETS: 3088 return airoha_tc_setup_qdisc_ets(dev, type_data); 3089 case TC_SETUP_QDISC_HTB: 3090 return airoha_tc_setup_qdisc_htb(dev, type_data); 3091 case TC_SETUP_BLOCK: 3092 case TC_SETUP_FT: 3093 return airoha_dev_setup_tc_block(dev, type_data); 3094 default: 3095 return -EOPNOTSUPP; 3096 } 3097 } 3098 3099 static const struct net_device_ops airoha_netdev_ops = { 3100 .ndo_init = airoha_dev_init, 3101 .ndo_open = airoha_dev_open, 3102 .ndo_stop = airoha_dev_stop, 3103 .ndo_change_mtu = airoha_dev_change_mtu, 3104 .ndo_select_queue = airoha_dev_select_queue, 3105 .ndo_start_xmit = airoha_dev_xmit, 3106 .ndo_get_stats64 = airoha_dev_get_stats64, 3107 .ndo_set_mac_address = airoha_dev_set_macaddr, 3108 .ndo_setup_tc = airoha_dev_tc_setup, 3109 }; 3110 3111 static const struct ethtool_ops airoha_ethtool_ops = { 3112 .get_drvinfo = airoha_ethtool_get_drvinfo, 3113 .get_eth_mac_stats = airoha_ethtool_get_mac_stats, 3114 .get_rmon_stats = airoha_ethtool_get_rmon_stats, 3115 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3116 .get_link = ethtool_op_get_link, 3117 }; 3118 3119 static int airoha_metadata_dst_alloc(struct airoha_gdm_port *port) 3120 { 3121 int i; 3122 3123 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) { 3124 struct metadata_dst *md_dst; 3125 3126 md_dst = metadata_dst_alloc(0, METADATA_HW_PORT_MUX, 3127 GFP_KERNEL); 3128 if (!md_dst) 3129 return -ENOMEM; 3130 3131 md_dst->u.port_info.port_id = i; 3132 port->dsa_meta[i] = md_dst; 3133 } 3134 3135 return 0; 3136 } 3137 3138 static void airoha_metadata_dst_free(struct airoha_gdm_port *port) 3139 { 3140 int i; 3141 3142 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) { 3143 if (!port->dsa_meta[i]) 3144 continue; 3145 3146 dst_release(&port->dsa_meta[i]->dst); 3147 } 3148 } 3149 3150 bool airoha_is_valid_gdm_dev(struct airoha_eth *eth, 3151 struct airoha_gdm_dev *dev) 3152 { 3153 int i; 3154 3155 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3156 struct airoha_gdm_port *port = eth->ports[i]; 3157 int j; 3158 3159 if (!port) 3160 continue; 3161 3162 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3163 if (port->devs[j] == dev) 3164 return true; 3165 } 3166 } 3167 3168 return false; 3169 } 3170 3171 static int airoha_alloc_gdm_device(struct airoha_eth *eth, 3172 struct airoha_gdm_port *port, 3173 int nbq, struct device_node *np) 3174 { 3175 struct net_device *netdev; 3176 struct airoha_gdm_dev *dev; 3177 u8 index; 3178 int err; 3179 3180 netdev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*dev), 3181 AIROHA_NUM_NETDEV_TX_RINGS, 3182 AIROHA_NUM_RX_RING); 3183 if (!netdev) { 3184 dev_err(eth->dev, "alloc_etherdev failed\n"); 3185 return -ENOMEM; 3186 } 3187 3188 netdev->netdev_ops = &airoha_netdev_ops; 3189 netdev->ethtool_ops = &airoha_ethtool_ops; 3190 netdev->max_mtu = AIROHA_MAX_MTU; 3191 netdev->watchdog_timeo = 5 * HZ; 3192 netdev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_TSO6 | 3193 NETIF_F_IPV6_CSUM | NETIF_F_SG | NETIF_F_TSO | 3194 NETIF_F_HW_TC; 3195 netdev->features |= netdev->hw_features; 3196 netdev->vlan_features = netdev->hw_features; 3197 SET_NETDEV_DEV(netdev, eth->dev); 3198 3199 /* reserve hw queues for HTB offloading */ 3200 err = netif_set_real_num_tx_queues(netdev, AIROHA_NUM_TX_RING); 3201 if (err) 3202 return err; 3203 3204 err = of_get_ethdev_address(np, netdev); 3205 if (err) { 3206 if (err == -EPROBE_DEFER) 3207 return err; 3208 3209 eth_hw_addr_random(netdev); 3210 dev_info(eth->dev, "generated random MAC address %pM\n", 3211 netdev->dev_addr); 3212 } 3213 3214 /* Allowed nbq for EN7581 on GDM3 port are 4 and 5 for PCIE0 3215 * and PCIE1 respectively. 3216 */ 3217 index = nbq; 3218 if (index && airoha_is_7581(eth) && port->id == AIROHA_GDM3_IDX) 3219 index -= 4; 3220 3221 if (index >= ARRAY_SIZE(port->devs) || port->devs[index]) { 3222 dev_err(eth->dev, "invalid nbq id: %d\n", nbq); 3223 return -EINVAL; 3224 } 3225 3226 netdev->dev.of_node = of_node_get(np); 3227 dev = netdev_priv(netdev); 3228 u64_stats_init(&dev->stats.syncp); 3229 dev->port = port; 3230 dev->eth = eth; 3231 dev->nbq = nbq; 3232 port->devs[index] = dev; 3233 3234 return 0; 3235 } 3236 3237 static int airoha_alloc_gdm_port(struct airoha_eth *eth, 3238 struct device_node *np) 3239 { 3240 const __be32 *id_ptr = of_get_property(np, "reg", NULL); 3241 struct airoha_gdm_port *port; 3242 struct device_node *node; 3243 int err, nbq, p, d = 0; 3244 u32 id; 3245 3246 if (!id_ptr) { 3247 dev_err(eth->dev, "missing gdm port id\n"); 3248 return -EINVAL; 3249 } 3250 3251 id = be32_to_cpup(id_ptr); 3252 p = id - 1; 3253 3254 if (!id || id > ARRAY_SIZE(eth->ports)) { 3255 dev_err(eth->dev, "invalid gdm port id: %d\n", id); 3256 return -EINVAL; 3257 } 3258 3259 if (eth->ports[p]) { 3260 dev_err(eth->dev, "duplicate gdm port id: %d\n", id); 3261 return -EINVAL; 3262 } 3263 3264 port = devm_kzalloc(eth->dev, sizeof(*port), GFP_KERNEL); 3265 if (!port) 3266 return -ENOMEM; 3267 3268 port->id = id; 3269 spin_lock_init(&port->stats_lock); 3270 eth->ports[p] = port; 3271 3272 err = airoha_metadata_dst_alloc(port); 3273 if (err) 3274 return err; 3275 3276 /* Default nbq value to ensure backward compatibility */ 3277 nbq = id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0; 3278 3279 for_each_child_of_node(np, node) { 3280 /* Multiple external serdes connected to the FE GDM port via an 3281 * external arbiter. 3282 */ 3283 const __be32 *nbq_ptr; 3284 3285 if (!of_device_is_compatible(node, "airoha,eth-port")) 3286 continue; 3287 3288 d++; 3289 if (!of_device_is_available(node)) 3290 continue; 3291 3292 nbq_ptr = of_get_property(node, "reg", NULL); 3293 if (!nbq_ptr) { 3294 dev_err(eth->dev, "missing nbq id\n"); 3295 of_node_put(node); 3296 return -EINVAL; 3297 } 3298 3299 /* Verify the provided nbq parameter is valid */ 3300 nbq = be32_to_cpup(nbq_ptr); 3301 err = eth->soc->ops.get_sport(port, nbq); 3302 if (err < 0) { 3303 of_node_put(node); 3304 return err; 3305 } 3306 3307 err = airoha_alloc_gdm_device(eth, port, nbq, node); 3308 if (err) { 3309 of_node_put(node); 3310 return err; 3311 } 3312 } 3313 3314 return !d ? airoha_alloc_gdm_device(eth, port, nbq, np) : 0; 3315 } 3316 3317 static int airoha_register_gdm_devices(struct airoha_eth *eth) 3318 { 3319 int i; 3320 3321 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3322 struct airoha_gdm_port *port = eth->ports[i]; 3323 int j; 3324 3325 if (!port) 3326 continue; 3327 3328 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3329 struct airoha_gdm_dev *dev = port->devs[j]; 3330 int err; 3331 3332 if (!dev) 3333 continue; 3334 3335 err = register_netdev(netdev_from_priv(dev)); 3336 if (err) 3337 return err; 3338 } 3339 } 3340 3341 set_bit(DEV_STATE_REGISTERED, ð->state); 3342 3343 return 0; 3344 } 3345 3346 static int airoha_probe(struct platform_device *pdev) 3347 { 3348 struct reset_control_bulk_data *xsi_rsts; 3349 struct device_node *np; 3350 struct airoha_eth *eth; 3351 int i, err; 3352 3353 eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL); 3354 if (!eth) 3355 return -ENOMEM; 3356 3357 eth->soc = of_device_get_match_data(&pdev->dev); 3358 if (!eth->soc) 3359 return -EINVAL; 3360 3361 eth->dev = &pdev->dev; 3362 3363 err = dma_set_mask_and_coherent(eth->dev, DMA_BIT_MASK(32)); 3364 if (err) { 3365 dev_err(eth->dev, "failed configuring DMA mask\n"); 3366 return err; 3367 } 3368 3369 eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe"); 3370 if (IS_ERR(eth->fe_regs)) 3371 return dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs), 3372 "failed to iomap fe regs\n"); 3373 3374 eth->rsts[0].id = "fe"; 3375 eth->rsts[1].id = "pdma"; 3376 eth->rsts[2].id = "qdma"; 3377 err = devm_reset_control_bulk_get_exclusive(eth->dev, 3378 ARRAY_SIZE(eth->rsts), 3379 eth->rsts); 3380 if (err) { 3381 dev_err(eth->dev, "failed to get bulk reset lines\n"); 3382 return err; 3383 } 3384 3385 xsi_rsts = devm_kcalloc(eth->dev, 3386 eth->soc->num_xsi_rsts, sizeof(*xsi_rsts), 3387 GFP_KERNEL); 3388 if (!xsi_rsts) 3389 return -ENOMEM; 3390 3391 eth->xsi_rsts = xsi_rsts; 3392 for (i = 0; i < eth->soc->num_xsi_rsts; i++) 3393 eth->xsi_rsts[i].id = eth->soc->xsi_rsts_names[i]; 3394 3395 err = devm_reset_control_bulk_get_exclusive(eth->dev, 3396 eth->soc->num_xsi_rsts, 3397 eth->xsi_rsts); 3398 if (err) { 3399 dev_err(eth->dev, "failed to get bulk xsi reset lines\n"); 3400 return err; 3401 } 3402 3403 eth->napi_dev = alloc_netdev_dummy(0); 3404 if (!eth->napi_dev) 3405 return -ENOMEM; 3406 3407 /* Enable threaded NAPI by default */ 3408 eth->napi_dev->threaded = true; 3409 strscpy(eth->napi_dev->name, "qdma_eth", sizeof(eth->napi_dev->name)); 3410 platform_set_drvdata(pdev, eth); 3411 3412 err = airoha_hw_init(pdev, eth); 3413 if (err) 3414 goto error_netdev_free; 3415 3416 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 3417 airoha_qdma_start_napi(ð->qdma[i]); 3418 3419 for_each_child_of_node(pdev->dev.of_node, np) { 3420 if (!of_device_is_compatible(np, "airoha,eth-mac")) 3421 continue; 3422 3423 if (!of_device_is_available(np)) 3424 continue; 3425 3426 err = airoha_alloc_gdm_port(eth, np); 3427 if (err) { 3428 of_node_put(np); 3429 goto error_napi_stop; 3430 } 3431 } 3432 3433 err = airoha_register_gdm_devices(eth); 3434 if (err) 3435 goto error_napi_stop; 3436 3437 return 0; 3438 3439 error_napi_stop: 3440 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 3441 airoha_qdma_stop_napi(ð->qdma[i]); 3442 3443 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3444 struct airoha_gdm_port *port = eth->ports[i]; 3445 int j; 3446 3447 if (!port) 3448 continue; 3449 3450 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3451 struct airoha_gdm_dev *dev = port->devs[j]; 3452 struct net_device *netdev; 3453 3454 if (!dev) 3455 continue; 3456 3457 netdev = netdev_from_priv(dev); 3458 if (netdev->reg_state == NETREG_REGISTERED) 3459 unregister_netdev(netdev); 3460 of_node_put(netdev->dev.of_node); 3461 } 3462 airoha_metadata_dst_free(port); 3463 } 3464 airoha_hw_cleanup(eth); 3465 error_netdev_free: 3466 free_netdev(eth->napi_dev); 3467 platform_set_drvdata(pdev, NULL); 3468 3469 return err; 3470 } 3471 3472 static void airoha_remove(struct platform_device *pdev) 3473 { 3474 struct airoha_eth *eth = platform_get_drvdata(pdev); 3475 int i; 3476 3477 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 3478 airoha_qdma_stop_napi(ð->qdma[i]); 3479 3480 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3481 struct airoha_gdm_port *port = eth->ports[i]; 3482 int j; 3483 3484 if (!port) 3485 continue; 3486 3487 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3488 struct airoha_gdm_dev *dev = port->devs[j]; 3489 struct net_device *netdev; 3490 3491 if (!dev) 3492 continue; 3493 3494 netdev = netdev_from_priv(dev); 3495 unregister_netdev(netdev); 3496 of_node_put(netdev->dev.of_node); 3497 } 3498 airoha_metadata_dst_free(port); 3499 } 3500 airoha_hw_cleanup(eth); 3501 3502 free_netdev(eth->napi_dev); 3503 platform_set_drvdata(pdev, NULL); 3504 } 3505 3506 static const char * const en7581_xsi_rsts_names[] = { 3507 "xsi-mac", 3508 "hsi0-mac", 3509 "hsi1-mac", 3510 "hsi-mac", 3511 "xfp-mac", 3512 }; 3513 3514 static int airoha_en7581_get_sport(struct airoha_gdm_port *port, int nbq) 3515 { 3516 switch (port->id) { 3517 case AIROHA_GDM3_IDX: 3518 /* 7581 SoC supports PCIe serdes on GDM3 port */ 3519 if (nbq == 4) 3520 return HSGMII_LAN_7581_PCIE0_SRCPORT; 3521 if (nbq == 5) 3522 return HSGMII_LAN_7581_PCIE1_SRCPORT; 3523 break; 3524 case AIROHA_GDM4_IDX: 3525 /* 7581 SoC supports eth and usb serdes on GDM4 port */ 3526 if (!nbq) 3527 return HSGMII_LAN_7581_ETH_SRCPORT; 3528 if (nbq == 1) 3529 return HSGMII_LAN_7581_USB_SRCPORT; 3530 break; 3531 default: 3532 break; 3533 } 3534 3535 return -EINVAL; 3536 } 3537 3538 static u32 airoha_en7581_get_vip_port(struct airoha_gdm_port *port, int nbq) 3539 { 3540 switch (port->id) { 3541 case AIROHA_GDM3_IDX: 3542 if (nbq == 4) 3543 return XSI_PCIE0_VIP_PORT_MASK; 3544 if (nbq == 5) 3545 return XSI_PCIE1_VIP_PORT_MASK; 3546 break; 3547 case AIROHA_GDM4_IDX: 3548 if (!nbq) 3549 return XSI_ETH_VIP_PORT_MASK; 3550 if (nbq == 1) 3551 return XSI_USB_VIP_PORT_MASK; 3552 break; 3553 default: 3554 break; 3555 } 3556 3557 return 0; 3558 } 3559 3560 static int airoha_en7581_get_dev_from_sport(struct airoha_qdma_desc *desc, 3561 u16 *port, u16 *dev) 3562 { 3563 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, 3564 le32_to_cpu(READ_ONCE(desc->msg1))); 3565 3566 *dev = 0; 3567 switch (sport) { 3568 case 0x10 ... 0x14: 3569 *port = 0; /* GDM1 */ 3570 break; 3571 case 0x2 ... 0x4: 3572 *port = sport - 1; 3573 break; 3574 case HSGMII_LAN_7581_PCIE1_SRCPORT: 3575 *dev = 1; 3576 fallthrough; 3577 case HSGMII_LAN_7581_PCIE0_SRCPORT: 3578 *port = 2; /* GDM3 */ 3579 break; 3580 case HSGMII_LAN_7581_USB_SRCPORT: 3581 *dev = 1; 3582 fallthrough; 3583 case HSGMII_LAN_7581_ETH_SRCPORT: 3584 *port = 3; /* GDM4 */ 3585 break; 3586 default: 3587 return -EINVAL; 3588 } 3589 3590 return 0; 3591 } 3592 3593 static const char * const an7583_xsi_rsts_names[] = { 3594 "xsi-mac", 3595 "hsi0-mac", 3596 "hsi1-mac", 3597 "xfp-mac", 3598 }; 3599 3600 static int airoha_an7583_get_sport(struct airoha_gdm_port *port, int nbq) 3601 { 3602 switch (port->id) { 3603 case AIROHA_GDM3_IDX: 3604 /* 7583 SoC supports eth serdes on GDM3 port */ 3605 if (!nbq) 3606 return HSGMII_LAN_7583_ETH_SRCPORT; 3607 break; 3608 case AIROHA_GDM4_IDX: 3609 /* 7583 SoC supports PCIe and USB serdes on GDM4 port */ 3610 if (!nbq) 3611 return HSGMII_LAN_7583_PCIE_SRCPORT; 3612 if (nbq == 1) 3613 return HSGMII_LAN_7583_USB_SRCPORT; 3614 break; 3615 default: 3616 break; 3617 } 3618 3619 return -EINVAL; 3620 } 3621 3622 static u32 airoha_an7583_get_vip_port(struct airoha_gdm_port *port, int nbq) 3623 { 3624 switch (port->id) { 3625 case AIROHA_GDM3_IDX: 3626 if (!nbq) 3627 return XSI_ETH_VIP_PORT_MASK; 3628 break; 3629 case AIROHA_GDM4_IDX: 3630 if (!nbq) 3631 return XSI_PCIE0_VIP_PORT_MASK; 3632 if (nbq == 1) 3633 return XSI_USB_VIP_PORT_MASK; 3634 break; 3635 default: 3636 break; 3637 } 3638 3639 return 0; 3640 } 3641 3642 static int airoha_an7583_get_dev_from_sport(struct airoha_qdma_desc *desc, 3643 u16 *port, u16 *dev) 3644 { 3645 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, 3646 le32_to_cpu(READ_ONCE(desc->msg1))); 3647 3648 *dev = 0; 3649 switch (sport) { 3650 case 0x10 ... 0x14: 3651 *port = 0; /* GDM1 */ 3652 break; 3653 case 0x2 ... 0x4: 3654 *port = sport - 1; 3655 break; 3656 case HSGMII_LAN_7583_ETH_SRCPORT: 3657 *port = 2; /* GDM3 */ 3658 break; 3659 case HSGMII_LAN_7583_USB_SRCPORT: 3660 *dev = 1; 3661 fallthrough; 3662 case HSGMII_LAN_7583_PCIE_SRCPORT: 3663 *port = 3; /* GDM4 */ 3664 break; 3665 default: 3666 return -EINVAL; 3667 } 3668 3669 return 0; 3670 } 3671 3672 static const struct airoha_eth_soc_data en7581_soc_data = { 3673 .version = 0x7581, 3674 .xsi_rsts_names = en7581_xsi_rsts_names, 3675 .num_xsi_rsts = ARRAY_SIZE(en7581_xsi_rsts_names), 3676 .num_ppe = 2, 3677 .ops = { 3678 .get_sport = airoha_en7581_get_sport, 3679 .get_vip_port = airoha_en7581_get_vip_port, 3680 .get_dev_from_sport = airoha_en7581_get_dev_from_sport, 3681 }, 3682 }; 3683 3684 static const struct airoha_eth_soc_data an7583_soc_data = { 3685 .version = 0x7583, 3686 .xsi_rsts_names = an7583_xsi_rsts_names, 3687 .num_xsi_rsts = ARRAY_SIZE(an7583_xsi_rsts_names), 3688 .num_ppe = 1, 3689 .ops = { 3690 .get_sport = airoha_an7583_get_sport, 3691 .get_vip_port = airoha_an7583_get_vip_port, 3692 .get_dev_from_sport = airoha_an7583_get_dev_from_sport, 3693 }, 3694 }; 3695 3696 static const struct of_device_id of_airoha_match[] = { 3697 { .compatible = "airoha,en7581-eth", .data = &en7581_soc_data }, 3698 { .compatible = "airoha,an7583-eth", .data = &an7583_soc_data }, 3699 { /* sentinel */ } 3700 }; 3701 MODULE_DEVICE_TABLE(of, of_airoha_match); 3702 3703 static struct platform_driver airoha_driver = { 3704 .probe = airoha_probe, 3705 .remove = airoha_remove, 3706 .driver = { 3707 .name = KBUILD_MODNAME, 3708 .of_match_table = of_airoha_match, 3709 }, 3710 }; 3711 module_platform_driver(airoha_driver); 3712 3713 MODULE_LICENSE("GPL v2"); 3714 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>"); 3715 MODULE_DESCRIPTION("Ethernet driver for Airoha SoC"); 3716