1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2024 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 #include <linux/of.h> 7 #include <linux/of_net.h> 8 #include <linux/of_reserved_mem.h> 9 #include <linux/platform_device.h> 10 #include <linux/tcp.h> 11 #include <linux/u64_stats_sync.h> 12 #include <net/dst_metadata.h> 13 #include <net/page_pool/helpers.h> 14 #include <net/pkt_cls.h> 15 #include <uapi/linux/ppp_defs.h> 16 17 #include "airoha_regs.h" 18 #include "airoha_eth.h" 19 20 u32 airoha_rr(void __iomem *base, u32 offset) 21 { 22 return readl(base + offset); 23 } 24 25 void airoha_wr(void __iomem *base, u32 offset, u32 val) 26 { 27 writel(val, base + offset); 28 } 29 30 u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val) 31 { 32 val |= (airoha_rr(base, offset) & ~mask); 33 airoha_wr(base, offset, val); 34 35 return val; 36 } 37 38 static void airoha_qdma_set_irqmask(struct airoha_irq_bank *irq_bank, 39 int index, u32 clear, u32 set) 40 { 41 struct airoha_qdma *qdma = irq_bank->qdma; 42 int bank = irq_bank - &qdma->irq_banks[0]; 43 unsigned long flags; 44 45 if (WARN_ON_ONCE(index >= ARRAY_SIZE(irq_bank->irqmask))) 46 return; 47 48 spin_lock_irqsave(&irq_bank->irq_lock, flags); 49 50 irq_bank->irqmask[index] &= ~clear; 51 irq_bank->irqmask[index] |= set; 52 airoha_qdma_wr(qdma, REG_INT_ENABLE(bank, index), 53 irq_bank->irqmask[index]); 54 /* Read irq_enable register in order to guarantee the update above 55 * completes in the spinlock critical section. 56 */ 57 airoha_qdma_rr(qdma, REG_INT_ENABLE(bank, index)); 58 59 spin_unlock_irqrestore(&irq_bank->irq_lock, flags); 60 } 61 62 static void airoha_qdma_irq_enable(struct airoha_irq_bank *irq_bank, 63 int index, u32 mask) 64 { 65 airoha_qdma_set_irqmask(irq_bank, index, 0, mask); 66 } 67 68 static void airoha_qdma_irq_disable(struct airoha_irq_bank *irq_bank, 69 int index, u32 mask) 70 { 71 airoha_qdma_set_irqmask(irq_bank, index, mask, 0); 72 } 73 74 static int airoha_set_macaddr(struct airoha_gdm_dev *dev, const u8 *addr) 75 { 76 u8 ref_addr[ETH_ALEN] __aligned(2); 77 struct airoha_eth *eth = dev->eth; 78 u32 reg, val, lmin, lmax; 79 int i; 80 81 eth_zero_addr(ref_addr); 82 lmin = (addr[3] << 16) | (addr[4] << 8) | addr[5]; 83 lmax = lmin; 84 85 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 86 struct airoha_gdm_port *port = eth->ports[i]; 87 int j; 88 89 if (!port) 90 continue; 91 92 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 93 struct airoha_gdm_dev *iter_dev; 94 struct net_device *netdev; 95 96 iter_dev = port->devs[j]; 97 if (!iter_dev || iter_dev == dev) 98 continue; 99 100 if (airoha_is_lan_gdm_dev(iter_dev) != 101 airoha_is_lan_gdm_dev(dev)) 102 continue; 103 104 netdev = netdev_from_priv(iter_dev); 105 if (netdev->reg_state != NETREG_REGISTERED) 106 continue; 107 108 ether_addr_copy(ref_addr, netdev->dev_addr); 109 val = (netdev->dev_addr[3] << 16) | 110 (netdev->dev_addr[4] << 8) | netdev->dev_addr[5]; 111 if (val < lmin) 112 lmin = val; 113 if (val > lmax) 114 lmax = val; 115 } 116 } 117 118 if (!is_zero_ether_addr(ref_addr) && memcmp(ref_addr, addr, 3)) { 119 /* According to the HW design, hw mac address MSBs must be 120 * the same for each net_device with the same LAN/WAN 121 * configuration. 122 */ 123 struct net_device *netdev = netdev_from_priv(dev); 124 125 dev_warn(eth->dev, 126 "%s: wrong mac addr, MSBs must be %02x:%02x:%02x\n", 127 netdev->name, ref_addr[0], ref_addr[1], 128 ref_addr[2]); 129 dev_warn(eth->dev, "FE hw forwarding won't work properly\n"); 130 131 return -EINVAL; 132 } 133 134 reg = airoha_is_lan_gdm_dev(dev) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H; 135 val = (addr[0] << 16) | (addr[1] << 8) | addr[2]; 136 airoha_fe_wr(eth, reg, val); 137 138 airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), lmin); 139 airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), lmax); 140 141 airoha_ppe_init_upd_mem(dev, addr); 142 143 return 0; 144 } 145 146 static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr, 147 u32 val) 148 { 149 airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK, 150 FIELD_PREP(GDM_OCFQ_MASK, val)); 151 airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK, 152 FIELD_PREP(GDM_MCFQ_MASK, val)); 153 airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK, 154 FIELD_PREP(GDM_BCFQ_MASK, val)); 155 airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK, 156 FIELD_PREP(GDM_UCFQ_MASK, val)); 157 } 158 159 static int airoha_set_vip_for_gdm_port(struct airoha_gdm_dev *dev, bool enable) 160 { 161 struct airoha_gdm_port *port = dev->port; 162 struct airoha_eth *eth = dev->eth; 163 u32 vip_port; 164 165 vip_port = eth->soc->ops.get_vip_port(port, dev->nbq); 166 if (enable) { 167 airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port); 168 airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port); 169 } else { 170 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port); 171 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port); 172 } 173 174 return 0; 175 } 176 177 static void airoha_fe_maccr_init(struct airoha_eth *eth) 178 { 179 int p; 180 181 for (p = 1; p <= ARRAY_SIZE(eth->ports); p++) 182 airoha_fe_set(eth, REG_GDM_FWD_CFG(p), 183 GDM_TCP_CKSUM_MASK | GDM_UDP_CKSUM_MASK | 184 GDM_IP4_CKSUM_MASK | GDM_DROP_CRC_ERR_MASK); 185 186 airoha_fe_rmw(eth, REG_CDM_VLAN_CTRL(1), CDM_VLAN_MASK, 187 FIELD_PREP(CDM_VLAN_MASK, 0x8100)); 188 189 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD); 190 } 191 192 static void airoha_fe_vip_setup(struct airoha_eth *eth) 193 { 194 airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC); 195 airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK); 196 197 airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP); 198 airoha_fe_wr(eth, REG_FE_VIP_EN(4), 199 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 200 PATN_EN_MASK); 201 202 airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP); 203 airoha_fe_wr(eth, REG_FE_VIP_EN(6), 204 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 205 PATN_EN_MASK); 206 207 airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP); 208 airoha_fe_wr(eth, REG_FE_VIP_EN(7), 209 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 210 PATN_EN_MASK); 211 212 /* BOOTP (0x43) */ 213 airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43); 214 airoha_fe_wr(eth, REG_FE_VIP_EN(8), 215 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK | 216 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 217 218 /* BOOTP (0x44) */ 219 airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44); 220 airoha_fe_wr(eth, REG_FE_VIP_EN(9), 221 PATN_FCPU_EN_MASK | PATN_SP_EN_MASK | 222 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 223 224 /* ISAKMP */ 225 airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4); 226 airoha_fe_wr(eth, REG_FE_VIP_EN(10), 227 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK | 228 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 229 230 airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP); 231 airoha_fe_wr(eth, REG_FE_VIP_EN(11), 232 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 233 PATN_EN_MASK); 234 235 /* DHCPv6 */ 236 airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223); 237 airoha_fe_wr(eth, REG_FE_VIP_EN(12), 238 PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK | 239 FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK); 240 241 airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP); 242 airoha_fe_wr(eth, REG_FE_VIP_EN(19), 243 PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) | 244 PATN_EN_MASK); 245 246 /* ETH->ETH_P_1905 (0x893a) */ 247 airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a); 248 airoha_fe_wr(eth, REG_FE_VIP_EN(20), 249 PATN_FCPU_EN_MASK | PATN_EN_MASK); 250 251 airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP); 252 airoha_fe_wr(eth, REG_FE_VIP_EN(21), 253 PATN_FCPU_EN_MASK | PATN_EN_MASK); 254 } 255 256 static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth, 257 u32 port, u32 queue) 258 { 259 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR, 260 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK, 261 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) | 262 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue)); 263 264 return airoha_fe_get(eth, REG_FE_PSE_QUEUE_CFG_VAL, 265 PSE_CFG_OQ_RSV_MASK); 266 } 267 268 static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth, 269 u32 port, u32 queue, u32 val) 270 { 271 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK, 272 FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val)); 273 airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR, 274 PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK | 275 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK, 276 FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) | 277 FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) | 278 PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK); 279 } 280 281 static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth) 282 { 283 return airoha_fe_get(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK); 284 } 285 286 static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth, 287 u32 port, u32 queue, u32 val) 288 { 289 u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue); 290 u32 tmp, all_rsv, fq_limit; 291 292 airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val); 293 294 /* modify all rsv */ 295 all_rsv = airoha_fe_get_pse_all_rsv(eth); 296 all_rsv += (val - orig_val); 297 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, 298 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); 299 300 /* modify hthd */ 301 fq_limit = airoha_fe_get(eth, PSE_FQ_CFG, PSE_FQ_LIMIT_MASK); 302 tmp = fq_limit - all_rsv - 0x20; 303 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD, 304 PSE_SHARE_USED_HTHD_MASK, 305 FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp)); 306 307 tmp = fq_limit - all_rsv - 0x100; 308 airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD, 309 PSE_SHARE_USED_MTHD_MASK, 310 FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp)); 311 tmp = (3 * tmp) >> 2; 312 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, 313 PSE_SHARE_USED_LTHD_MASK, 314 FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp)); 315 316 return 0; 317 } 318 319 static void airoha_fe_pse_ports_init(struct airoha_eth *eth) 320 { 321 const u32 pse_port_num_queues[] = { 322 [FE_PSE_PORT_CDM1] = 6, 323 [FE_PSE_PORT_GDM1] = 6, 324 [FE_PSE_PORT_GDM2] = 32, 325 [FE_PSE_PORT_GDM3] = 6, 326 [FE_PSE_PORT_PPE1] = 4, 327 [FE_PSE_PORT_CDM2] = 6, 328 [FE_PSE_PORT_CDM3] = 8, 329 [FE_PSE_PORT_CDM4] = 10, 330 [FE_PSE_PORT_PPE2] = 4, 331 [FE_PSE_PORT_GDM4] = 2, 332 [FE_PSE_PORT_CDM5] = 2, 333 }; 334 int q; 335 336 if (airoha_ppe_is_enabled(eth, 1)) { 337 u32 all_rsv; 338 339 /* hw misses PPE2 oq rsv */ 340 all_rsv = airoha_fe_get_pse_all_rsv(eth); 341 all_rsv += PSE_RSV_PAGES * 342 pse_port_num_queues[FE_PSE_PORT_PPE2]; 343 airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, 344 FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); 345 } 346 347 /* CDM1 */ 348 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++) 349 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q, 350 PSE_QUEUE_RSV_PAGES); 351 /* GDM1 */ 352 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++) 353 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q, 354 PSE_QUEUE_RSV_PAGES); 355 /* GDM2 */ 356 for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++) 357 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0); 358 /* GDM3 */ 359 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++) 360 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q, 361 PSE_QUEUE_RSV_PAGES); 362 /* PPE1 */ 363 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) { 364 if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2) 365 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 366 PSE_QUEUE_RSV_PAGES); 367 else 368 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0); 369 } 370 /* CDM2 */ 371 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++) 372 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q, 373 PSE_QUEUE_RSV_PAGES); 374 /* CDM3 */ 375 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++) 376 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0); 377 /* CDM4 */ 378 for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++) 379 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q, 380 PSE_QUEUE_RSV_PAGES); 381 if (airoha_ppe_is_enabled(eth, 1)) { 382 /* PPE2 */ 383 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) { 384 if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2) 385 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, 386 q, 387 PSE_QUEUE_RSV_PAGES); 388 else 389 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, 390 q, 0); 391 } 392 } 393 /* GDM4 */ 394 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++) 395 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q, 396 PSE_QUEUE_RSV_PAGES); 397 /* CDM5 */ 398 for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++) 399 airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q, 400 PSE_QUEUE_RSV_PAGES); 401 } 402 403 static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth) 404 { 405 int i; 406 407 for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) { 408 int err, j; 409 u32 val; 410 411 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0); 412 413 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) | 414 MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK; 415 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val); 416 err = read_poll_timeout(airoha_fe_rr, val, 417 val & MC_VLAN_CFG_CMD_DONE_MASK, 418 USEC_PER_MSEC, 5 * USEC_PER_MSEC, 419 false, eth, REG_MC_VLAN_CFG); 420 if (err) 421 return err; 422 423 for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) { 424 airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0); 425 426 val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) | 427 FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) | 428 MC_VLAN_CFG_RW_MASK; 429 airoha_fe_wr(eth, REG_MC_VLAN_CFG, val); 430 err = read_poll_timeout(airoha_fe_rr, val, 431 val & MC_VLAN_CFG_CMD_DONE_MASK, 432 USEC_PER_MSEC, 433 5 * USEC_PER_MSEC, false, eth, 434 REG_MC_VLAN_CFG); 435 if (err) 436 return err; 437 } 438 } 439 440 return 0; 441 } 442 443 static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth) 444 { 445 /* CDM1_CRSN_QSEL */ 446 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_22 >> 2), 447 CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 448 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 449 CDM_CRSN_QSEL_Q1)); 450 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_08 >> 2), 451 CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 452 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 453 CDM_CRSN_QSEL_Q1)); 454 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_21 >> 2), 455 CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 456 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 457 CDM_CRSN_QSEL_Q1)); 458 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_24 >> 2), 459 CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 460 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 461 CDM_CRSN_QSEL_Q6)); 462 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_25 >> 2), 463 CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 464 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 465 CDM_CRSN_QSEL_Q1)); 466 /* CDM2_CRSN_QSEL */ 467 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_08 >> 2), 468 CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 469 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08), 470 CDM_CRSN_QSEL_Q1)); 471 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_21 >> 2), 472 CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 473 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21), 474 CDM_CRSN_QSEL_Q1)); 475 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_22 >> 2), 476 CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 477 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22), 478 CDM_CRSN_QSEL_Q1)); 479 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_24 >> 2), 480 CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 481 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24), 482 CDM_CRSN_QSEL_Q6)); 483 airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_25 >> 2), 484 CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 485 FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25), 486 CDM_CRSN_QSEL_Q1)); 487 } 488 489 static int airoha_fe_init(struct airoha_eth *eth) 490 { 491 airoha_fe_maccr_init(eth); 492 493 /* PSE IQ reserve */ 494 airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK, 495 FIELD_PREP(PSE_IQ_RES1_P2_MASK, 0x10)); 496 airoha_fe_rmw(eth, REG_PSE_IQ_REV2, 497 PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK, 498 FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) | 499 FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34)); 500 501 /* enable FE copy engine for KA/DPI */ 502 airoha_fe_wr(eth, REG_FE_PCE_CFG, PCE_DPI_EN_MASK | PCE_KA_EN_MASK); 503 /* set vip queue selection to ring 1 */ 504 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(1), CDM_VIP_QSEL_MASK, 505 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4)); 506 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_VIP_QSEL_MASK, 507 FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4)); 508 /* set GDM4 source interface offset to 8 */ 509 airoha_fe_rmw(eth, REG_GDM_SRC_PORT_SET(4), 510 GDM_SPORT_OFF2_MASK | 511 GDM_SPORT_OFF1_MASK | 512 GDM_SPORT_OFF0_MASK, 513 FIELD_PREP(GDM_SPORT_OFF2_MASK, 8) | 514 FIELD_PREP(GDM_SPORT_OFF1_MASK, 8) | 515 FIELD_PREP(GDM_SPORT_OFF0_MASK, 8)); 516 517 /* set PSE Page as 128B */ 518 airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG, 519 FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK, 520 FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, 2) | 521 FE_DMA_GLO_PG_SZ_MASK); 522 airoha_fe_wr(eth, REG_FE_RST_GLO_CFG, 523 FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK | 524 FE_RST_GDM4_MBI_ARB_MASK); 525 usleep_range(1000, 2000); 526 527 /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1 528 * connect other rings to PSE Port0 OQ-0 529 */ 530 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4)); 531 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28)); 532 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4)); 533 airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28)); 534 535 airoha_fe_vip_setup(eth); 536 airoha_fe_pse_ports_init(eth); 537 538 airoha_fe_set(eth, REG_GDM_MISC_CFG, 539 GDM2_RDM_ACK_WAIT_PREF_MASK | 540 GDM2_CHN_VLD_MODE_MASK); 541 airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_OAM_QSEL_MASK, 542 FIELD_PREP(CDM_OAM_QSEL_MASK, 15)); 543 544 /* init fragment and assemble Force Port */ 545 /* NPU Core-3, NPU Bridge Channel-3 */ 546 airoha_fe_rmw(eth, REG_IP_FRAG_FP, 547 IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK, 548 FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) | 549 FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3)); 550 /* QDMA LAN, RX Ring-22 */ 551 airoha_fe_rmw(eth, REG_IP_FRAG_FP, 552 IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK, 553 FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) | 554 FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22)); 555 556 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM3_IDX), GDM_PAD_EN_MASK); 557 airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM4_IDX), GDM_PAD_EN_MASK); 558 559 /* Enable split for MIB counters for GDM3 and GDM4 */ 560 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM3_IDX), 561 FE_GDM_TX_MIB_SPLIT_EN_MASK | 562 FE_GDM_RX_MIB_SPLIT_EN_MASK); 563 airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM4_IDX), 564 FE_GDM_TX_MIB_SPLIT_EN_MASK | 565 FE_GDM_RX_MIB_SPLIT_EN_MASK); 566 567 airoha_fe_crsn_qsel_init(eth); 568 569 airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK); 570 airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK); 571 572 /* default aging mode for mbi unlock issue */ 573 airoha_fe_rmw(eth, REG_GDM_CHN_RLS(2), 574 MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK, 575 FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) | 576 FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3)); 577 578 /* disable IFC by default */ 579 airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK); 580 581 /* enable 1:N vlan action, init vlan table */ 582 airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK); 583 584 return airoha_fe_mc_vlan_clear(eth); 585 } 586 587 static int airoha_qdma_fill_rx_queue(struct airoha_queue *q) 588 { 589 struct airoha_qdma *qdma = q->qdma; 590 int qid = q - &qdma->q_rx[0]; 591 int nframes = 0; 592 593 while (q->queued < q->ndesc - 1) { 594 struct airoha_queue_entry *e = &q->entry[q->head]; 595 struct airoha_qdma_desc *desc = &q->desc[q->head]; 596 struct page *page; 597 int offset; 598 u32 val; 599 600 page = page_pool_dev_alloc_frag(q->page_pool, &offset, 601 q->buf_size); 602 if (!page) 603 break; 604 605 q->head = (q->head + 1) % q->ndesc; 606 q->queued++; 607 nframes++; 608 609 offset += AIROHA_RX_HEADROOM; 610 e->buf = page_address(page) + offset; 611 e->dma_addr = page_pool_get_dma_addr(page) + offset; 612 e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size)); 613 614 val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len); 615 WRITE_ONCE(desc->ctrl, cpu_to_le32(val)); 616 WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr)); 617 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head); 618 WRITE_ONCE(desc->data, cpu_to_le32(val)); 619 WRITE_ONCE(desc->msg0, 0); 620 WRITE_ONCE(desc->msg1, 0); 621 WRITE_ONCE(desc->msg2, 0); 622 WRITE_ONCE(desc->msg3, 0); 623 } 624 625 if (nframes) 626 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), 627 RX_RING_CPU_IDX_MASK, 628 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); 629 630 return nframes; 631 } 632 633 static struct airoha_gdm_dev * 634 airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc *desc) 635 { 636 struct airoha_gdm_port *port; 637 u16 p, d; 638 639 if (eth->soc->ops.get_dev_from_sport(desc, &p, &d)) 640 return ERR_PTR(-ENODEV); 641 642 if (p >= ARRAY_SIZE(eth->ports)) 643 return ERR_PTR(-ENODEV); 644 645 port = eth->ports[p]; 646 if (!port) 647 return ERR_PTR(-ENODEV); 648 649 if (d >= ARRAY_SIZE(port->devs)) 650 return ERR_PTR(-ENODEV); 651 652 return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV); 653 } 654 655 static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) 656 { 657 enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool); 658 struct airoha_qdma *qdma = q->qdma; 659 struct airoha_eth *eth = qdma->eth; 660 int qid = q - &qdma->q_rx[0]; 661 int done = 0; 662 663 while (done < budget) { 664 struct airoha_queue_entry *e = &q->entry[q->tail]; 665 struct airoha_qdma_desc *desc = &q->desc[q->tail]; 666 u32 hash, reason, msg1, desc_ctrl; 667 struct airoha_gdm_dev *dev; 668 struct net_device *netdev; 669 int data_len, len; 670 struct page *page; 671 672 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); 673 if (!(desc_ctrl & QDMA_DESC_DONE_MASK)) 674 break; 675 676 dma_rmb(); 677 678 q->tail = (q->tail + 1) % q->ndesc; 679 q->queued--; 680 681 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, 682 dir); 683 684 page = virt_to_head_page(e->buf); 685 len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl); 686 data_len = q->skb ? AIROHA_RX_LEN(q->buf_size) : e->dma_len; 687 if (!len || data_len < len) 688 goto free_frag; 689 690 dev = airoha_qdma_get_gdm_dev(eth, desc); 691 if (IS_ERR(dev)) 692 goto free_frag; 693 694 netdev = netdev_from_priv(dev); 695 if (!q->skb) { /* first buffer */ 696 q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM, 697 q->buf_size); 698 if (!q->skb) 699 goto free_frag; 700 701 skb_reserve(q->skb, AIROHA_RX_HEADROOM); 702 __skb_put(q->skb, len); 703 skb_mark_for_recycle(q->skb); 704 q->skb->dev = netdev; 705 q->skb->protocol = eth_type_trans(q->skb, netdev); 706 q->skb->ip_summed = CHECKSUM_UNNECESSARY; 707 skb_record_rx_queue(q->skb, qid); 708 } else { /* scattered frame */ 709 struct skb_shared_info *shinfo = skb_shinfo(q->skb); 710 int nr_frags = shinfo->nr_frags; 711 712 if (nr_frags >= ARRAY_SIZE(shinfo->frags)) 713 goto free_frag; 714 715 skb_add_rx_frag(q->skb, nr_frags, page, 716 e->buf - page_address(page), len, 717 q->buf_size); 718 } 719 720 if (FIELD_GET(QDMA_DESC_MORE_MASK, desc_ctrl)) 721 continue; 722 723 if (netdev_uses_dsa(netdev)) { 724 struct airoha_gdm_port *port = dev->port; 725 726 /* PPE module requires untagged packets to work 727 * properly and it provides DSA port index via the 728 * DMA descriptor. Report DSA tag to the DSA stack 729 * via skb dst info. 730 */ 731 u32 msg0 = le32_to_cpu(READ_ONCE(desc->msg0)); 732 u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0); 733 734 if (sptag < ARRAY_SIZE(port->dsa_meta) && 735 port->dsa_meta[sptag]) 736 skb_dst_set_noref(q->skb, 737 &port->dsa_meta[sptag]->dst); 738 } 739 740 msg1 = le32_to_cpu(READ_ONCE(desc->msg1)); 741 hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1); 742 if (hash != AIROHA_RXD4_FOE_ENTRY) 743 skb_set_hash(q->skb, jhash_1word(hash, 0), 744 PKT_HASH_TYPE_L4); 745 746 reason = FIELD_GET(AIROHA_RXD4_PPE_CPU_REASON, msg1); 747 if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) 748 airoha_ppe_check_skb(ð->ppe->dev, q->skb, hash, 749 false); 750 751 done++; 752 napi_gro_receive(&q->napi, q->skb); 753 q->skb = NULL; 754 continue; 755 free_frag: 756 if (q->skb) { 757 dev_kfree_skb(q->skb); 758 q->skb = NULL; 759 } 760 page_pool_put_full_page(q->page_pool, page, true); 761 } 762 airoha_qdma_fill_rx_queue(q); 763 764 return done; 765 } 766 767 static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget) 768 { 769 struct airoha_queue *q = container_of(napi, struct airoha_queue, napi); 770 int cur, done = 0; 771 772 do { 773 cur = airoha_qdma_rx_process(q, budget - done); 774 done += cur; 775 } while (cur && done < budget); 776 777 if (done < budget && napi_complete(napi)) { 778 struct airoha_qdma *qdma = q->qdma; 779 int i, qid = q - &qdma->q_rx[0]; 780 int intr_reg = qid < RX_DONE_HIGH_OFFSET ? QDMA_INT_REG_IDX1 781 : QDMA_INT_REG_IDX2; 782 783 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 784 if (!(BIT(qid) & RX_IRQ_BANK_PIN_MASK(i))) 785 continue; 786 787 airoha_qdma_irq_enable(&qdma->irq_banks[i], intr_reg, 788 BIT(qid % RX_DONE_HIGH_OFFSET)); 789 } 790 } 791 792 return done; 793 } 794 795 static int airoha_qdma_init_rx_queue(struct airoha_queue *q, 796 struct airoha_qdma *qdma, int ndesc) 797 { 798 const struct page_pool_params pp_params = { 799 .order = 0, 800 .pool_size = 256, 801 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 802 .dma_dir = DMA_FROM_DEVICE, 803 .max_len = PAGE_SIZE, 804 .nid = NUMA_NO_NODE, 805 .dev = qdma->eth->dev, 806 .napi = &q->napi, 807 }; 808 struct airoha_eth *eth = qdma->eth; 809 int qid = q - &qdma->q_rx[0], thr; 810 dma_addr_t dma_addr; 811 812 q->buf_size = PAGE_SIZE / 2; 813 q->qdma = qdma; 814 815 q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry), 816 GFP_KERNEL); 817 if (!q->entry) 818 return -ENOMEM; 819 820 q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc), 821 &dma_addr, GFP_KERNEL); 822 if (!q->desc) 823 return -ENOMEM; 824 825 q->page_pool = page_pool_create(&pp_params); 826 if (IS_ERR(q->page_pool)) { 827 int err = PTR_ERR(q->page_pool); 828 829 q->page_pool = NULL; 830 return err; 831 } 832 833 q->ndesc = ndesc; 834 netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll); 835 836 airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr); 837 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), 838 RX_RING_SIZE_MASK, 839 FIELD_PREP(RX_RING_SIZE_MASK, ndesc)); 840 841 thr = clamp(ndesc >> 3, 1, 32); 842 airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK, 843 FIELD_PREP(RX_RING_THR_MASK, thr)); 844 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, 845 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head)); 846 airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid), RX_RING_SG_EN_MASK); 847 848 airoha_qdma_fill_rx_queue(q); 849 850 return 0; 851 } 852 853 static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q) 854 { 855 struct airoha_qdma *qdma = q->qdma; 856 struct airoha_eth *eth = qdma->eth; 857 int qid = q - &qdma->q_rx[0]; 858 859 while (q->queued) { 860 struct airoha_queue_entry *e = &q->entry[q->tail]; 861 struct airoha_qdma_desc *desc = &q->desc[q->tail]; 862 struct page *page = virt_to_head_page(e->buf); 863 864 dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, 865 page_pool_get_dma_dir(q->page_pool)); 866 page_pool_put_full_page(q->page_pool, page, false); 867 /* Reset DMA descriptor */ 868 WRITE_ONCE(desc->ctrl, 0); 869 WRITE_ONCE(desc->addr, 0); 870 WRITE_ONCE(desc->data, 0); 871 WRITE_ONCE(desc->msg0, 0); 872 WRITE_ONCE(desc->msg1, 0); 873 WRITE_ONCE(desc->msg2, 0); 874 WRITE_ONCE(desc->msg3, 0); 875 876 q->tail = (q->tail + 1) % q->ndesc; 877 q->queued--; 878 } 879 880 q->head = q->tail; 881 /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is 882 * empty. 883 */ 884 airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK, 885 FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); 886 airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, 887 FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail)); 888 } 889 890 static int airoha_qdma_init_rx(struct airoha_qdma *qdma) 891 { 892 int i; 893 894 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 895 int err; 896 897 if (!(RX_DONE_INT_MASK & BIT(i))) { 898 /* rx-queue not binded to irq */ 899 continue; 900 } 901 902 err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma, 903 RX_DSCP_NUM(i)); 904 if (err) 905 return err; 906 } 907 908 return 0; 909 } 910 911 static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q) 912 { 913 struct airoha_qdma *qdma = q->qdma; 914 struct airoha_eth *eth = qdma->eth; 915 int i, qid = q - &qdma->q_tx[0]; 916 917 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 918 struct airoha_gdm_port *port = eth->ports[i]; 919 int d; 920 921 if (!port) 922 continue; 923 924 for (d = 0; d < ARRAY_SIZE(port->devs); d++) { 925 struct airoha_gdm_dev *dev = port->devs[d]; 926 struct net_device *netdev; 927 int j; 928 929 if (!dev) 930 continue; 931 932 if (dev->qdma != qdma) 933 continue; 934 935 netdev = netdev_from_priv(dev); 936 for (j = 0; j < netdev->num_tx_queues; j++) { 937 if (airoha_qdma_get_txq(qdma, j) != qid) 938 continue; 939 940 netif_wake_subqueue(netdev, j); 941 } 942 } 943 } 944 q->txq_stopped = false; 945 } 946 947 static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget) 948 { 949 struct airoha_tx_irq_queue *irq_q; 950 int id, done = 0, irq_queued; 951 struct airoha_qdma *qdma; 952 struct airoha_eth *eth; 953 u32 status, head; 954 955 irq_q = container_of(napi, struct airoha_tx_irq_queue, napi); 956 qdma = irq_q->qdma; 957 id = irq_q - &qdma->q_tx_irq[0]; 958 eth = qdma->eth; 959 960 status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id)); 961 head = FIELD_GET(IRQ_HEAD_IDX_MASK, status); 962 head = head % irq_q->size; 963 irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status); 964 965 while (irq_queued > 0 && done < budget) { 966 u32 qid, val = irq_q->q[head]; 967 struct airoha_qdma_desc *desc; 968 struct airoha_queue_entry *e; 969 struct airoha_queue *q; 970 u32 index, desc_ctrl; 971 struct sk_buff *skb; 972 973 if (val == 0xff) 974 break; 975 976 irq_q->q[head] = 0xff; /* mark as done */ 977 head = (head + 1) % irq_q->size; 978 irq_queued--; 979 done++; 980 981 qid = FIELD_GET(IRQ_RING_IDX_MASK, val); 982 if (qid >= ARRAY_SIZE(qdma->q_tx)) 983 continue; 984 985 q = &qdma->q_tx[qid]; 986 if (!q->ndesc) 987 continue; 988 989 index = FIELD_GET(IRQ_DESC_IDX_MASK, val); 990 if (index >= q->ndesc) 991 continue; 992 993 spin_lock_bh(&q->lock); 994 995 if (!q->queued) 996 goto unlock; 997 998 desc = &q->desc[index]; 999 desc_ctrl = le32_to_cpu(desc->ctrl); 1000 1001 if (!(desc_ctrl & QDMA_DESC_DONE_MASK) && 1002 !(desc_ctrl & QDMA_DESC_DROP_MASK)) 1003 goto unlock; 1004 1005 e = &q->entry[index]; 1006 skb = e->skb; 1007 e->skb = NULL; 1008 1009 dma_unmap_single(eth->dev, e->dma_addr, e->dma_len, 1010 DMA_TO_DEVICE); 1011 e->dma_addr = 0; 1012 list_add_tail(&e->list, &q->tx_list); 1013 1014 WRITE_ONCE(desc->msg0, 0); 1015 WRITE_ONCE(desc->msg1, 0); 1016 q->queued--; 1017 1018 if (skb) { 1019 struct netdev_queue *txq; 1020 1021 txq = skb_get_tx_queue(skb->dev, skb); 1022 netdev_tx_completed_queue(txq, 1, skb->len); 1023 dev_kfree_skb_any(skb); 1024 } 1025 1026 if (q->txq_stopped && q->ndesc - q->queued >= q->free_thr) { 1027 /* Since multiple net_device TX queues can share the 1028 * same hw QDMA TX queue, there is no guarantee we have 1029 * inflight packets queued in hw belonging to a 1030 * net_device TX queue stopped in the xmit path. 1031 * In order to avoid any potential net_device TX queue 1032 * stall, we need to wake all the net_device TX queues 1033 * feeding the same hw QDMA TX queue. 1034 */ 1035 airoha_qdma_wake_netdev_txqs(q); 1036 } 1037 1038 unlock: 1039 spin_unlock_bh(&q->lock); 1040 } 1041 1042 if (done) { 1043 int i, len = done >> 7; 1044 1045 for (i = 0; i < len; i++) 1046 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id), 1047 IRQ_CLEAR_LEN_MASK, 0x80); 1048 airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id), 1049 IRQ_CLEAR_LEN_MASK, (done & 0x7f)); 1050 } 1051 1052 if (done < budget && napi_complete(napi)) 1053 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0, 1054 TX_DONE_INT_MASK(id)); 1055 1056 return done; 1057 } 1058 1059 static int airoha_qdma_init_tx_queue(struct airoha_queue *q, 1060 struct airoha_qdma *qdma, int size) 1061 { 1062 struct airoha_eth *eth = qdma->eth; 1063 int i, qid = q - &qdma->q_tx[0]; 1064 dma_addr_t dma_addr; 1065 1066 spin_lock_init(&q->lock); 1067 q->qdma = qdma; 1068 q->free_thr = 1 + MAX_SKB_FRAGS; 1069 INIT_LIST_HEAD(&q->tx_list); 1070 1071 q->entry = devm_kzalloc(eth->dev, size * sizeof(*q->entry), 1072 GFP_KERNEL); 1073 if (!q->entry) 1074 return -ENOMEM; 1075 1076 q->desc = dmam_alloc_coherent(eth->dev, size * sizeof(*q->desc), 1077 &dma_addr, GFP_KERNEL); 1078 if (!q->desc) 1079 return -ENOMEM; 1080 1081 for (i = 0; i < size; i++) { 1082 u32 val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1); 1083 1084 list_add_tail(&q->entry[i].list, &q->tx_list); 1085 WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val)); 1086 } 1087 q->ndesc = size; 1088 1089 /* xmit ring drop default setting */ 1090 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(qid), 1091 TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK); 1092 1093 airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr); 1094 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK, 1095 FIELD_PREP(TX_RING_CPU_IDX_MASK, 0)); 1096 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK, 1097 FIELD_PREP(TX_RING_DMA_IDX_MASK, 0)); 1098 1099 return 0; 1100 } 1101 1102 static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q, 1103 struct airoha_qdma *qdma, int size) 1104 { 1105 int id = irq_q - &qdma->q_tx_irq[0]; 1106 struct airoha_eth *eth = qdma->eth; 1107 dma_addr_t dma_addr; 1108 1109 irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32), 1110 &dma_addr, GFP_KERNEL); 1111 if (!irq_q->q) 1112 return -ENOMEM; 1113 1114 memset(irq_q->q, 0xff, size * sizeof(u32)); 1115 irq_q->size = size; 1116 irq_q->qdma = qdma; 1117 1118 netif_napi_add_tx(eth->napi_dev, &irq_q->napi, 1119 airoha_qdma_tx_napi_poll); 1120 1121 airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr); 1122 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK, 1123 FIELD_PREP(TX_IRQ_DEPTH_MASK, size)); 1124 airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK, 1125 FIELD_PREP(TX_IRQ_THR_MASK, 1)); 1126 1127 return 0; 1128 } 1129 1130 static int airoha_qdma_init_tx(struct airoha_qdma *qdma) 1131 { 1132 int i, err; 1133 1134 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1135 err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma, 1136 IRQ_QUEUE_LEN(i)); 1137 if (err) 1138 return err; 1139 } 1140 1141 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1142 err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma, 1143 TX_DSCP_NUM); 1144 if (err) 1145 return err; 1146 } 1147 1148 return 0; 1149 } 1150 1151 static void airoha_qdma_tx_cleanup(struct airoha_qdma *qdma) 1152 { 1153 u32 status; 1154 int i; 1155 1156 airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG, 1157 GLOBAL_CFG_TX_DMA_EN_MASK); 1158 if (read_poll_timeout(airoha_qdma_rr, status, 1159 !(status & GLOBAL_CFG_TX_DMA_BUSY_MASK), 1160 USEC_PER_MSEC, 50 * USEC_PER_MSEC, true, 1161 qdma, REG_QDMA_GLOBAL_CFG)) 1162 dev_warn(qdma->eth->dev, "QDMA TX DMA busy timeout\n"); 1163 1164 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1165 struct airoha_queue *q = &qdma->q_tx[i]; 1166 u16 index = 0; 1167 int j; 1168 1169 if (!q->ndesc) 1170 continue; 1171 1172 spin_lock_bh(&q->lock); 1173 1174 q->flushing = true; 1175 for (j = 0; j < q->ndesc; j++) { 1176 struct airoha_queue_entry *e = &q->entry[j]; 1177 struct airoha_qdma_desc *desc = &q->desc[j]; 1178 struct sk_buff *skb = e->skb; 1179 1180 if (!e->dma_addr) 1181 continue; 1182 1183 dma_unmap_single(qdma->eth->dev, e->dma_addr, 1184 e->dma_len, DMA_TO_DEVICE); 1185 e->dma_addr = 0; 1186 list_add_tail(&e->list, &q->tx_list); 1187 1188 WRITE_ONCE(desc->ctrl, 0); 1189 WRITE_ONCE(desc->addr, 0); 1190 WRITE_ONCE(desc->data, 0); 1191 WRITE_ONCE(desc->msg0, 0); 1192 WRITE_ONCE(desc->msg1, 0); 1193 WRITE_ONCE(desc->msg2, 0); 1194 1195 if (skb) { 1196 struct netdev_queue *txq; 1197 1198 txq = skb_get_tx_queue(skb->dev, skb); 1199 netdev_tx_completed_queue(txq, 1, skb->len); 1200 dev_kfree_skb_any(skb); 1201 e->skb = NULL; 1202 } 1203 1204 q->queued--; 1205 } 1206 1207 if (!list_empty(&q->tx_list)) { 1208 struct airoha_queue_entry *e; 1209 1210 e = list_first_entry(&q->tx_list, 1211 struct airoha_queue_entry, list); 1212 index = e - q->entry; 1213 } 1214 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(i), TX_RING_CPU_IDX_MASK, 1215 FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); 1216 airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(i), TX_RING_DMA_IDX_MASK, 1217 FIELD_PREP(TX_RING_DMA_IDX_MASK, index)); 1218 1219 spin_unlock_bh(&q->lock); 1220 } 1221 } 1222 1223 static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma) 1224 { 1225 int size, index, num_desc = HW_DSCP_NUM; 1226 struct airoha_eth *eth = qdma->eth; 1227 int id = qdma - ð->qdma[0]; 1228 u32 status, buf_size; 1229 dma_addr_t dma_addr; 1230 const char *name; 1231 1232 name = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d-buf", id); 1233 if (!name) 1234 return -ENOMEM; 1235 1236 buf_size = id ? AIROHA_MAX_PACKET_SIZE / 2 : AIROHA_MAX_PACKET_SIZE; 1237 index = of_property_match_string(eth->dev->of_node, 1238 "memory-region-names", name); 1239 if (index >= 0) { 1240 struct reserved_mem *rmem; 1241 struct device_node *np; 1242 1243 /* Consume reserved memory for hw forwarding buffers queue if 1244 * available in the DTS 1245 */ 1246 np = of_parse_phandle(eth->dev->of_node, "memory-region", 1247 index); 1248 if (!np) 1249 return -ENODEV; 1250 1251 rmem = of_reserved_mem_lookup(np); 1252 of_node_put(np); 1253 if (!rmem) 1254 return -ENODEV; 1255 1256 dma_addr = rmem->base; 1257 /* Compute the number of hw descriptors according to the 1258 * reserved memory size and the payload buffer size 1259 */ 1260 num_desc = div_u64(rmem->size, buf_size); 1261 } else { 1262 size = buf_size * num_desc; 1263 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, 1264 GFP_KERNEL)) 1265 return -ENOMEM; 1266 } 1267 1268 airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr); 1269 1270 size = num_desc * sizeof(struct airoha_qdma_fwd_desc); 1271 if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, GFP_KERNEL)) 1272 return -ENOMEM; 1273 1274 airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr); 1275 /* QDMA0: 2KB. QDMA1: 1KB */ 1276 airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG, 1277 HW_FWD_DSCP_PAYLOAD_SIZE_MASK, 1278 FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK, !!id)); 1279 airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK, 1280 FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128)); 1281 airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG, 1282 LMGR_INIT_START | LMGR_SRAM_MODE_MASK | 1283 HW_FWD_DESC_NUM_MASK, 1284 FIELD_PREP(HW_FWD_DESC_NUM_MASK, num_desc) | 1285 LMGR_INIT_START | LMGR_SRAM_MODE_MASK); 1286 1287 return read_poll_timeout(airoha_qdma_rr, status, 1288 !(status & LMGR_INIT_START), USEC_PER_MSEC, 1289 30 * USEC_PER_MSEC, true, qdma, 1290 REG_LMGR_INIT_CFG); 1291 } 1292 1293 static void airoha_qdma_init_qos(struct airoha_qdma *qdma) 1294 { 1295 airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK); 1296 airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK); 1297 1298 airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG, 1299 PSE_BUF_ESTIMATE_EN_MASK); 1300 1301 airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG, 1302 EGRESS_RATE_METER_EN_MASK | 1303 EGRESS_RATE_METER_EQ_RATE_EN_MASK); 1304 /* 2047us x 31 = 63.457ms */ 1305 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG, 1306 EGRESS_RATE_METER_WINDOW_SZ_MASK, 1307 FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK, 0x1f)); 1308 airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG, 1309 EGRESS_RATE_METER_TIMESLICE_MASK, 1310 FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK, 0x7ff)); 1311 1312 /* ratelimit init */ 1313 airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK); 1314 /* fast-tick 25us */ 1315 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK, 1316 FIELD_PREP(GLB_FAST_TICK_MASK, 25)); 1317 airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK, 1318 FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40)); 1319 1320 airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK); 1321 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK, 1322 FIELD_PREP(EGRESS_FAST_TICK_MASK, 25)); 1323 airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, 1324 EGRESS_SLOW_TICK_RATIO_MASK, 1325 FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40)); 1326 1327 airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK); 1328 airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG, 1329 INGRESS_TRTCM_MODE_MASK); 1330 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK, 1331 FIELD_PREP(INGRESS_FAST_TICK_MASK, 125)); 1332 airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, 1333 INGRESS_SLOW_TICK_RATIO_MASK, 1334 FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8)); 1335 1336 airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK); 1337 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK, 1338 FIELD_PREP(SLA_FAST_TICK_MASK, 25)); 1339 airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK, 1340 FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40)); 1341 } 1342 1343 static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma) 1344 { 1345 int i; 1346 1347 for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) { 1348 /* Tx-cpu transferred count */ 1349 airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0); 1350 airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1), 1351 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK | 1352 CNTR_ALL_DSCP_RING_EN_MASK | 1353 FIELD_PREP(CNTR_CHAN_MASK, i)); 1354 /* Tx-fwd transferred count */ 1355 airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0); 1356 airoha_qdma_wr(qdma, REG_CNTR_CFG((i << 1) + 1), 1357 CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK | 1358 CNTR_ALL_DSCP_RING_EN_MASK | 1359 FIELD_PREP(CNTR_SRC_MASK, 1) | 1360 FIELD_PREP(CNTR_CHAN_MASK, i)); 1361 } 1362 } 1363 1364 static int airoha_qdma_hw_init(struct airoha_qdma *qdma) 1365 { 1366 int i; 1367 1368 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 1369 /* clear pending irqs */ 1370 airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff); 1371 /* setup rx irqs */ 1372 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX0, 1373 INT_RX0_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1374 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX1, 1375 INT_RX1_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1376 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX2, 1377 INT_RX2_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1378 airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX3, 1379 INT_RX3_MASK(RX_IRQ_BANK_PIN_MASK(i))); 1380 } 1381 /* setup tx irqs */ 1382 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0, 1383 TX_COHERENT_LOW_INT_MASK | INT_TX_MASK); 1384 airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX4, 1385 TX_COHERENT_HIGH_INT_MASK); 1386 1387 /* setup irq binding */ 1388 for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { 1389 if (!qdma->q_tx[i].ndesc) 1390 continue; 1391 1392 if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i)) 1393 airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i), 1394 TX_RING_IRQ_BLOCKING_CFG_MASK); 1395 else 1396 airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i), 1397 TX_RING_IRQ_BLOCKING_CFG_MASK); 1398 } 1399 1400 airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG, 1401 FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) | 1402 GLOBAL_CFG_CPU_TXR_RR_MASK | 1403 GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK | 1404 GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK | 1405 GLOBAL_CFG_MULTICAST_EN_MASK | 1406 GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK | 1407 GLOBAL_CFG_TX_WB_DONE_MASK | 1408 FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2)); 1409 1410 airoha_qdma_init_qos(qdma); 1411 1412 /* disable qdma rx delay interrupt */ 1413 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1414 if (!qdma->q_rx[i].ndesc) 1415 continue; 1416 1417 airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i), 1418 RX_DELAY_INT_MASK); 1419 } 1420 1421 airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG, 1422 TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN); 1423 airoha_qdma_init_qos_stats(qdma); 1424 1425 return 0; 1426 } 1427 1428 static irqreturn_t airoha_irq_handler(int irq, void *dev_instance) 1429 { 1430 struct airoha_irq_bank *irq_bank = dev_instance; 1431 struct airoha_qdma *qdma = irq_bank->qdma; 1432 u32 rx_intr_mask = 0, rx_intr1, rx_intr2; 1433 u32 intr[ARRAY_SIZE(irq_bank->irqmask)]; 1434 int i; 1435 1436 for (i = 0; i < ARRAY_SIZE(intr); i++) { 1437 intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i)); 1438 intr[i] &= irq_bank->irqmask[i]; 1439 airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]); 1440 } 1441 1442 if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state)) 1443 return IRQ_NONE; 1444 1445 rx_intr1 = intr[1] & RX_DONE_LOW_INT_MASK; 1446 if (rx_intr1) { 1447 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX1, rx_intr1); 1448 rx_intr_mask |= rx_intr1; 1449 } 1450 1451 rx_intr2 = intr[2] & RX_DONE_HIGH_INT_MASK; 1452 if (rx_intr2) { 1453 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX2, rx_intr2); 1454 rx_intr_mask |= (rx_intr2 << 16); 1455 } 1456 1457 for (i = 0; rx_intr_mask && i < ARRAY_SIZE(qdma->q_rx); i++) { 1458 if (!qdma->q_rx[i].ndesc) 1459 continue; 1460 1461 if (rx_intr_mask & BIT(i)) 1462 napi_schedule(&qdma->q_rx[i].napi); 1463 } 1464 1465 if (intr[0] & INT_TX_MASK) { 1466 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1467 if (!(intr[0] & TX_DONE_INT_MASK(i))) 1468 continue; 1469 1470 airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX0, 1471 TX_DONE_INT_MASK(i)); 1472 napi_schedule(&qdma->q_tx_irq[i].napi); 1473 } 1474 } 1475 1476 return IRQ_HANDLED; 1477 } 1478 1479 static int airoha_qdma_init_irq_banks(struct platform_device *pdev, 1480 struct airoha_qdma *qdma) 1481 { 1482 struct airoha_eth *eth = qdma->eth; 1483 int i, id = qdma - ð->qdma[0]; 1484 1485 for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) { 1486 struct airoha_irq_bank *irq_bank = &qdma->irq_banks[i]; 1487 int err, irq_index = 4 * id + i; 1488 const char *name; 1489 1490 spin_lock_init(&irq_bank->irq_lock); 1491 irq_bank->qdma = qdma; 1492 1493 irq_bank->irq = platform_get_irq(pdev, irq_index); 1494 if (irq_bank->irq < 0) 1495 return irq_bank->irq; 1496 1497 name = devm_kasprintf(eth->dev, GFP_KERNEL, 1498 KBUILD_MODNAME ".%d", irq_index); 1499 if (!name) 1500 return -ENOMEM; 1501 1502 err = devm_request_irq(eth->dev, irq_bank->irq, 1503 airoha_irq_handler, IRQF_SHARED, name, 1504 irq_bank); 1505 if (err) 1506 return err; 1507 } 1508 1509 return 0; 1510 } 1511 1512 static int airoha_qdma_init(struct platform_device *pdev, 1513 struct airoha_eth *eth, 1514 struct airoha_qdma *qdma) 1515 { 1516 int err, id = qdma - ð->qdma[0]; 1517 const char *res; 1518 1519 qdma->eth = eth; 1520 res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id); 1521 if (!res) 1522 return -ENOMEM; 1523 1524 qdma->regs = devm_platform_ioremap_resource_byname(pdev, res); 1525 if (IS_ERR(qdma->regs)) 1526 return dev_err_probe(eth->dev, PTR_ERR(qdma->regs), 1527 "failed to iomap qdma%d regs\n", id); 1528 1529 err = airoha_qdma_init_irq_banks(pdev, qdma); 1530 if (err) 1531 return err; 1532 1533 err = airoha_qdma_init_rx(qdma); 1534 if (err) 1535 return err; 1536 1537 err = airoha_qdma_init_tx(qdma); 1538 if (err) 1539 return err; 1540 1541 err = airoha_qdma_init_hfwd_queues(qdma); 1542 if (err) 1543 return err; 1544 1545 return airoha_qdma_hw_init(qdma); 1546 } 1547 1548 static void airoha_qdma_cleanup(struct airoha_eth *eth, 1549 struct airoha_qdma *qdma) 1550 { 1551 int i; 1552 1553 if (test_bit(DEV_STATE_INITIALIZED, ð->state)) { 1554 u32 status; 1555 1556 airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG, 1557 GLOBAL_CFG_RX_DMA_EN_MASK); 1558 if (read_poll_timeout(airoha_qdma_rr, status, 1559 !(status & GLOBAL_CFG_RX_DMA_BUSY_MASK), 1560 USEC_PER_MSEC, 50 * USEC_PER_MSEC, true, 1561 qdma, REG_QDMA_GLOBAL_CFG)) 1562 dev_warn(eth->dev, "QDMA RX DMA busy timeout\n"); 1563 } 1564 1565 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1566 if (!qdma->q_rx[i].ndesc) 1567 continue; 1568 1569 netif_napi_del(&qdma->q_rx[i].napi); 1570 airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]); 1571 if (qdma->q_rx[i].page_pool) { 1572 page_pool_destroy(qdma->q_rx[i].page_pool); 1573 qdma->q_rx[i].page_pool = NULL; 1574 } 1575 } 1576 1577 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { 1578 if (!qdma->q_tx_irq[i].size) 1579 continue; 1580 1581 netif_napi_del(&qdma->q_tx_irq[i].napi); 1582 } 1583 1584 } 1585 1586 static int airoha_hw_init(struct platform_device *pdev, 1587 struct airoha_eth *eth) 1588 { 1589 int err, i; 1590 1591 /* disable xsi */ 1592 err = reset_control_bulk_assert(eth->soc->num_xsi_rsts, eth->xsi_rsts); 1593 if (err) 1594 return err; 1595 1596 err = reset_control_bulk_assert(ARRAY_SIZE(eth->rsts), eth->rsts); 1597 if (err) 1598 return err; 1599 1600 msleep(20); 1601 err = reset_control_bulk_deassert(ARRAY_SIZE(eth->rsts), eth->rsts); 1602 if (err) 1603 return err; 1604 1605 msleep(20); 1606 err = airoha_fe_init(eth); 1607 if (err) 1608 return err; 1609 1610 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 1611 err = airoha_qdma_init(pdev, eth, ð->qdma[i]); 1612 if (err) 1613 goto error; 1614 } 1615 1616 err = airoha_ppe_init(eth); 1617 if (err) 1618 goto error; 1619 1620 set_bit(DEV_STATE_INITIALIZED, ð->state); 1621 1622 return 0; 1623 error: 1624 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 1625 airoha_qdma_cleanup(eth, ð->qdma[i]); 1626 1627 return err; 1628 } 1629 1630 static void airoha_hw_cleanup(struct airoha_eth *eth) 1631 { 1632 int i; 1633 1634 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) 1635 airoha_qdma_cleanup(eth, ð->qdma[i]); 1636 airoha_ppe_deinit(eth); 1637 } 1638 1639 static void airoha_qdma_start_napi(struct airoha_qdma *qdma) 1640 { 1641 int i; 1642 1643 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) 1644 napi_enable(&qdma->q_tx_irq[i].napi); 1645 1646 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1647 if (!qdma->q_rx[i].ndesc) 1648 continue; 1649 1650 napi_enable(&qdma->q_rx[i].napi); 1651 } 1652 } 1653 1654 static void airoha_qdma_stop_napi(struct airoha_qdma *qdma) 1655 { 1656 int i; 1657 1658 for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) 1659 napi_disable(&qdma->q_tx_irq[i].napi); 1660 1661 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 1662 if (!qdma->q_rx[i].ndesc) 1663 continue; 1664 1665 napi_disable(&qdma->q_rx[i].napi); 1666 } 1667 } 1668 1669 static void airoha_dev_get_hw_stats(struct airoha_gdm_dev *dev) 1670 { 1671 struct airoha_gdm_port *port = dev->port; 1672 struct airoha_eth *eth = dev->eth; 1673 u32 val, i = 0; 1674 1675 /* Read relevant MIB for GDM with multiple port attached */ 1676 if (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX) 1677 airoha_fe_rmw(eth, REG_FE_GDM_MIB_CFG(port->id), 1678 FE_TX_MIB_ID_MASK | FE_RX_MIB_ID_MASK, 1679 FIELD_PREP(FE_TX_MIB_ID_MASK, dev->nbq) | 1680 FIELD_PREP(FE_RX_MIB_ID_MASK, dev->nbq)); 1681 1682 u64_stats_update_begin(&dev->stats.syncp); 1683 1684 /* TX */ 1685 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_H(port->id)); 1686 dev->stats.tx_ok_pkts += ((u64)val << 32); 1687 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_L(port->id)); 1688 dev->stats.tx_ok_pkts += val; 1689 1690 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_H(port->id)); 1691 dev->stats.tx_ok_bytes += ((u64)val << 32); 1692 val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_L(port->id)); 1693 dev->stats.tx_ok_bytes += val; 1694 1695 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_DROP_CNT(port->id)); 1696 dev->stats.tx_drops += val; 1697 1698 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_BC_CNT(port->id)); 1699 dev->stats.tx_broadcast += val; 1700 1701 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_MC_CNT(port->id)); 1702 dev->stats.tx_multicast += val; 1703 1704 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_RUNT_CNT(port->id)); 1705 dev->stats.tx_len[i] += val; 1706 1707 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_H(port->id)); 1708 dev->stats.tx_len[i] += ((u64)val << 32); 1709 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_L(port->id)); 1710 dev->stats.tx_len[i++] += val; 1711 1712 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_H(port->id)); 1713 dev->stats.tx_len[i] += ((u64)val << 32); 1714 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_L(port->id)); 1715 dev->stats.tx_len[i++] += val; 1716 1717 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_H(port->id)); 1718 dev->stats.tx_len[i] += ((u64)val << 32); 1719 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_L(port->id)); 1720 dev->stats.tx_len[i++] += val; 1721 1722 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_H(port->id)); 1723 dev->stats.tx_len[i] += ((u64)val << 32); 1724 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_L(port->id)); 1725 dev->stats.tx_len[i++] += val; 1726 1727 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_H(port->id)); 1728 dev->stats.tx_len[i] += ((u64)val << 32); 1729 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_L(port->id)); 1730 dev->stats.tx_len[i++] += val; 1731 1732 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_H(port->id)); 1733 dev->stats.tx_len[i] += ((u64)val << 32); 1734 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_L(port->id)); 1735 dev->stats.tx_len[i++] += val; 1736 1737 val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_LONG_CNT(port->id)); 1738 dev->stats.tx_len[i++] += val; 1739 1740 /* RX */ 1741 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_H(port->id)); 1742 dev->stats.rx_ok_pkts += ((u64)val << 32); 1743 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_L(port->id)); 1744 dev->stats.rx_ok_pkts += val; 1745 1746 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_H(port->id)); 1747 dev->stats.rx_ok_bytes += ((u64)val << 32); 1748 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_L(port->id)); 1749 dev->stats.rx_ok_bytes += val; 1750 1751 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_DROP_CNT(port->id)); 1752 dev->stats.rx_drops += val; 1753 1754 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_BC_CNT(port->id)); 1755 dev->stats.rx_broadcast += val; 1756 1757 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_MC_CNT(port->id)); 1758 dev->stats.rx_multicast += val; 1759 1760 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ERROR_DROP_CNT(port->id)); 1761 dev->stats.rx_errors += val; 1762 1763 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_CRC_ERR_CNT(port->id)); 1764 dev->stats.rx_crc_error += val; 1765 1766 val = airoha_fe_rr(eth, REG_FE_GDM_RX_OVERFLOW_DROP_CNT(port->id)); 1767 dev->stats.rx_over_errors += val; 1768 1769 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_FRAG_CNT(port->id)); 1770 dev->stats.rx_fragment += val; 1771 1772 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_JABBER_CNT(port->id)); 1773 dev->stats.rx_jabber += val; 1774 1775 i = 0; 1776 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_RUNT_CNT(port->id)); 1777 dev->stats.rx_len[i] += val; 1778 1779 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_H(port->id)); 1780 dev->stats.rx_len[i] += ((u64)val << 32); 1781 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_L(port->id)); 1782 dev->stats.rx_len[i++] += val; 1783 1784 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_H(port->id)); 1785 dev->stats.rx_len[i] += ((u64)val << 32); 1786 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_L(port->id)); 1787 dev->stats.rx_len[i++] += val; 1788 1789 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_H(port->id)); 1790 dev->stats.rx_len[i] += ((u64)val << 32); 1791 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_L(port->id)); 1792 dev->stats.rx_len[i++] += val; 1793 1794 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_H(port->id)); 1795 dev->stats.rx_len[i] += ((u64)val << 32); 1796 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_L(port->id)); 1797 dev->stats.rx_len[i++] += val; 1798 1799 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_H(port->id)); 1800 dev->stats.rx_len[i] += ((u64)val << 32); 1801 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_L(port->id)); 1802 dev->stats.rx_len[i++] += val; 1803 1804 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_H(port->id)); 1805 dev->stats.rx_len[i] += ((u64)val << 32); 1806 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_L(port->id)); 1807 dev->stats.rx_len[i++] += val; 1808 1809 val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_LONG_CNT(port->id)); 1810 dev->stats.rx_len[i++] += val; 1811 1812 u64_stats_update_end(&dev->stats.syncp); 1813 } 1814 1815 static void airoha_update_hw_stats(struct airoha_gdm_dev *dev) 1816 { 1817 struct airoha_gdm_port *port = dev->port; 1818 int i; 1819 1820 spin_lock(&port->stats_lock); 1821 1822 for (i = 0; i < ARRAY_SIZE(port->devs); i++) { 1823 if (port->devs[i]) 1824 airoha_dev_get_hw_stats(port->devs[i]); 1825 } 1826 1827 /* Reset MIB counters */ 1828 airoha_fe_set(dev->eth, REG_FE_GDM_MIB_CLEAR(port->id), 1829 FE_GDM_MIB_RX_CLEAR_MASK | FE_GDM_MIB_TX_CLEAR_MASK); 1830 1831 spin_unlock(&port->stats_lock); 1832 } 1833 1834 static int airoha_dev_open(struct net_device *netdev) 1835 { 1836 int err, len = ETH_HLEN + netdev->mtu + ETH_FCS_LEN; 1837 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1838 struct airoha_gdm_port *port = dev->port; 1839 u32 cur_len, pse_port = FE_PSE_PORT_PPE1; 1840 struct airoha_qdma *qdma = dev->qdma; 1841 1842 netif_tx_start_all_queues(netdev); 1843 err = airoha_set_vip_for_gdm_port(dev, true); 1844 if (err) 1845 return err; 1846 1847 if (netdev_uses_dsa(netdev)) 1848 airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id), 1849 GDM_STAG_EN_MASK); 1850 else 1851 airoha_fe_clear(qdma->eth, REG_GDM_INGRESS_CFG(port->id), 1852 GDM_STAG_EN_MASK); 1853 1854 cur_len = airoha_fe_get(qdma->eth, REG_GDM_LEN_CFG(port->id), 1855 GDM_LONG_LEN_MASK); 1856 if (!port->users || len > cur_len) { 1857 /* Opening a sibling net_device with a larger MTU updates the 1858 * MTU of already running devices. This is required to allow 1859 * multiple net_devices with different MTUs to share the same 1860 * GDM port. 1861 */ 1862 airoha_fe_rmw(qdma->eth, REG_GDM_LEN_CFG(port->id), 1863 GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK, 1864 FIELD_PREP(GDM_SHORT_LEN_MASK, 60) | 1865 FIELD_PREP(GDM_LONG_LEN_MASK, len)); 1866 } 1867 port->users++; 1868 1869 if (!airoha_is_lan_gdm_dev(dev) && 1870 airoha_ppe_is_enabled(qdma->eth, 1)) 1871 pse_port = FE_PSE_PORT_PPE2; 1872 airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id), 1873 pse_port); 1874 1875 return 0; 1876 } 1877 1878 static void airoha_set_port_mtu(struct airoha_eth *eth, 1879 struct airoha_gdm_port *port) 1880 { 1881 u32 len = 0; 1882 int i; 1883 1884 for (i = 0; i < ARRAY_SIZE(port->devs); i++) { 1885 struct airoha_gdm_dev *dev = port->devs[i]; 1886 struct net_device *netdev; 1887 1888 if (!dev) 1889 continue; 1890 1891 netdev = netdev_from_priv(dev); 1892 if (netif_running(netdev)) 1893 len = max_t(u32, len, netdev->mtu); 1894 } 1895 len += ETH_HLEN + ETH_FCS_LEN; 1896 1897 airoha_fe_rmw(eth, REG_GDM_LEN_CFG(port->id), 1898 GDM_LONG_LEN_MASK, 1899 FIELD_PREP(GDM_LONG_LEN_MASK, len)); 1900 } 1901 1902 static int airoha_dev_stop(struct net_device *netdev) 1903 { 1904 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1905 struct airoha_gdm_port *port = dev->port; 1906 struct airoha_qdma *qdma = dev->qdma; 1907 1908 netif_tx_disable(netdev); 1909 airoha_set_vip_for_gdm_port(dev, false); 1910 1911 if (--port->users) 1912 airoha_set_port_mtu(dev->eth, port); 1913 else 1914 airoha_set_gdm_port_fwd_cfg(qdma->eth, 1915 REG_GDM_FWD_CFG(port->id), 1916 FE_PSE_PORT_DROP); 1917 return 0; 1918 } 1919 1920 static int airoha_dev_set_macaddr(struct net_device *netdev, void *p) 1921 { 1922 struct airoha_gdm_dev *dev = netdev_priv(netdev); 1923 struct sockaddr *addr = p; 1924 int err; 1925 1926 err = eth_prepare_mac_addr_change(netdev, p); 1927 if (err) 1928 return err; 1929 1930 err = airoha_set_macaddr(dev, addr->sa_data); 1931 if (err) 1932 return err; 1933 1934 eth_commit_mac_addr_change(netdev, p); 1935 1936 return 0; 1937 } 1938 1939 static int airoha_enable_gdm2_loopback(struct airoha_gdm_dev *dev) 1940 { 1941 struct airoha_gdm_port *port = dev->port; 1942 struct airoha_eth *eth = dev->eth; 1943 u32 val, pse_port, chan; 1944 int i, src_port; 1945 1946 src_port = eth->soc->ops.get_sport(port, dev->nbq); 1947 if (src_port < 0) 1948 return src_port; 1949 1950 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1951 FE_PSE_PORT_DROP); 1952 airoha_fe_clear(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1953 GDM_STRIP_CRC_MASK); 1954 1955 /* Enable GDM2 loopback */ 1956 airoha_fe_wr(eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), 0xffffffff); 1957 airoha_fe_wr(eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX), 0xffff); 1958 1959 chan = port->id == AIROHA_GDM3_IDX ? airoha_is_7581(eth) ? 4 : 3 : 0; 1960 airoha_fe_rmw(eth, REG_GDM_LPBK_CFG(AIROHA_GDM2_IDX), 1961 LPBK_CHAN_MASK | LPBK_MODE_MASK | LPBK_EN_MASK, 1962 FIELD_PREP(LPBK_CHAN_MASK, chan) | 1963 LBK_GAP_MODE_MASK | LBK_LEN_MODE_MASK | 1964 LBK_CHAN_MODE_MASK | LPBK_EN_MASK); 1965 airoha_fe_rmw(eth, REG_GDM_LEN_CFG(AIROHA_GDM2_IDX), 1966 GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK, 1967 FIELD_PREP(GDM_SHORT_LEN_MASK, 60) | 1968 FIELD_PREP(GDM_LONG_LEN_MASK, AIROHA_MAX_MTU)); 1969 /* Forward the traffic to the proper GDM port */ 1970 pse_port = port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3 1971 : FE_PSE_PORT_GDM4; 1972 airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX), 1973 pse_port); 1974 1975 /* Disable VIP and IFC for GDM2 */ 1976 airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX)); 1977 airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX)); 1978 1979 airoha_fe_rmw(eth, REG_FE_WAN_PORT, 1980 WAN1_EN_MASK | WAN1_MASK | WAN0_MASK, 1981 FIELD_PREP(WAN0_MASK, src_port)); 1982 val = src_port & SP_CPORT_DFT_MASK; 1983 airoha_fe_rmw(eth, 1984 REG_SP_DFT_CPORT(src_port >> fls(SP_CPORT_DFT_MASK)), 1985 SP_CPORT_MASK(val), 1986 __field_prep(SP_CPORT_MASK(val), FE_PSE_PORT_CDM2)); 1987 1988 for (i = 0; i < eth->soc->num_ppe; i++) 1989 airoha_ppe_set_cpu_port(dev, i, AIROHA_GDM2_IDX); 1990 1991 if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) { 1992 u32 mask = FC_ID_OF_SRC_PORT_MASK(dev->nbq); 1993 1994 airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask, 1995 __field_prep(mask, AIROHA_GDM2_IDX)); 1996 } 1997 1998 return 0; 1999 } 2000 2001 static struct airoha_gdm_dev * 2002 airoha_get_wan_gdm_dev(struct airoha_eth *eth) 2003 { 2004 int i; 2005 2006 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 2007 struct airoha_gdm_port *port = eth->ports[i]; 2008 int j; 2009 2010 if (!port) 2011 continue; 2012 2013 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 2014 struct airoha_gdm_dev *dev = port->devs[j]; 2015 2016 if (dev && !airoha_is_lan_gdm_dev(dev)) 2017 return dev; 2018 } 2019 } 2020 2021 return NULL; 2022 } 2023 2024 static void airoha_dev_set_qdma(struct airoha_gdm_dev *dev) 2025 { 2026 struct net_device *netdev = netdev_from_priv(dev); 2027 struct airoha_eth *eth = dev->eth; 2028 int ppe_id; 2029 2030 /* QDMA0 is used for lan ports while QDMA1 is used for WAN ports */ 2031 dev->qdma = ð->qdma[!airoha_is_lan_gdm_dev(dev)]; 2032 netdev->irq = dev->qdma->irq_banks[0].irq; 2033 2034 ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1); 2035 airoha_ppe_set_cpu_port(dev, ppe_id, airoha_get_fe_port(dev)); 2036 } 2037 2038 static int airoha_dev_init(struct net_device *netdev) 2039 { 2040 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2041 struct airoha_gdm_port *port = dev->port; 2042 2043 switch (port->id) { 2044 case AIROHA_GDM3_IDX: 2045 case AIROHA_GDM4_IDX: 2046 if (airoha_get_wan_gdm_dev(dev->eth)) 2047 break; 2048 fallthrough; 2049 case AIROHA_GDM2_IDX: 2050 /* GDM2 is always used as wan */ 2051 dev->flags |= AIROHA_PRIV_F_WAN; 2052 break; 2053 default: 2054 break; 2055 } 2056 2057 airoha_dev_set_qdma(dev); 2058 airoha_set_macaddr(dev, netdev->dev_addr); 2059 2060 if (!airoha_is_lan_gdm_dev(dev) && 2061 (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX)) { 2062 int err; 2063 2064 err = airoha_enable_gdm2_loopback(dev); 2065 if (err) 2066 return err; 2067 } 2068 2069 return 0; 2070 } 2071 2072 static void airoha_dev_get_stats64(struct net_device *netdev, 2073 struct rtnl_link_stats64 *storage) 2074 { 2075 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2076 unsigned int start; 2077 2078 airoha_update_hw_stats(dev); 2079 do { 2080 start = u64_stats_fetch_begin(&dev->stats.syncp); 2081 storage->rx_packets = dev->stats.rx_ok_pkts; 2082 storage->tx_packets = dev->stats.tx_ok_pkts; 2083 storage->rx_bytes = dev->stats.rx_ok_bytes; 2084 storage->tx_bytes = dev->stats.tx_ok_bytes; 2085 storage->multicast = dev->stats.rx_multicast; 2086 storage->rx_errors = dev->stats.rx_errors; 2087 storage->rx_dropped = dev->stats.rx_drops; 2088 storage->tx_dropped = dev->stats.tx_drops; 2089 storage->rx_crc_errors = dev->stats.rx_crc_error; 2090 storage->rx_over_errors = dev->stats.rx_over_errors; 2091 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2092 } 2093 2094 static int airoha_dev_change_mtu(struct net_device *netdev, int mtu) 2095 { 2096 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2097 struct airoha_gdm_port *port = dev->port; 2098 2099 WRITE_ONCE(netdev->mtu, mtu); 2100 if (port->users) 2101 airoha_set_port_mtu(dev->eth, port); 2102 2103 return 0; 2104 } 2105 2106 static u16 airoha_dev_select_queue(struct net_device *netdev, 2107 struct sk_buff *skb, 2108 struct net_device *sb_dev) 2109 { 2110 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2111 struct airoha_gdm_port *port = dev->port; 2112 int queue, channel; 2113 2114 /* For dsa device select QoS channel according to the dsa user port 2115 * index, rely on port id otherwise. Select QoS queue based on the 2116 * skb priority. 2117 */ 2118 channel = netdev_uses_dsa(netdev) ? skb_get_queue_mapping(skb) : port->id; 2119 channel = channel % AIROHA_NUM_QOS_CHANNELS; 2120 queue = skb->priority % AIROHA_NUM_QOS_QUEUES; 2121 queue = channel * AIROHA_NUM_QOS_QUEUES + queue; 2122 2123 return queue < netdev->num_tx_queues ? queue : 0; 2124 } 2125 2126 static u32 airoha_get_dsa_tag(struct sk_buff *skb, struct net_device *dev) 2127 { 2128 #if IS_ENABLED(CONFIG_NET_DSA) 2129 struct ethhdr *ehdr; 2130 u8 xmit_tpid; 2131 u16 tag; 2132 2133 if (!netdev_uses_dsa(dev)) 2134 return 0; 2135 2136 if (dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 2137 return 0; 2138 2139 if (skb_cow_head(skb, 0)) 2140 return 0; 2141 2142 ehdr = (struct ethhdr *)skb->data; 2143 tag = be16_to_cpu(ehdr->h_proto); 2144 xmit_tpid = tag >> 8; 2145 2146 switch (xmit_tpid) { 2147 case MTK_HDR_XMIT_TAGGED_TPID_8100: 2148 ehdr->h_proto = cpu_to_be16(ETH_P_8021Q); 2149 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_8100 << 8); 2150 break; 2151 case MTK_HDR_XMIT_TAGGED_TPID_88A8: 2152 ehdr->h_proto = cpu_to_be16(ETH_P_8021AD); 2153 tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_88A8 << 8); 2154 break; 2155 default: 2156 /* PPE module requires untagged DSA packets to work properly, 2157 * so move DSA tag to DMA descriptor. 2158 */ 2159 memmove(skb->data + MTK_HDR_LEN, skb->data, 2 * ETH_ALEN); 2160 __skb_pull(skb, MTK_HDR_LEN); 2161 break; 2162 } 2163 2164 return tag; 2165 #else 2166 return 0; 2167 #endif 2168 } 2169 2170 int airoha_get_fe_port(struct airoha_gdm_dev *dev) 2171 { 2172 struct airoha_gdm_port *port = dev->port; 2173 struct airoha_eth *eth = dev->eth; 2174 2175 switch (eth->soc->version) { 2176 case 0x7583: 2177 return port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3 2178 : port->id; 2179 case 0x7581: 2180 default: 2181 return port->id == AIROHA_GDM4_IDX ? FE_PSE_PORT_GDM4 2182 : port->id; 2183 } 2184 } 2185 2186 static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, 2187 struct net_device *netdev) 2188 { 2189 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2190 struct airoha_qdma *qdma = dev->qdma; 2191 u32 nr_frags, tag, msg0, msg1, len; 2192 struct airoha_queue_entry *e; 2193 struct netdev_queue *txq; 2194 struct airoha_queue *q; 2195 LIST_HEAD(tx_list); 2196 int i = 0, qid; 2197 void *data; 2198 u16 index; 2199 u8 fport; 2200 2201 qid = airoha_qdma_get_txq(qdma, skb_get_queue_mapping(skb)); 2202 tag = airoha_get_dsa_tag(skb, netdev); 2203 2204 msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK, 2205 qid / AIROHA_NUM_QOS_QUEUES) | 2206 FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK, 2207 qid % AIROHA_NUM_QOS_QUEUES) | 2208 FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK, tag); 2209 if (skb->ip_summed == CHECKSUM_PARTIAL) 2210 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TCO_MASK, 1) | 2211 FIELD_PREP(QDMA_ETH_TXMSG_UCO_MASK, 1) | 2212 FIELD_PREP(QDMA_ETH_TXMSG_ICO_MASK, 1); 2213 2214 /* TSO: fill MSS info in tcp checksum field */ 2215 if (skb_is_gso(skb)) { 2216 if (skb_cow_head(skb, 0)) 2217 goto error; 2218 2219 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | 2220 SKB_GSO_TCPV6)) { 2221 __be16 csum = cpu_to_be16(skb_shinfo(skb)->gso_size); 2222 2223 tcp_hdr(skb)->check = (__force __sum16)csum; 2224 msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TSO_MASK, 1); 2225 } 2226 } 2227 2228 fport = airoha_get_fe_port(dev); 2229 msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) | 2230 FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) | 2231 FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f); 2232 2233 q = &qdma->q_tx[qid]; 2234 if (WARN_ON_ONCE(!q->ndesc)) 2235 goto error; 2236 2237 spin_lock_bh(&q->lock); 2238 2239 if (q->flushing) 2240 goto error_unlock; 2241 2242 txq = skb_get_tx_queue(netdev, skb); 2243 nr_frags = 1 + skb_shinfo(skb)->nr_frags; 2244 2245 if (q->queued + nr_frags >= q->ndesc) { 2246 /* not enough space in the queue */ 2247 netif_tx_stop_queue(txq); 2248 q->txq_stopped = true; 2249 spin_unlock_bh(&q->lock); 2250 return NETDEV_TX_BUSY; 2251 } 2252 2253 len = skb_headlen(skb); 2254 data = skb->data; 2255 2256 e = list_first_entry(&q->tx_list, struct airoha_queue_entry, 2257 list); 2258 index = e - q->entry; 2259 2260 while (true) { 2261 struct airoha_qdma_desc *desc = &q->desc[index]; 2262 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2263 dma_addr_t addr; 2264 u32 val; 2265 2266 addr = dma_map_single(netdev->dev.parent, data, len, 2267 DMA_TO_DEVICE); 2268 if (unlikely(dma_mapping_error(netdev->dev.parent, addr))) 2269 goto error_unmap; 2270 2271 list_move_tail(&e->list, &tx_list); 2272 e->skb = i == nr_frags - 1 ? skb : NULL; 2273 e->dma_addr = addr; 2274 e->dma_len = len; 2275 2276 e = list_first_entry(&q->tx_list, struct airoha_queue_entry, 2277 list); 2278 index = e - q->entry; 2279 2280 val = FIELD_PREP(QDMA_DESC_LEN_MASK, len); 2281 if (i < nr_frags - 1) 2282 val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1); 2283 WRITE_ONCE(desc->ctrl, cpu_to_le32(val)); 2284 WRITE_ONCE(desc->addr, cpu_to_le32(addr)); 2285 val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, index); 2286 WRITE_ONCE(desc->data, cpu_to_le32(val)); 2287 WRITE_ONCE(desc->msg0, cpu_to_le32(msg0)); 2288 WRITE_ONCE(desc->msg1, cpu_to_le32(msg1)); 2289 WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff)); 2290 2291 if (++i == nr_frags) 2292 break; 2293 2294 data = skb_frag_address(frag); 2295 len = skb_frag_size(frag); 2296 } 2297 q->queued += i; 2298 2299 skb_tx_timestamp(skb); 2300 netdev_tx_sent_queue(txq, skb->len); 2301 if (q->ndesc - q->queued < q->free_thr) { 2302 netif_tx_stop_queue(txq); 2303 q->txq_stopped = true; 2304 } 2305 2306 if (netif_xmit_stopped(txq) || !netdev_xmit_more()) 2307 airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), 2308 TX_RING_CPU_IDX_MASK, 2309 FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); 2310 2311 spin_unlock_bh(&q->lock); 2312 2313 return NETDEV_TX_OK; 2314 2315 error_unmap: 2316 list_for_each_entry(e, &tx_list, list) { 2317 dma_unmap_single(netdev->dev.parent, e->dma_addr, e->dma_len, 2318 DMA_TO_DEVICE); 2319 e->dma_addr = 0; 2320 } 2321 list_splice(&tx_list, &q->tx_list); 2322 error_unlock: 2323 spin_unlock_bh(&q->lock); 2324 error: 2325 dev_kfree_skb_any(skb); 2326 netdev->stats.tx_dropped++; 2327 2328 return NETDEV_TX_OK; 2329 } 2330 2331 static void airoha_ethtool_get_drvinfo(struct net_device *netdev, 2332 struct ethtool_drvinfo *info) 2333 { 2334 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2335 struct airoha_eth *eth = dev->eth; 2336 2337 strscpy(info->driver, eth->dev->driver->name, sizeof(info->driver)); 2338 strscpy(info->bus_info, dev_name(eth->dev), sizeof(info->bus_info)); 2339 } 2340 2341 static void airoha_ethtool_get_mac_stats(struct net_device *netdev, 2342 struct ethtool_eth_mac_stats *stats) 2343 { 2344 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2345 unsigned int start; 2346 2347 airoha_update_hw_stats(dev); 2348 do { 2349 start = u64_stats_fetch_begin(&dev->stats.syncp); 2350 stats->FramesTransmittedOK = dev->stats.tx_ok_pkts; 2351 stats->OctetsTransmittedOK = dev->stats.tx_ok_bytes; 2352 stats->MulticastFramesXmittedOK = dev->stats.tx_multicast; 2353 stats->BroadcastFramesXmittedOK = dev->stats.tx_broadcast; 2354 stats->FramesReceivedOK = dev->stats.rx_ok_pkts; 2355 stats->OctetsReceivedOK = dev->stats.rx_ok_bytes; 2356 stats->BroadcastFramesReceivedOK = dev->stats.rx_broadcast; 2357 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2358 } 2359 2360 static const struct ethtool_rmon_hist_range airoha_ethtool_rmon_ranges[] = { 2361 { 0, 64 }, 2362 { 65, 127 }, 2363 { 128, 255 }, 2364 { 256, 511 }, 2365 { 512, 1023 }, 2366 { 1024, 1518 }, 2367 { 1519, 10239 }, 2368 {}, 2369 }; 2370 2371 static void 2372 airoha_ethtool_get_rmon_stats(struct net_device *netdev, 2373 struct ethtool_rmon_stats *stats, 2374 const struct ethtool_rmon_hist_range **ranges) 2375 { 2376 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2377 struct airoha_hw_stats *hw_stats = &dev->stats; 2378 unsigned int start; 2379 2380 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) != 2381 ARRAY_SIZE(hw_stats->tx_len) + 1); 2382 BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) != 2383 ARRAY_SIZE(hw_stats->rx_len) + 1); 2384 2385 *ranges = airoha_ethtool_rmon_ranges; 2386 airoha_update_hw_stats(dev); 2387 do { 2388 int i; 2389 2390 start = u64_stats_fetch_begin(&dev->stats.syncp); 2391 stats->fragments = hw_stats->rx_fragment; 2392 stats->jabbers = hw_stats->rx_jabber; 2393 for (i = 0; i < ARRAY_SIZE(airoha_ethtool_rmon_ranges) - 1; 2394 i++) { 2395 stats->hist[i] = hw_stats->rx_len[i]; 2396 stats->hist_tx[i] = hw_stats->tx_len[i]; 2397 } 2398 } while (u64_stats_fetch_retry(&dev->stats.syncp, start)); 2399 } 2400 2401 static int airoha_qdma_set_chan_tx_sched(struct net_device *netdev, 2402 int channel, enum tx_sched_mode mode, 2403 const u16 *weights, u8 n_weights) 2404 { 2405 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2406 int i; 2407 2408 for (i = 0; i < AIROHA_NUM_QOS_QUEUES; i++) 2409 airoha_qdma_clear(dev->qdma, REG_QUEUE_CLOSE_CFG(channel), 2410 TXQ_DISABLE_CHAN_QUEUE_MASK(channel, i)); 2411 2412 for (i = 0; i < n_weights; i++) { 2413 u32 status; 2414 int err; 2415 2416 airoha_qdma_wr(dev->qdma, REG_TXWRR_WEIGHT_CFG, 2417 TWRR_RW_CMD_MASK | 2418 FIELD_PREP(TWRR_CHAN_IDX_MASK, channel) | 2419 FIELD_PREP(TWRR_QUEUE_IDX_MASK, i) | 2420 FIELD_PREP(TWRR_VALUE_MASK, weights[i])); 2421 err = read_poll_timeout(airoha_qdma_rr, status, 2422 status & TWRR_RW_CMD_DONE, 2423 USEC_PER_MSEC, 10 * USEC_PER_MSEC, 2424 true, dev->qdma, REG_TXWRR_WEIGHT_CFG); 2425 if (err) 2426 return err; 2427 } 2428 2429 airoha_qdma_rmw(dev->qdma, REG_CHAN_QOS_MODE(channel >> 3), 2430 CHAN_QOS_MODE_MASK(channel), 2431 __field_prep(CHAN_QOS_MODE_MASK(channel), mode)); 2432 2433 return 0; 2434 } 2435 2436 static int airoha_qdma_set_tx_prio_sched(struct net_device *dev, int channel) 2437 { 2438 static const u16 w[AIROHA_NUM_QOS_QUEUES] = {}; 2439 2440 return airoha_qdma_set_chan_tx_sched(dev, channel, TC_SCH_SP, w, 2441 ARRAY_SIZE(w)); 2442 } 2443 2444 static int airoha_qdma_set_tx_ets_sched(struct net_device *dev, int channel, 2445 struct tc_ets_qopt_offload *opt) 2446 { 2447 struct tc_ets_qopt_offload_replace_params *p = &opt->replace_params; 2448 enum tx_sched_mode mode = TC_SCH_SP; 2449 u16 w[AIROHA_NUM_QOS_QUEUES] = {}; 2450 int i, nstrict = 0; 2451 2452 if (p->bands > AIROHA_NUM_QOS_QUEUES) 2453 return -EINVAL; 2454 2455 for (i = 0; i < p->bands; i++) { 2456 if (!p->quanta[i]) 2457 nstrict++; 2458 } 2459 2460 /* this configuration is not supported by the hw */ 2461 if (nstrict == AIROHA_NUM_QOS_QUEUES - 1) 2462 return -EINVAL; 2463 2464 /* EN7581 SoC supports fixed QoS band priority where WRR queues have 2465 * lowest priorities with respect to SP ones. 2466 * e.g: WRR0, WRR1, .., WRRm, SP0, SP1, .., SPn 2467 */ 2468 for (i = 0; i < nstrict; i++) { 2469 if (p->priomap[p->bands - i - 1] != i) 2470 return -EINVAL; 2471 } 2472 2473 for (i = 0; i < p->bands - nstrict; i++) { 2474 if (p->priomap[i] != nstrict + i) 2475 return -EINVAL; 2476 2477 w[i] = p->weights[nstrict + i]; 2478 } 2479 2480 if (!nstrict) 2481 mode = TC_SCH_WRR8; 2482 else if (nstrict < AIROHA_NUM_QOS_QUEUES - 1) 2483 mode = nstrict + 1; 2484 2485 return airoha_qdma_set_chan_tx_sched(dev, channel, mode, w, 2486 ARRAY_SIZE(w)); 2487 } 2488 2489 static int airoha_qdma_get_tx_ets_stats(struct net_device *netdev, int channel, 2490 struct tc_ets_qopt_offload *opt) 2491 { 2492 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2493 struct airoha_qdma *qdma = dev->qdma; 2494 2495 u64 cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1)); 2496 u64 fwd_tx_packets = airoha_qdma_rr(qdma, 2497 REG_CNTR_VAL((channel << 1) + 1)); 2498 u64 tx_packets = (cpu_tx_packets - dev->cpu_tx_packets) + 2499 (fwd_tx_packets - dev->fwd_tx_packets); 2500 2501 _bstats_update(opt->stats.bstats, 0, tx_packets); 2502 dev->cpu_tx_packets = cpu_tx_packets; 2503 dev->fwd_tx_packets = fwd_tx_packets; 2504 2505 return 0; 2506 } 2507 2508 static int airoha_tc_setup_qdisc_ets(struct net_device *dev, 2509 struct tc_ets_qopt_offload *opt) 2510 { 2511 int channel; 2512 2513 if (opt->parent == TC_H_ROOT) 2514 return -EINVAL; 2515 2516 channel = TC_H_MAJ(opt->handle) >> 16; 2517 channel = channel % AIROHA_NUM_QOS_CHANNELS; 2518 2519 switch (opt->command) { 2520 case TC_ETS_REPLACE: 2521 return airoha_qdma_set_tx_ets_sched(dev, channel, opt); 2522 case TC_ETS_DESTROY: 2523 /* PRIO is default qdisc scheduler */ 2524 return airoha_qdma_set_tx_prio_sched(dev, channel); 2525 case TC_ETS_STATS: 2526 return airoha_qdma_get_tx_ets_stats(dev, channel, opt); 2527 default: 2528 return -EOPNOTSUPP; 2529 } 2530 } 2531 2532 static int airoha_qdma_get_rl_param(struct airoha_qdma *qdma, int queue_id, 2533 u32 addr, enum trtcm_param_type param, 2534 u32 *val_low, u32 *val_high) 2535 { 2536 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id); 2537 u32 val, config = FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) | 2538 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) | 2539 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx); 2540 2541 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2542 if (read_poll_timeout(airoha_qdma_rr, val, 2543 val & RATE_LIMIT_PARAM_RW_DONE_MASK, 2544 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, qdma, 2545 REG_TRTCM_CFG_PARAM(addr))) 2546 return -ETIMEDOUT; 2547 2548 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr)); 2549 if (val_high) 2550 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr)); 2551 2552 return 0; 2553 } 2554 2555 static int airoha_qdma_set_rl_param(struct airoha_qdma *qdma, int queue_id, 2556 u32 addr, enum trtcm_param_type param, 2557 u32 val) 2558 { 2559 u32 idx = QDMA_METER_IDX(queue_id), group = QDMA_METER_GROUP(queue_id); 2560 u32 config = RATE_LIMIT_PARAM_RW_MASK | 2561 FIELD_PREP(RATE_LIMIT_PARAM_TYPE_MASK, param) | 2562 FIELD_PREP(RATE_LIMIT_METER_GROUP_MASK, group) | 2563 FIELD_PREP(RATE_LIMIT_PARAM_INDEX_MASK, idx); 2564 2565 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); 2566 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2567 2568 return read_poll_timeout(airoha_qdma_rr, val, 2569 val & RATE_LIMIT_PARAM_RW_DONE_MASK, 2570 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2571 qdma, REG_TRTCM_CFG_PARAM(addr)); 2572 } 2573 2574 static int airoha_qdma_set_rl_config(struct airoha_qdma *qdma, int queue_id, 2575 u32 addr, bool enable, u32 enable_mask) 2576 { 2577 u32 val; 2578 int err; 2579 2580 err = airoha_qdma_get_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE, 2581 &val, NULL); 2582 if (err) 2583 return err; 2584 2585 val = enable ? val | enable_mask : val & ~enable_mask; 2586 2587 return airoha_qdma_set_rl_param(qdma, queue_id, addr, TRTCM_MISC_MODE, 2588 val); 2589 } 2590 2591 static int airoha_qdma_set_rl_token_bucket(struct airoha_qdma *qdma, 2592 int queue_id, u32 rate_val, 2593 u32 bucket_size) 2594 { 2595 u32 val, config, tick, unit, rate, rate_frac; 2596 int err; 2597 2598 err = airoha_qdma_get_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2599 TRTCM_MISC_MODE, &config, NULL); 2600 if (err) 2601 return err; 2602 2603 val = airoha_qdma_rr(qdma, REG_INGRESS_TRTCM_CFG); 2604 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val); 2605 if (config & TRTCM_TICK_SEL) 2606 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val); 2607 if (!tick) 2608 return -EINVAL; 2609 2610 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick; 2611 if (!unit) 2612 return -EINVAL; 2613 2614 rate = rate_val / unit; 2615 rate_frac = rate_val % unit; 2616 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit; 2617 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) | 2618 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac); 2619 2620 err = airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2621 TRTCM_TOKEN_RATE_MODE, rate); 2622 if (err) 2623 return err; 2624 2625 val = bucket_size; 2626 if (!(config & TRTCM_PKT_MODE)) 2627 val = max_t(u32, val, MIN_TOKEN_SIZE); 2628 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET); 2629 2630 return airoha_qdma_set_rl_param(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2631 TRTCM_BUCKETSIZE_SHIFT_MODE, val); 2632 } 2633 2634 static int airoha_qdma_init_rl_config(struct airoha_qdma *qdma, int queue_id, 2635 bool enable, enum trtcm_unit_type unit) 2636 { 2637 bool tick_sel = queue_id == 0 || queue_id == 2 || queue_id == 8; 2638 enum trtcm_param mode = TRTCM_METER_MODE; 2639 int err; 2640 2641 mode |= unit == TRTCM_PACKET_UNIT ? TRTCM_PKT_MODE : 0; 2642 err = airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2643 enable, mode); 2644 if (err) 2645 return err; 2646 2647 return airoha_qdma_set_rl_config(qdma, queue_id, REG_INGRESS_TRTCM_CFG, 2648 tick_sel, TRTCM_TICK_SEL); 2649 } 2650 2651 static int airoha_qdma_get_trtcm_param(struct airoha_qdma *qdma, int channel, 2652 u32 addr, enum trtcm_param_type param, 2653 enum trtcm_mode_type mode, 2654 u32 *val_low, u32 *val_high) 2655 { 2656 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel); 2657 u32 val, config = FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) | 2658 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) | 2659 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) | 2660 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode); 2661 2662 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2663 if (read_poll_timeout(airoha_qdma_rr, val, 2664 val & TRTCM_PARAM_RW_DONE_MASK, 2665 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2666 qdma, REG_TRTCM_CFG_PARAM(addr))) 2667 return -ETIMEDOUT; 2668 2669 *val_low = airoha_qdma_rr(qdma, REG_TRTCM_DATA_LOW(addr)); 2670 if (val_high) 2671 *val_high = airoha_qdma_rr(qdma, REG_TRTCM_DATA_HIGH(addr)); 2672 2673 return 0; 2674 } 2675 2676 static int airoha_qdma_set_trtcm_param(struct airoha_qdma *qdma, int channel, 2677 u32 addr, enum trtcm_param_type param, 2678 enum trtcm_mode_type mode, u32 val) 2679 { 2680 u32 idx = QDMA_METER_IDX(channel), group = QDMA_METER_GROUP(channel); 2681 u32 config = TRTCM_PARAM_RW_MASK | 2682 FIELD_PREP(TRTCM_PARAM_TYPE_MASK, param) | 2683 FIELD_PREP(TRTCM_METER_GROUP_MASK, group) | 2684 FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) | 2685 FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode); 2686 2687 airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); 2688 airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); 2689 2690 return read_poll_timeout(airoha_qdma_rr, val, 2691 val & TRTCM_PARAM_RW_DONE_MASK, 2692 USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, 2693 qdma, REG_TRTCM_CFG_PARAM(addr)); 2694 } 2695 2696 static int airoha_qdma_set_trtcm_config(struct airoha_qdma *qdma, int channel, 2697 u32 addr, enum trtcm_mode_type mode, 2698 bool enable, u32 enable_mask) 2699 { 2700 u32 val; 2701 2702 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2703 mode, &val, NULL)) 2704 return -EINVAL; 2705 2706 val = enable ? val | enable_mask : val & ~enable_mask; 2707 2708 return airoha_qdma_set_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2709 mode, val); 2710 } 2711 2712 static int airoha_qdma_set_trtcm_token_bucket(struct airoha_qdma *qdma, 2713 int channel, u32 addr, 2714 enum trtcm_mode_type mode, 2715 u32 rate_val, u32 bucket_size) 2716 { 2717 u32 val, config, tick, unit, rate, rate_frac; 2718 int err; 2719 2720 if (airoha_qdma_get_trtcm_param(qdma, channel, addr, TRTCM_MISC_MODE, 2721 mode, &config, NULL)) 2722 return -EINVAL; 2723 2724 val = airoha_qdma_rr(qdma, addr); 2725 tick = FIELD_GET(INGRESS_FAST_TICK_MASK, val); 2726 if (config & TRTCM_TICK_SEL) 2727 tick *= FIELD_GET(INGRESS_SLOW_TICK_RATIO_MASK, val); 2728 if (!tick) 2729 return -EINVAL; 2730 2731 unit = (config & TRTCM_PKT_MODE) ? 1000000 / tick : 8000 / tick; 2732 if (!unit) 2733 return -EINVAL; 2734 2735 rate = rate_val / unit; 2736 rate_frac = rate_val % unit; 2737 rate_frac = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate_frac) / unit; 2738 rate = FIELD_PREP(TRTCM_TOKEN_RATE_MASK, rate) | 2739 FIELD_PREP(TRTCM_TOKEN_RATE_FRACTION_MASK, rate_frac); 2740 2741 err = airoha_qdma_set_trtcm_param(qdma, channel, addr, 2742 TRTCM_TOKEN_RATE_MODE, mode, rate); 2743 if (err) 2744 return err; 2745 2746 val = max_t(u32, bucket_size, MIN_TOKEN_SIZE); 2747 val = min_t(u32, __fls(val), MAX_TOKEN_SIZE_OFFSET); 2748 2749 return airoha_qdma_set_trtcm_param(qdma, channel, addr, 2750 TRTCM_BUCKETSIZE_SHIFT_MODE, 2751 mode, val); 2752 } 2753 2754 static int airoha_qdma_set_tx_rate_limit(struct net_device *netdev, 2755 int channel, u32 rate, 2756 u32 bucket_size) 2757 { 2758 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2759 int i, err; 2760 2761 for (i = 0; i <= TRTCM_PEAK_MODE; i++) { 2762 err = airoha_qdma_set_trtcm_config(dev->qdma, channel, 2763 REG_EGRESS_TRTCM_CFG, i, 2764 !!rate, TRTCM_METER_MODE); 2765 if (err) 2766 return err; 2767 2768 err = airoha_qdma_set_trtcm_token_bucket(dev->qdma, channel, 2769 REG_EGRESS_TRTCM_CFG, 2770 i, rate, bucket_size); 2771 if (err) 2772 return err; 2773 } 2774 2775 return 0; 2776 } 2777 2778 static int airoha_tc_htb_modify_queue(struct net_device *dev, 2779 struct tc_htb_qopt_offload *opt) 2780 { 2781 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 2782 u32 rate = div_u64(opt->rate, 1000) << 3; /* kbps */ 2783 int err; 2784 2785 if (opt->parent_classid != TC_HTB_CLASSID_ROOT) { 2786 NL_SET_ERR_MSG_MOD(opt->extack, "invalid parent classid"); 2787 return -EINVAL; 2788 } 2789 2790 err = airoha_qdma_set_tx_rate_limit(dev, channel, rate, opt->quantum); 2791 if (err) 2792 NL_SET_ERR_MSG_MOD(opt->extack, 2793 "failed configuring htb offload"); 2794 2795 return err; 2796 } 2797 2798 static int airoha_tc_htb_alloc_leaf_queue(struct net_device *netdev, 2799 struct tc_htb_qopt_offload *opt) 2800 { 2801 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 2802 int err, num_tx_queues = AIROHA_NUM_TX_RING + channel + 1; 2803 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2804 struct airoha_qdma *qdma = dev->qdma; 2805 2806 /* Here we need to check the requested QDMA channel is not already 2807 * in use by another net_device running on the same QDMA block. 2808 */ 2809 if (test_and_set_bit(channel, qdma->qos_channel_map)) { 2810 NL_SET_ERR_MSG_MOD(opt->extack, 2811 "qdma qos channel already in use"); 2812 return -EBUSY; 2813 } 2814 2815 err = airoha_tc_htb_modify_queue(netdev, opt); 2816 if (err) 2817 goto error; 2818 2819 if (num_tx_queues > netdev->real_num_tx_queues) { 2820 err = netif_set_real_num_tx_queues(netdev, num_tx_queues); 2821 if (err) { 2822 airoha_qdma_set_tx_rate_limit(netdev, channel, 0, 2823 opt->quantum); 2824 NL_SET_ERR_MSG_MOD(opt->extack, 2825 "failed setting real_num_tx_queues"); 2826 goto error; 2827 } 2828 } 2829 2830 set_bit(channel, dev->qos_sq_bmap); 2831 opt->qid = AIROHA_NUM_TX_RING + channel; 2832 2833 return 0; 2834 error: 2835 clear_bit(channel, qdma->qos_channel_map); 2836 2837 return err; 2838 } 2839 2840 static int airoha_qdma_set_rx_meter(struct airoha_gdm_dev *dev, 2841 u32 rate, u32 bucket_size, 2842 enum trtcm_unit_type unit_type) 2843 { 2844 struct airoha_qdma *qdma = dev->qdma; 2845 int i; 2846 2847 for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { 2848 int err; 2849 2850 if (!qdma->q_rx[i].ndesc) 2851 continue; 2852 2853 err = airoha_qdma_init_rl_config(qdma, i, !!rate, unit_type); 2854 if (err) 2855 return err; 2856 2857 err = airoha_qdma_set_rl_token_bucket(qdma, i, rate, 2858 bucket_size); 2859 if (err) 2860 return err; 2861 } 2862 2863 return 0; 2864 } 2865 2866 static int airoha_tc_matchall_act_validate(struct tc_cls_matchall_offload *f) 2867 { 2868 const struct flow_action *actions = &f->rule->action; 2869 const struct flow_action_entry *act; 2870 2871 if (!flow_action_has_entries(actions)) { 2872 NL_SET_ERR_MSG_MOD(f->common.extack, 2873 "filter run with no actions"); 2874 return -EINVAL; 2875 } 2876 2877 if (!flow_offload_has_one_action(actions)) { 2878 NL_SET_ERR_MSG_MOD(f->common.extack, 2879 "only once action per filter is supported"); 2880 return -EOPNOTSUPP; 2881 } 2882 2883 act = &actions->entries[0]; 2884 if (act->id != FLOW_ACTION_POLICE) { 2885 NL_SET_ERR_MSG_MOD(f->common.extack, "unsupported action"); 2886 return -EOPNOTSUPP; 2887 } 2888 2889 if (act->police.exceed.act_id != FLOW_ACTION_DROP) { 2890 NL_SET_ERR_MSG_MOD(f->common.extack, 2891 "invalid exceed action id"); 2892 return -EOPNOTSUPP; 2893 } 2894 2895 if (act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { 2896 NL_SET_ERR_MSG_MOD(f->common.extack, 2897 "invalid notexceed action id"); 2898 return -EOPNOTSUPP; 2899 } 2900 2901 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && 2902 !flow_action_is_last_entry(actions, act)) { 2903 NL_SET_ERR_MSG_MOD(f->common.extack, 2904 "action accept must be last"); 2905 return -EOPNOTSUPP; 2906 } 2907 2908 if (act->police.peakrate_bytes_ps || act->police.avrate || 2909 act->police.overhead || act->police.mtu) { 2910 NL_SET_ERR_MSG_MOD(f->common.extack, 2911 "peakrate/avrate/overhead/mtu unsupported"); 2912 return -EOPNOTSUPP; 2913 } 2914 2915 return 0; 2916 } 2917 2918 static int airoha_dev_tc_matchall(struct net_device *netdev, 2919 struct tc_cls_matchall_offload *f) 2920 { 2921 enum trtcm_unit_type unit_type = TRTCM_BYTE_UNIT; 2922 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2923 u32 rate = 0, bucket_size = 0; 2924 2925 switch (f->command) { 2926 case TC_CLSMATCHALL_REPLACE: { 2927 const struct flow_action_entry *act; 2928 int err; 2929 2930 err = airoha_tc_matchall_act_validate(f); 2931 if (err) 2932 return err; 2933 2934 act = &f->rule->action.entries[0]; 2935 if (act->police.rate_pkt_ps) { 2936 rate = act->police.rate_pkt_ps; 2937 bucket_size = act->police.burst_pkt; 2938 unit_type = TRTCM_PACKET_UNIT; 2939 } else { 2940 rate = div_u64(act->police.rate_bytes_ps, 1000); 2941 rate = rate << 3; /* Kbps */ 2942 bucket_size = act->police.burst; 2943 } 2944 fallthrough; 2945 } 2946 case TC_CLSMATCHALL_DESTROY: 2947 return airoha_qdma_set_rx_meter(dev, rate, bucket_size, 2948 unit_type); 2949 default: 2950 return -EOPNOTSUPP; 2951 } 2952 } 2953 2954 static int airoha_dev_setup_tc_block_cb(enum tc_setup_type type, 2955 void *type_data, void *cb_priv) 2956 { 2957 struct net_device *netdev = cb_priv; 2958 struct airoha_gdm_dev *dev = netdev_priv(netdev); 2959 struct airoha_eth *eth = dev->eth; 2960 2961 if (!tc_can_offload(netdev)) 2962 return -EOPNOTSUPP; 2963 2964 switch (type) { 2965 case TC_SETUP_CLSFLOWER: 2966 return airoha_ppe_setup_tc_block_cb(ð->ppe->dev, type_data); 2967 case TC_SETUP_CLSMATCHALL: 2968 return airoha_dev_tc_matchall(netdev, type_data); 2969 default: 2970 return -EOPNOTSUPP; 2971 } 2972 } 2973 2974 static int airoha_dev_setup_tc_block(struct net_device *dev, 2975 struct flow_block_offload *f) 2976 { 2977 flow_setup_cb_t *cb = airoha_dev_setup_tc_block_cb; 2978 static LIST_HEAD(block_cb_list); 2979 struct flow_block_cb *block_cb; 2980 2981 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 2982 return -EOPNOTSUPP; 2983 2984 f->driver_block_list = &block_cb_list; 2985 switch (f->command) { 2986 case FLOW_BLOCK_BIND: 2987 block_cb = flow_block_cb_lookup(f->block, cb, dev); 2988 if (block_cb) { 2989 flow_block_cb_incref(block_cb); 2990 return 0; 2991 } 2992 block_cb = flow_block_cb_alloc(cb, dev, dev, NULL); 2993 if (IS_ERR(block_cb)) 2994 return PTR_ERR(block_cb); 2995 2996 flow_block_cb_incref(block_cb); 2997 flow_block_cb_add(block_cb, f); 2998 list_add_tail(&block_cb->driver_list, &block_cb_list); 2999 return 0; 3000 case FLOW_BLOCK_UNBIND: 3001 block_cb = flow_block_cb_lookup(f->block, cb, dev); 3002 if (!block_cb) 3003 return -ENOENT; 3004 3005 if (!flow_block_cb_decref(block_cb)) { 3006 flow_block_cb_remove(block_cb, f); 3007 list_del(&block_cb->driver_list); 3008 } 3009 return 0; 3010 default: 3011 return -EOPNOTSUPP; 3012 } 3013 } 3014 3015 static void airoha_tc_remove_htb_queue(struct net_device *netdev, int queue) 3016 { 3017 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3018 int num_tx_queues = AIROHA_NUM_TX_RING; 3019 struct airoha_qdma *qdma = dev->qdma; 3020 3021 airoha_qdma_set_tx_rate_limit(netdev, queue, 0, 0); 3022 3023 clear_bit(queue, qdma->qos_channel_map); 3024 clear_bit(queue, dev->qos_sq_bmap); 3025 3026 if (!bitmap_empty(dev->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS)) 3027 num_tx_queues += find_last_bit(dev->qos_sq_bmap, 3028 AIROHA_NUM_QOS_CHANNELS) + 1; 3029 netif_set_real_num_tx_queues(netdev, num_tx_queues); 3030 } 3031 3032 static int airoha_tc_htb_delete_leaf_queue(struct net_device *netdev, 3033 struct tc_htb_qopt_offload *opt) 3034 { 3035 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 3036 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3037 3038 if (!test_bit(channel, dev->qos_sq_bmap)) { 3039 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); 3040 return -EINVAL; 3041 } 3042 3043 airoha_tc_remove_htb_queue(netdev, channel); 3044 3045 return 0; 3046 } 3047 3048 static int airoha_tc_htb_destroy(struct net_device *netdev) 3049 { 3050 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3051 int q; 3052 3053 for_each_set_bit(q, dev->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS) 3054 airoha_tc_remove_htb_queue(netdev, q); 3055 3056 return 0; 3057 } 3058 3059 static int airoha_tc_get_htb_get_leaf_queue(struct net_device *netdev, 3060 struct tc_htb_qopt_offload *opt) 3061 { 3062 u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; 3063 struct airoha_gdm_dev *dev = netdev_priv(netdev); 3064 3065 if (!test_bit(channel, dev->qos_sq_bmap)) { 3066 NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); 3067 return -EINVAL; 3068 } 3069 3070 opt->qid = AIROHA_NUM_TX_RING + channel; 3071 3072 return 0; 3073 } 3074 3075 static int airoha_tc_setup_qdisc_htb(struct net_device *dev, 3076 struct tc_htb_qopt_offload *opt) 3077 { 3078 switch (opt->command) { 3079 case TC_HTB_CREATE: 3080 break; 3081 case TC_HTB_DESTROY: 3082 return airoha_tc_htb_destroy(dev); 3083 case TC_HTB_NODE_MODIFY: 3084 return airoha_tc_htb_modify_queue(dev, opt); 3085 case TC_HTB_LEAF_ALLOC_QUEUE: 3086 return airoha_tc_htb_alloc_leaf_queue(dev, opt); 3087 case TC_HTB_LEAF_DEL: 3088 case TC_HTB_LEAF_DEL_LAST: 3089 case TC_HTB_LEAF_DEL_LAST_FORCE: 3090 return airoha_tc_htb_delete_leaf_queue(dev, opt); 3091 case TC_HTB_LEAF_QUERY_QUEUE: 3092 return airoha_tc_get_htb_get_leaf_queue(dev, opt); 3093 default: 3094 return -EOPNOTSUPP; 3095 } 3096 3097 return 0; 3098 } 3099 3100 static int airoha_dev_tc_setup(struct net_device *dev, 3101 enum tc_setup_type type, void *type_data) 3102 { 3103 switch (type) { 3104 case TC_SETUP_QDISC_ETS: 3105 return airoha_tc_setup_qdisc_ets(dev, type_data); 3106 case TC_SETUP_QDISC_HTB: 3107 return airoha_tc_setup_qdisc_htb(dev, type_data); 3108 case TC_SETUP_BLOCK: 3109 case TC_SETUP_FT: 3110 return airoha_dev_setup_tc_block(dev, type_data); 3111 default: 3112 return -EOPNOTSUPP; 3113 } 3114 } 3115 3116 static const struct net_device_ops airoha_netdev_ops = { 3117 .ndo_init = airoha_dev_init, 3118 .ndo_open = airoha_dev_open, 3119 .ndo_stop = airoha_dev_stop, 3120 .ndo_change_mtu = airoha_dev_change_mtu, 3121 .ndo_select_queue = airoha_dev_select_queue, 3122 .ndo_start_xmit = airoha_dev_xmit, 3123 .ndo_get_stats64 = airoha_dev_get_stats64, 3124 .ndo_set_mac_address = airoha_dev_set_macaddr, 3125 .ndo_setup_tc = airoha_dev_tc_setup, 3126 }; 3127 3128 static const struct ethtool_ops airoha_ethtool_ops = { 3129 .get_drvinfo = airoha_ethtool_get_drvinfo, 3130 .get_eth_mac_stats = airoha_ethtool_get_mac_stats, 3131 .get_rmon_stats = airoha_ethtool_get_rmon_stats, 3132 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3133 .get_link = ethtool_op_get_link, 3134 }; 3135 3136 static int airoha_metadata_dst_alloc(struct airoha_gdm_port *port) 3137 { 3138 int i; 3139 3140 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) { 3141 struct metadata_dst *md_dst; 3142 3143 md_dst = metadata_dst_alloc(0, METADATA_HW_PORT_MUX, 3144 GFP_KERNEL); 3145 if (!md_dst) 3146 return -ENOMEM; 3147 3148 md_dst->u.port_info.port_id = i; 3149 port->dsa_meta[i] = md_dst; 3150 } 3151 3152 return 0; 3153 } 3154 3155 static void airoha_metadata_dst_free(struct airoha_gdm_port *port) 3156 { 3157 int i; 3158 3159 for (i = 0; i < ARRAY_SIZE(port->dsa_meta); i++) { 3160 if (!port->dsa_meta[i]) 3161 continue; 3162 3163 dst_release(&port->dsa_meta[i]->dst); 3164 } 3165 } 3166 3167 bool airoha_is_valid_gdm_dev(struct airoha_eth *eth, 3168 struct airoha_gdm_dev *dev) 3169 { 3170 int i; 3171 3172 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3173 struct airoha_gdm_port *port = eth->ports[i]; 3174 int j; 3175 3176 if (!port) 3177 continue; 3178 3179 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3180 if (port->devs[j] == dev) 3181 return true; 3182 } 3183 } 3184 3185 return false; 3186 } 3187 3188 static int airoha_alloc_gdm_device(struct airoha_eth *eth, 3189 struct airoha_gdm_port *port, 3190 int nbq, struct device_node *np) 3191 { 3192 struct net_device *netdev; 3193 struct airoha_gdm_dev *dev; 3194 u8 index; 3195 int err; 3196 3197 netdev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*dev), 3198 AIROHA_NUM_NETDEV_TX_RINGS, 3199 AIROHA_NUM_RX_RING); 3200 if (!netdev) { 3201 dev_err(eth->dev, "alloc_etherdev failed\n"); 3202 return -ENOMEM; 3203 } 3204 3205 netdev->netdev_ops = &airoha_netdev_ops; 3206 netdev->ethtool_ops = &airoha_ethtool_ops; 3207 netdev->max_mtu = AIROHA_MAX_MTU; 3208 netdev->watchdog_timeo = 5 * HZ; 3209 netdev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_TSO6 | 3210 NETIF_F_IPV6_CSUM | NETIF_F_SG | NETIF_F_TSO | 3211 NETIF_F_HW_TC; 3212 netdev->features |= netdev->hw_features; 3213 netdev->vlan_features = netdev->hw_features; 3214 SET_NETDEV_DEV(netdev, eth->dev); 3215 3216 /* reserve hw queues for HTB offloading */ 3217 err = netif_set_real_num_tx_queues(netdev, AIROHA_NUM_TX_RING); 3218 if (err) 3219 return err; 3220 3221 err = of_get_ethdev_address(np, netdev); 3222 if (err) { 3223 if (err == -EPROBE_DEFER) 3224 return err; 3225 3226 eth_hw_addr_random(netdev); 3227 dev_info(eth->dev, "generated random MAC address %pM\n", 3228 netdev->dev_addr); 3229 } 3230 3231 /* Allowed nbq for EN7581 on GDM3 port are 4 and 5 for PCIE0 3232 * and PCIE1 respectively. 3233 */ 3234 index = nbq; 3235 if (index && airoha_is_7581(eth) && port->id == AIROHA_GDM3_IDX) 3236 index -= 4; 3237 3238 if (index >= ARRAY_SIZE(port->devs) || port->devs[index]) { 3239 dev_err(eth->dev, "invalid nbq id: %d\n", nbq); 3240 return -EINVAL; 3241 } 3242 3243 netdev->dev.of_node = of_node_get(np); 3244 dev = netdev_priv(netdev); 3245 u64_stats_init(&dev->stats.syncp); 3246 dev->port = port; 3247 dev->eth = eth; 3248 dev->nbq = nbq; 3249 port->devs[index] = dev; 3250 3251 return 0; 3252 } 3253 3254 static int airoha_alloc_gdm_port(struct airoha_eth *eth, 3255 struct device_node *np) 3256 { 3257 const __be32 *id_ptr = of_get_property(np, "reg", NULL); 3258 struct airoha_gdm_port *port; 3259 struct device_node *node; 3260 int err, nbq, p, d = 0; 3261 u32 id; 3262 3263 if (!id_ptr) { 3264 dev_err(eth->dev, "missing gdm port id\n"); 3265 return -EINVAL; 3266 } 3267 3268 id = be32_to_cpup(id_ptr); 3269 p = id - 1; 3270 3271 if (!id || id > ARRAY_SIZE(eth->ports)) { 3272 dev_err(eth->dev, "invalid gdm port id: %d\n", id); 3273 return -EINVAL; 3274 } 3275 3276 if (eth->ports[p]) { 3277 dev_err(eth->dev, "duplicate gdm port id: %d\n", id); 3278 return -EINVAL; 3279 } 3280 3281 port = devm_kzalloc(eth->dev, sizeof(*port), GFP_KERNEL); 3282 if (!port) 3283 return -ENOMEM; 3284 3285 port->id = id; 3286 spin_lock_init(&port->stats_lock); 3287 eth->ports[p] = port; 3288 3289 err = airoha_metadata_dst_alloc(port); 3290 if (err) 3291 return err; 3292 3293 /* Default nbq value to ensure backward compatibility */ 3294 nbq = id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0; 3295 3296 for_each_child_of_node(np, node) { 3297 /* Multiple external serdes connected to the FE GDM port via an 3298 * external arbiter. 3299 */ 3300 const __be32 *nbq_ptr; 3301 3302 if (!of_device_is_compatible(node, "airoha,eth-port")) 3303 continue; 3304 3305 d++; 3306 if (!of_device_is_available(node)) 3307 continue; 3308 3309 nbq_ptr = of_get_property(node, "reg", NULL); 3310 if (!nbq_ptr) { 3311 dev_err(eth->dev, "missing nbq id\n"); 3312 of_node_put(node); 3313 return -EINVAL; 3314 } 3315 3316 /* Verify the provided nbq parameter is valid */ 3317 nbq = be32_to_cpup(nbq_ptr); 3318 err = eth->soc->ops.get_sport(port, nbq); 3319 if (err < 0) { 3320 of_node_put(node); 3321 return err; 3322 } 3323 3324 err = airoha_alloc_gdm_device(eth, port, nbq, node); 3325 if (err) { 3326 of_node_put(node); 3327 return err; 3328 } 3329 } 3330 3331 return !d ? airoha_alloc_gdm_device(eth, port, nbq, np) : 0; 3332 } 3333 3334 static int airoha_register_gdm_devices(struct airoha_eth *eth) 3335 { 3336 int i; 3337 3338 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3339 struct airoha_gdm_port *port = eth->ports[i]; 3340 int j; 3341 3342 if (!port) 3343 continue; 3344 3345 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3346 struct airoha_gdm_dev *dev = port->devs[j]; 3347 int err; 3348 3349 if (!dev) 3350 continue; 3351 3352 err = register_netdev(netdev_from_priv(dev)); 3353 if (err) 3354 return err; 3355 } 3356 } 3357 3358 set_bit(DEV_STATE_REGISTERED, ð->state); 3359 3360 return 0; 3361 } 3362 3363 static int airoha_probe(struct platform_device *pdev) 3364 { 3365 struct reset_control_bulk_data *xsi_rsts; 3366 struct device_node *np; 3367 struct airoha_eth *eth; 3368 int i, err; 3369 3370 eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL); 3371 if (!eth) 3372 return -ENOMEM; 3373 3374 eth->soc = of_device_get_match_data(&pdev->dev); 3375 if (!eth->soc) 3376 return -EINVAL; 3377 3378 eth->dev = &pdev->dev; 3379 3380 err = dma_set_mask_and_coherent(eth->dev, DMA_BIT_MASK(32)); 3381 if (err) { 3382 dev_err(eth->dev, "failed configuring DMA mask\n"); 3383 return err; 3384 } 3385 3386 eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe"); 3387 if (IS_ERR(eth->fe_regs)) 3388 return dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs), 3389 "failed to iomap fe regs\n"); 3390 3391 eth->rsts[0].id = "fe"; 3392 eth->rsts[1].id = "pdma"; 3393 eth->rsts[2].id = "qdma"; 3394 err = devm_reset_control_bulk_get_exclusive(eth->dev, 3395 ARRAY_SIZE(eth->rsts), 3396 eth->rsts); 3397 if (err) { 3398 dev_err(eth->dev, "failed to get bulk reset lines\n"); 3399 return err; 3400 } 3401 3402 xsi_rsts = devm_kcalloc(eth->dev, 3403 eth->soc->num_xsi_rsts, sizeof(*xsi_rsts), 3404 GFP_KERNEL); 3405 if (!xsi_rsts) 3406 return -ENOMEM; 3407 3408 eth->xsi_rsts = xsi_rsts; 3409 for (i = 0; i < eth->soc->num_xsi_rsts; i++) 3410 eth->xsi_rsts[i].id = eth->soc->xsi_rsts_names[i]; 3411 3412 err = devm_reset_control_bulk_get_exclusive(eth->dev, 3413 eth->soc->num_xsi_rsts, 3414 eth->xsi_rsts); 3415 if (err) { 3416 dev_err(eth->dev, "failed to get bulk xsi reset lines\n"); 3417 return err; 3418 } 3419 3420 eth->napi_dev = alloc_netdev_dummy(0); 3421 if (!eth->napi_dev) 3422 return -ENOMEM; 3423 3424 /* Enable threaded NAPI by default */ 3425 eth->napi_dev->threaded = true; 3426 strscpy(eth->napi_dev->name, "qdma_eth", sizeof(eth->napi_dev->name)); 3427 platform_set_drvdata(pdev, eth); 3428 3429 err = airoha_hw_init(pdev, eth); 3430 if (err) 3431 goto error_netdev_free; 3432 3433 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 3434 airoha_qdma_start_napi(ð->qdma[i]); 3435 airoha_qdma_set(ð->qdma[i], REG_QDMA_GLOBAL_CFG, 3436 GLOBAL_CFG_TX_DMA_EN_MASK | 3437 GLOBAL_CFG_RX_DMA_EN_MASK); 3438 } 3439 3440 for_each_child_of_node(pdev->dev.of_node, np) { 3441 if (!of_device_is_compatible(np, "airoha,eth-mac")) 3442 continue; 3443 3444 if (!of_device_is_available(np)) 3445 continue; 3446 3447 err = airoha_alloc_gdm_port(eth, np); 3448 if (err) { 3449 of_node_put(np); 3450 goto error_napi_stop; 3451 } 3452 } 3453 3454 err = airoha_register_gdm_devices(eth); 3455 if (err) 3456 goto error_napi_stop; 3457 3458 return 0; 3459 3460 error_napi_stop: 3461 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 3462 airoha_qdma_stop_napi(ð->qdma[i]); 3463 airoha_qdma_tx_cleanup(ð->qdma[i]); 3464 } 3465 3466 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3467 struct airoha_gdm_port *port = eth->ports[i]; 3468 int j; 3469 3470 if (!port) 3471 continue; 3472 3473 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3474 struct airoha_gdm_dev *dev = port->devs[j]; 3475 struct net_device *netdev; 3476 3477 if (!dev) 3478 continue; 3479 3480 netdev = netdev_from_priv(dev); 3481 if (netdev->reg_state == NETREG_REGISTERED) 3482 unregister_netdev(netdev); 3483 of_node_put(netdev->dev.of_node); 3484 } 3485 airoha_metadata_dst_free(port); 3486 } 3487 airoha_hw_cleanup(eth); 3488 error_netdev_free: 3489 free_netdev(eth->napi_dev); 3490 platform_set_drvdata(pdev, NULL); 3491 3492 return err; 3493 } 3494 3495 static void airoha_remove(struct platform_device *pdev) 3496 { 3497 struct airoha_eth *eth = platform_get_drvdata(pdev); 3498 int i; 3499 3500 for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { 3501 airoha_qdma_stop_napi(ð->qdma[i]); 3502 airoha_qdma_tx_cleanup(ð->qdma[i]); 3503 } 3504 3505 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 3506 struct airoha_gdm_port *port = eth->ports[i]; 3507 int j; 3508 3509 if (!port) 3510 continue; 3511 3512 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 3513 struct airoha_gdm_dev *dev = port->devs[j]; 3514 struct net_device *netdev; 3515 3516 if (!dev) 3517 continue; 3518 3519 netdev = netdev_from_priv(dev); 3520 unregister_netdev(netdev); 3521 of_node_put(netdev->dev.of_node); 3522 } 3523 airoha_metadata_dst_free(port); 3524 } 3525 airoha_hw_cleanup(eth); 3526 3527 free_netdev(eth->napi_dev); 3528 platform_set_drvdata(pdev, NULL); 3529 } 3530 3531 static const char * const en7581_xsi_rsts_names[] = { 3532 "xsi-mac", 3533 "hsi0-mac", 3534 "hsi1-mac", 3535 "hsi-mac", 3536 "xfp-mac", 3537 }; 3538 3539 static int airoha_en7581_get_sport(struct airoha_gdm_port *port, int nbq) 3540 { 3541 switch (port->id) { 3542 case AIROHA_GDM3_IDX: 3543 /* 7581 SoC supports PCIe serdes on GDM3 port */ 3544 if (nbq == 4) 3545 return HSGMII_LAN_7581_PCIE0_SRCPORT; 3546 if (nbq == 5) 3547 return HSGMII_LAN_7581_PCIE1_SRCPORT; 3548 break; 3549 case AIROHA_GDM4_IDX: 3550 /* 7581 SoC supports eth and usb serdes on GDM4 port */ 3551 if (!nbq) 3552 return HSGMII_LAN_7581_ETH_SRCPORT; 3553 if (nbq == 1) 3554 return HSGMII_LAN_7581_USB_SRCPORT; 3555 break; 3556 default: 3557 break; 3558 } 3559 3560 return -EINVAL; 3561 } 3562 3563 static u32 airoha_en7581_get_vip_port(struct airoha_gdm_port *port, int nbq) 3564 { 3565 switch (port->id) { 3566 case AIROHA_GDM3_IDX: 3567 if (nbq == 4) 3568 return XSI_PCIE0_VIP_PORT_MASK; 3569 if (nbq == 5) 3570 return XSI_PCIE1_VIP_PORT_MASK; 3571 break; 3572 case AIROHA_GDM4_IDX: 3573 if (!nbq) 3574 return XSI_ETH_VIP_PORT_MASK; 3575 if (nbq == 1) 3576 return XSI_USB_VIP_PORT_MASK; 3577 break; 3578 default: 3579 break; 3580 } 3581 3582 return 0; 3583 } 3584 3585 static int airoha_en7581_get_dev_from_sport(struct airoha_qdma_desc *desc, 3586 u16 *port, u16 *dev) 3587 { 3588 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, 3589 le32_to_cpu(READ_ONCE(desc->msg1))); 3590 3591 *dev = 0; 3592 switch (sport) { 3593 case 0x10 ... 0x14: 3594 *port = 0; /* GDM1 */ 3595 break; 3596 case 0x2 ... 0x4: 3597 *port = sport - 1; 3598 break; 3599 case HSGMII_LAN_7581_PCIE1_SRCPORT: 3600 *dev = 1; 3601 fallthrough; 3602 case HSGMII_LAN_7581_PCIE0_SRCPORT: 3603 *port = 2; /* GDM3 */ 3604 break; 3605 case HSGMII_LAN_7581_USB_SRCPORT: 3606 *dev = 1; 3607 fallthrough; 3608 case HSGMII_LAN_7581_ETH_SRCPORT: 3609 *port = 3; /* GDM4 */ 3610 break; 3611 default: 3612 return -EINVAL; 3613 } 3614 3615 return 0; 3616 } 3617 3618 static const char * const an7583_xsi_rsts_names[] = { 3619 "xsi-mac", 3620 "hsi0-mac", 3621 "hsi1-mac", 3622 "xfp-mac", 3623 }; 3624 3625 static int airoha_an7583_get_sport(struct airoha_gdm_port *port, int nbq) 3626 { 3627 switch (port->id) { 3628 case AIROHA_GDM3_IDX: 3629 /* 7583 SoC supports eth serdes on GDM3 port */ 3630 if (!nbq) 3631 return HSGMII_LAN_7583_ETH_SRCPORT; 3632 break; 3633 case AIROHA_GDM4_IDX: 3634 /* 7583 SoC supports PCIe and USB serdes on GDM4 port */ 3635 if (!nbq) 3636 return HSGMII_LAN_7583_PCIE_SRCPORT; 3637 if (nbq == 1) 3638 return HSGMII_LAN_7583_USB_SRCPORT; 3639 break; 3640 default: 3641 break; 3642 } 3643 3644 return -EINVAL; 3645 } 3646 3647 static u32 airoha_an7583_get_vip_port(struct airoha_gdm_port *port, int nbq) 3648 { 3649 switch (port->id) { 3650 case AIROHA_GDM3_IDX: 3651 if (!nbq) 3652 return XSI_ETH_VIP_PORT_MASK; 3653 break; 3654 case AIROHA_GDM4_IDX: 3655 if (!nbq) 3656 return XSI_PCIE0_VIP_PORT_MASK; 3657 if (nbq == 1) 3658 return XSI_USB_VIP_PORT_MASK; 3659 break; 3660 default: 3661 break; 3662 } 3663 3664 return 0; 3665 } 3666 3667 static int airoha_an7583_get_dev_from_sport(struct airoha_qdma_desc *desc, 3668 u16 *port, u16 *dev) 3669 { 3670 u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, 3671 le32_to_cpu(READ_ONCE(desc->msg1))); 3672 3673 *dev = 0; 3674 switch (sport) { 3675 case 0x10 ... 0x14: 3676 *port = 0; /* GDM1 */ 3677 break; 3678 case 0x2 ... 0x4: 3679 *port = sport - 1; 3680 break; 3681 case HSGMII_LAN_7583_ETH_SRCPORT: 3682 *port = 2; /* GDM3 */ 3683 break; 3684 case HSGMII_LAN_7583_USB_SRCPORT: 3685 *dev = 1; 3686 fallthrough; 3687 case HSGMII_LAN_7583_PCIE_SRCPORT: 3688 *port = 3; /* GDM4 */ 3689 break; 3690 default: 3691 return -EINVAL; 3692 } 3693 3694 return 0; 3695 } 3696 3697 static const struct airoha_eth_soc_data en7581_soc_data = { 3698 .version = 0x7581, 3699 .xsi_rsts_names = en7581_xsi_rsts_names, 3700 .num_xsi_rsts = ARRAY_SIZE(en7581_xsi_rsts_names), 3701 .num_ppe = 2, 3702 .ops = { 3703 .get_sport = airoha_en7581_get_sport, 3704 .get_vip_port = airoha_en7581_get_vip_port, 3705 .get_dev_from_sport = airoha_en7581_get_dev_from_sport, 3706 }, 3707 }; 3708 3709 static const struct airoha_eth_soc_data an7583_soc_data = { 3710 .version = 0x7583, 3711 .xsi_rsts_names = an7583_xsi_rsts_names, 3712 .num_xsi_rsts = ARRAY_SIZE(an7583_xsi_rsts_names), 3713 .num_ppe = 1, 3714 .ops = { 3715 .get_sport = airoha_an7583_get_sport, 3716 .get_vip_port = airoha_an7583_get_vip_port, 3717 .get_dev_from_sport = airoha_an7583_get_dev_from_sport, 3718 }, 3719 }; 3720 3721 static const struct of_device_id of_airoha_match[] = { 3722 { .compatible = "airoha,en7581-eth", .data = &en7581_soc_data }, 3723 { .compatible = "airoha,an7583-eth", .data = &an7583_soc_data }, 3724 { /* sentinel */ } 3725 }; 3726 MODULE_DEVICE_TABLE(of, of_airoha_match); 3727 3728 static struct platform_driver airoha_driver = { 3729 .probe = airoha_probe, 3730 .remove = airoha_remove, 3731 .driver = { 3732 .name = KBUILD_MODNAME, 3733 .of_match_table = of_airoha_match, 3734 }, 3735 }; 3736 module_platform_driver(airoha_driver); 3737 3738 MODULE_LICENSE("GPL v2"); 3739 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>"); 3740 MODULE_DESCRIPTION("Ethernet driver for Airoha SoC"); 3741