1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 7 #include <linux/ip.h> 8 #include <linux/ipv6.h> 9 #include <linux/of_platform.h> 10 #include <linux/platform_device.h> 11 #include <linux/rhashtable.h> 12 #include <net/ipv6.h> 13 #include <net/pkt_cls.h> 14 15 #include "airoha_regs.h" 16 #include "airoha_eth.h" 17 18 static DEFINE_MUTEX(flow_offload_mutex); 19 static DEFINE_SPINLOCK(ppe_lock); 20 21 static const struct rhashtable_params airoha_flow_table_params = { 22 .head_offset = offsetof(struct airoha_flow_table_entry, node), 23 .key_offset = offsetof(struct airoha_flow_table_entry, cookie), 24 .key_len = sizeof(unsigned long), 25 .automatic_shrinking = true, 26 }; 27 28 static const struct rhashtable_params airoha_l2_flow_table_params = { 29 .head_offset = offsetof(struct airoha_flow_table_entry, l2_node), 30 .key_offset = offsetof(struct airoha_flow_table_entry, data.bridge), 31 .key_len = 2 * ETH_ALEN, 32 .automatic_shrinking = true, 33 }; 34 35 static int airoha_ppe_get_num_stats_entries(struct airoha_ppe *ppe) 36 { 37 if (!IS_ENABLED(CONFIG_NET_AIROHA_FLOW_STATS)) 38 return -EOPNOTSUPP; 39 40 if (airoha_is_7583(ppe->eth)) 41 return -EOPNOTSUPP; 42 43 return PPE_STATS_NUM_ENTRIES; 44 } 45 46 static int airoha_ppe_get_total_num_stats_entries(struct airoha_ppe *ppe) 47 { 48 int num_stats = airoha_ppe_get_num_stats_entries(ppe); 49 50 if (num_stats > 0) { 51 struct airoha_eth *eth = ppe->eth; 52 53 num_stats = num_stats * eth->soc->num_ppe; 54 } 55 56 return num_stats; 57 } 58 59 static u32 airoha_ppe_get_total_sram_num_entries(struct airoha_ppe *ppe) 60 { 61 struct airoha_eth *eth = ppe->eth; 62 63 return PPE_SRAM_NUM_ENTRIES * eth->soc->num_ppe; 64 } 65 66 u32 airoha_ppe_get_total_num_entries(struct airoha_ppe *ppe) 67 { 68 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 69 70 return sram_num_entries + PPE_DRAM_NUM_ENTRIES; 71 } 72 73 bool airoha_ppe_is_enabled(struct airoha_eth *eth, int index) 74 { 75 if (index >= eth->soc->num_ppe) 76 return false; 77 78 return airoha_fe_rr(eth, REG_PPE_GLO_CFG(index)) & PPE_GLO_CFG_EN_MASK; 79 } 80 81 static u32 airoha_ppe_get_timestamp(struct airoha_ppe *ppe) 82 { 83 return airoha_fe_get(ppe->eth, REG_FE_FOE_TS, 84 AIROHA_FOE_IB1_BIND_TIMESTAMP); 85 } 86 87 void airoha_ppe_set_cpu_port(struct airoha_gdm_dev *dev, u8 ppe_id, u8 fport) 88 { 89 struct airoha_qdma *qdma = dev->qdma; 90 struct airoha_eth *eth = qdma->eth; 91 u8 qdma_id = qdma - ð->qdma[0]; 92 u32 fe_cpu_port; 93 94 fe_cpu_port = qdma_id ? FE_PSE_PORT_CDM2 : FE_PSE_PORT_CDM1; 95 airoha_fe_rmw(eth, REG_PPE_DFT_CPORT(ppe_id, fport), 96 DFT_CPORT_MASK(fport), 97 __field_prep(DFT_CPORT_MASK(fport), fe_cpu_port)); 98 } 99 100 static void airoha_ppe_hw_init(struct airoha_ppe *ppe) 101 { 102 u32 sram_ppe_num_data_entries = PPE_SRAM_NUM_ENTRIES, sram_num_entries; 103 u32 sram_tb_size, dram_num_entries; 104 struct airoha_eth *eth = ppe->eth; 105 int i, sram_num_stats_entries; 106 107 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 108 sram_tb_size = sram_num_entries * sizeof(struct airoha_foe_entry); 109 dram_num_entries = PPE_RAM_NUM_ENTRIES_SHIFT(PPE_DRAM_NUM_ENTRIES); 110 111 sram_num_stats_entries = airoha_ppe_get_num_stats_entries(ppe); 112 if (sram_num_stats_entries > 0) 113 sram_ppe_num_data_entries -= sram_num_stats_entries; 114 sram_ppe_num_data_entries = 115 PPE_RAM_NUM_ENTRIES_SHIFT(sram_ppe_num_data_entries); 116 117 for (i = 0; i < eth->soc->num_ppe; i++) { 118 int p; 119 120 airoha_fe_wr(eth, REG_PPE_TB_BASE(i), 121 ppe->foe_dma + sram_tb_size); 122 123 airoha_fe_rmw(eth, REG_PPE_BND_AGE0(i), 124 PPE_BIND_AGE0_DELTA_NON_L4 | 125 PPE_BIND_AGE0_DELTA_UDP, 126 FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 60) | 127 FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 60)); 128 airoha_fe_rmw(eth, REG_PPE_BND_AGE1(i), 129 PPE_BIND_AGE1_DELTA_TCP_FIN | 130 PPE_BIND_AGE1_DELTA_TCP, 131 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 132 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 60)); 133 134 airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i), 135 PPE_SRAM_TABLE_EN_MASK | 136 PPE_SRAM_HASH1_EN_MASK | 137 PPE_DRAM_TABLE_EN_MASK | 138 PPE_SRAM_HASH0_MODE_MASK | 139 PPE_SRAM_HASH1_MODE_MASK | 140 PPE_DRAM_HASH0_MODE_MASK | 141 PPE_DRAM_HASH1_MODE_MASK, 142 FIELD_PREP(PPE_SRAM_TABLE_EN_MASK, 1) | 143 FIELD_PREP(PPE_SRAM_HASH1_EN_MASK, 1) | 144 FIELD_PREP(PPE_SRAM_HASH1_MODE_MASK, 1) | 145 FIELD_PREP(PPE_DRAM_HASH1_MODE_MASK, 3)); 146 147 airoha_fe_rmw(eth, REG_PPE_TB_CFG(i), 148 PPE_TB_CFG_SEARCH_MISS_MASK | 149 PPE_SRAM_TB_NUM_ENTRY_MASK | 150 PPE_DRAM_TB_NUM_ENTRY_MASK | 151 PPE_TB_CFG_KEEPALIVE_MASK | 152 PPE_TB_ENTRY_SIZE_MASK, 153 FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) | 154 FIELD_PREP(PPE_TB_ENTRY_SIZE_MASK, 0) | 155 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 156 sram_ppe_num_data_entries) | 157 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 158 dram_num_entries)); 159 160 airoha_fe_rmw(eth, REG_PPE_BIND_RATE(i), 161 PPE_BIND_RATE_L2B_BIND_MASK | 162 PPE_BIND_RATE_BIND_MASK, 163 FIELD_PREP(PPE_BIND_RATE_L2B_BIND_MASK, 0x1e) | 164 FIELD_PREP(PPE_BIND_RATE_BIND_MASK, 0x1e)); 165 166 airoha_fe_wr(eth, REG_PPE_HASH_SEED(i), PPE_HASH_SEED); 167 airoha_fe_clear(eth, REG_PPE_PPE_FLOW_CFG(i), 168 PPE_FLOW_CFG_IP6_6RD_MASK); 169 170 for (p = 0; p < ARRAY_SIZE(eth->ports); p++) 171 airoha_fe_rmw(eth, REG_PPE_MTU(i, p), 172 FP0_EGRESS_MTU_MASK | 173 FP1_EGRESS_MTU_MASK, 174 FIELD_PREP(FP0_EGRESS_MTU_MASK, 175 AIROHA_MAX_MTU) | 176 FIELD_PREP(FP1_EGRESS_MTU_MASK, 177 AIROHA_MAX_MTU)); 178 } 179 180 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 181 struct airoha_gdm_port *port = eth->ports[i]; 182 int j; 183 184 if (!port) 185 continue; 186 187 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 188 struct airoha_gdm_dev *dev = port->devs[j]; 189 int ppe_id; 190 u8 fport; 191 192 if (!dev) 193 continue; 194 195 ppe_id = !airoha_is_lan_gdm_dev(dev) && 196 airoha_ppe_is_enabled(eth, 1); 197 fport = airoha_get_fe_port(dev); 198 airoha_ppe_set_cpu_port(dev, ppe_id, fport); 199 } 200 } 201 } 202 203 static void airoha_ppe_flow_mangle_eth(const struct flow_action_entry *act, void *eth) 204 { 205 void *dest = eth + act->mangle.offset; 206 const void *src = &act->mangle.val; 207 208 if (act->mangle.offset > 8) 209 return; 210 211 if (act->mangle.mask == 0xffff) { 212 src += 2; 213 dest += 2; 214 } 215 216 memcpy(dest, src, act->mangle.mask ? 2 : 4); 217 } 218 219 static int airoha_ppe_flow_mangle_ports(const struct flow_action_entry *act, 220 struct airoha_flow_data *data) 221 { 222 u32 val = be32_to_cpu((__force __be32)act->mangle.val); 223 224 switch (act->mangle.offset) { 225 case 0: 226 if ((__force __be32)act->mangle.mask == ~cpu_to_be32(0xffff)) 227 data->dst_port = cpu_to_be16(val); 228 else 229 data->src_port = cpu_to_be16(val >> 16); 230 break; 231 case 2: 232 data->dst_port = cpu_to_be16(val); 233 break; 234 default: 235 return -EINVAL; 236 } 237 238 return 0; 239 } 240 241 static int airoha_ppe_flow_mangle_ipv4(const struct flow_action_entry *act, 242 struct airoha_flow_data *data) 243 { 244 __be32 *dest; 245 246 switch (act->mangle.offset) { 247 case offsetof(struct iphdr, saddr): 248 dest = &data->v4.src_addr; 249 break; 250 case offsetof(struct iphdr, daddr): 251 dest = &data->v4.dst_addr; 252 break; 253 default: 254 return -EINVAL; 255 } 256 257 memcpy(dest, &act->mangle.val, sizeof(u32)); 258 259 return 0; 260 } 261 262 static int airoha_ppe_get_wdma_info(struct net_device *dev, const u8 *addr, 263 struct airoha_wdma_info *info) 264 { 265 struct net_device_path_stack stack; 266 struct net_device_path *path; 267 int err; 268 269 if (!dev) 270 return -ENODEV; 271 272 rcu_read_lock(); 273 err = dev_fill_forward_path(dev, addr, &stack); 274 rcu_read_unlock(); 275 if (err) 276 return err; 277 278 path = &stack.path[stack.num_paths - 1]; 279 if (path->type != DEV_PATH_MTK_WDMA) 280 return -EINVAL; 281 282 info->idx = path->mtk_wdma.wdma_idx; 283 info->bss = path->mtk_wdma.bss; 284 info->wcid = path->mtk_wdma.wcid; 285 286 return 0; 287 } 288 289 static int airoha_get_dsa_port(struct net_device **dev) 290 { 291 #if IS_ENABLED(CONFIG_NET_DSA) 292 struct dsa_port *dp = dsa_port_from_netdev(*dev); 293 294 if (IS_ERR(dp)) 295 return -ENODEV; 296 297 *dev = dsa_port_to_conduit(dp); 298 return dp->index; 299 #else 300 return -ENODEV; 301 #endif 302 } 303 304 static void airoha_ppe_foe_set_bridge_addrs(struct airoha_foe_bridge *br, 305 struct ethhdr *eh) 306 { 307 br->dest_mac_hi = get_unaligned_be32(eh->h_dest); 308 br->dest_mac_lo = get_unaligned_be16(eh->h_dest + 4); 309 br->src_mac_hi = get_unaligned_be16(eh->h_source); 310 br->src_mac_lo = get_unaligned_be32(eh->h_source + 2); 311 } 312 313 static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth, 314 struct airoha_foe_entry *hwe, 315 struct net_device *netdev, int type, 316 struct airoha_flow_data *data, 317 int l4proto) 318 { 319 u32 qdata = FIELD_PREP(AIROHA_FOE_SHAPER_ID, 0x7f), ports_pad, val; 320 int wlan_etype = -EINVAL, dsa_port = airoha_get_dsa_port(&netdev); 321 struct airoha_foe_mac_info_common *l2; 322 u8 smac_id = 0xf; 323 324 memset(hwe, 0, sizeof(*hwe)); 325 326 val = FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, AIROHA_FOE_STATE_BIND) | 327 FIELD_PREP(AIROHA_FOE_IB1_BIND_PACKET_TYPE, type) | 328 FIELD_PREP(AIROHA_FOE_IB1_BIND_UDP, l4proto == IPPROTO_UDP) | 329 FIELD_PREP(AIROHA_FOE_IB1_BIND_VLAN_LAYER, data->vlan.num) | 330 FIELD_PREP(AIROHA_FOE_IB1_BIND_VPM, data->vlan.num) | 331 FIELD_PREP(AIROHA_FOE_IB1_BIND_PPPOE, data->pppoe.num) | 332 AIROHA_FOE_IB1_BIND_TTL; 333 hwe->ib1 = val; 334 335 val = FIELD_PREP(AIROHA_FOE_IB2_PORT_AG, 0x1f); 336 if (netdev) { 337 struct airoha_wdma_info info = {}; 338 339 if (!airoha_ppe_get_wdma_info(netdev, data->eth.h_dest, 340 &info)) { 341 val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, info.idx) | 342 FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, 343 FE_PSE_PORT_CDM4); 344 qdata |= FIELD_PREP(AIROHA_FOE_ACTDP, info.bss); 345 wlan_etype = FIELD_PREP(AIROHA_FOE_MAC_WDMA_BAND, 346 info.idx) | 347 FIELD_PREP(AIROHA_FOE_MAC_WDMA_WCID, 348 info.wcid); 349 } else { 350 struct airoha_gdm_dev *dev = netdev_priv(netdev); 351 struct airoha_gdm_port *port; 352 u8 pse_port, channel; 353 354 if (!airoha_is_valid_gdm_dev(eth, dev)) 355 return -EINVAL; 356 357 port = dev->port; 358 if (dsa_port >= 0 || airoha_is_lan_gdm_dev(dev)) 359 pse_port = port->id == 4 ? FE_PSE_PORT_GDM4 360 : port->id; 361 else 362 pse_port = 2; /* uplink relies on GDM2 363 * loopback 364 */ 365 366 /* For traffic forwarded to DSA devices select QoS 367 * channel according to the DSA user port index, rely 368 * on port id otherwise. 369 */ 370 channel = dsa_port >= 0 ? dsa_port : port->id; 371 channel = channel % AIROHA_NUM_QOS_CHANNELS; 372 qdata |= FIELD_PREP(AIROHA_FOE_CHANNEL, channel); 373 374 val |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, pse_port) | 375 AIROHA_FOE_IB2_PSE_QOS; 376 /* For downlink traffic consume SRAM memory for hw 377 * forwarding descriptors queue. 378 */ 379 if (airoha_is_lan_gdm_dev(dev)) 380 val |= AIROHA_FOE_IB2_FAST_PATH; 381 if (dsa_port >= 0) 382 val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, 383 dsa_port); 384 385 smac_id = port->id; 386 } 387 } 388 389 if (is_multicast_ether_addr(data->eth.h_dest)) 390 val |= AIROHA_FOE_IB2_MULTICAST; 391 392 ports_pad = 0xa5a5a500 | (l4proto & 0xff); 393 if (type == PPE_PKT_TYPE_IPV4_ROUTE) 394 hwe->ipv4.orig_tuple.ports = ports_pad; 395 if (type == PPE_PKT_TYPE_IPV6_ROUTE_3T) 396 hwe->ipv6.ports = ports_pad; 397 398 if (type == PPE_PKT_TYPE_BRIDGE) { 399 airoha_ppe_foe_set_bridge_addrs(&hwe->bridge, &data->eth); 400 hwe->bridge.data = qdata; 401 hwe->bridge.ib2 = val; 402 l2 = &hwe->bridge.l2.common; 403 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 404 hwe->ipv6.data = qdata; 405 hwe->ipv6.ib2 = val; 406 l2 = &hwe->ipv6.l2; 407 l2->etype = ETH_P_IPV6; 408 } else { 409 hwe->ipv4.data = qdata; 410 hwe->ipv4.ib2 = val; 411 l2 = &hwe->ipv4.l2.common; 412 l2->etype = ETH_P_IP; 413 } 414 415 l2->dest_mac_hi = get_unaligned_be32(data->eth.h_dest); 416 l2->dest_mac_lo = get_unaligned_be16(data->eth.h_dest + 4); 417 if (type <= PPE_PKT_TYPE_IPV4_DSLITE) { 418 struct airoha_foe_mac_info *mac_info; 419 420 l2->src_mac_hi = get_unaligned_be32(data->eth.h_source); 421 hwe->ipv4.l2.src_mac_lo = 422 get_unaligned_be16(data->eth.h_source + 4); 423 424 mac_info = (struct airoha_foe_mac_info *)l2; 425 mac_info->pppoe_id = data->pppoe.sid; 426 } else { 427 l2->src_mac_hi = FIELD_PREP(AIROHA_FOE_MAC_SMAC_ID, smac_id) | 428 FIELD_PREP(AIROHA_FOE_MAC_PPPOE_ID, 429 data->pppoe.sid); 430 } 431 432 if (data->vlan.num) { 433 l2->vlan1 = data->vlan.hdr[0].id; 434 if (data->vlan.num == 2) 435 l2->vlan2 = data->vlan.hdr[1].id; 436 } 437 438 if (wlan_etype >= 0) { 439 l2->etype = wlan_etype; 440 } else if (dsa_port >= 0) { 441 l2->etype = BIT(dsa_port); 442 l2->etype |= !data->vlan.num ? BIT(15) : 0; 443 } else if (data->pppoe.num) { 444 l2->etype = ETH_P_PPP_SES; 445 } 446 447 return 0; 448 } 449 450 static int airoha_ppe_foe_entry_set_ipv4_tuple(struct airoha_foe_entry *hwe, 451 struct airoha_flow_data *data, 452 bool egress) 453 { 454 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 455 struct airoha_foe_ipv4_tuple *t; 456 457 switch (type) { 458 case PPE_PKT_TYPE_IPV4_HNAPT: 459 if (egress) { 460 t = &hwe->ipv4.new_tuple; 461 break; 462 } 463 fallthrough; 464 case PPE_PKT_TYPE_IPV4_DSLITE: 465 case PPE_PKT_TYPE_IPV4_ROUTE: 466 t = &hwe->ipv4.orig_tuple; 467 break; 468 default: 469 WARN_ON_ONCE(1); 470 return -EINVAL; 471 } 472 473 t->src_ip = be32_to_cpu(data->v4.src_addr); 474 t->dest_ip = be32_to_cpu(data->v4.dst_addr); 475 476 if (type != PPE_PKT_TYPE_IPV4_ROUTE) { 477 t->src_port = be16_to_cpu(data->src_port); 478 t->dest_port = be16_to_cpu(data->dst_port); 479 } 480 481 return 0; 482 } 483 484 static int airoha_ppe_foe_entry_set_ipv6_tuple(struct airoha_foe_entry *hwe, 485 struct airoha_flow_data *data) 486 487 { 488 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 489 u32 *src, *dest; 490 491 switch (type) { 492 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 493 case PPE_PKT_TYPE_IPV6_6RD: 494 hwe->ipv6.src_port = be16_to_cpu(data->src_port); 495 hwe->ipv6.dest_port = be16_to_cpu(data->dst_port); 496 fallthrough; 497 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 498 src = hwe->ipv6.src_ip; 499 dest = hwe->ipv6.dest_ip; 500 break; 501 default: 502 WARN_ON_ONCE(1); 503 return -EINVAL; 504 } 505 506 ipv6_addr_be32_to_cpu(src, data->v6.src_addr.s6_addr32); 507 ipv6_addr_be32_to_cpu(dest, data->v6.dst_addr.s6_addr32); 508 509 return 0; 510 } 511 512 static u32 airoha_ppe_foe_get_entry_hash(struct airoha_ppe *ppe, 513 struct airoha_foe_entry *hwe) 514 { 515 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 516 u32 ppe_hash_mask = airoha_ppe_get_total_num_entries(ppe) - 1; 517 u32 hash, hv1, hv2, hv3; 518 519 switch (type) { 520 case PPE_PKT_TYPE_IPV4_ROUTE: 521 case PPE_PKT_TYPE_IPV4_HNAPT: 522 hv1 = hwe->ipv4.orig_tuple.ports; 523 hv2 = hwe->ipv4.orig_tuple.dest_ip; 524 hv3 = hwe->ipv4.orig_tuple.src_ip; 525 break; 526 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 527 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 528 hv1 = hwe->ipv6.src_ip[3] ^ hwe->ipv6.dest_ip[3]; 529 hv1 ^= hwe->ipv6.ports; 530 531 hv2 = hwe->ipv6.src_ip[2] ^ hwe->ipv6.dest_ip[2]; 532 hv2 ^= hwe->ipv6.dest_ip[0]; 533 534 hv3 = hwe->ipv6.src_ip[1] ^ hwe->ipv6.dest_ip[1]; 535 hv3 ^= hwe->ipv6.src_ip[0]; 536 break; 537 case PPE_PKT_TYPE_BRIDGE: { 538 struct airoha_foe_mac_info *l2 = &hwe->bridge.l2; 539 540 hv1 = l2->common.src_mac_hi & 0xffff; 541 hv1 = hv1 << 16 | l2->src_mac_lo; 542 543 hv2 = l2->common.dest_mac_lo; 544 hv2 = hv2 << 16; 545 hv2 = hv2 | ((l2->common.src_mac_hi & 0xffff0000) >> 16); 546 547 hv3 = l2->common.dest_mac_hi; 548 break; 549 } 550 case PPE_PKT_TYPE_IPV4_DSLITE: 551 case PPE_PKT_TYPE_IPV6_6RD: 552 default: 553 WARN_ON_ONCE(1); 554 return ppe_hash_mask; 555 } 556 557 hash = (hv1 & hv2) | ((~hv1) & hv3); 558 hash = (hash >> 24) | ((hash & 0xffffff) << 8); 559 hash ^= hv1 ^ hv2 ^ hv3; 560 hash ^= hash >> 16; 561 hash &= ppe_hash_mask; 562 563 return hash; 564 } 565 566 static int airoha_ppe_foe_get_flow_stats_index(struct airoha_ppe *ppe, 567 u32 hash, u32 *index) 568 { 569 int ppe_num_stats_entries; 570 571 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 572 if (ppe_num_stats_entries < 0) 573 return ppe_num_stats_entries; 574 575 *index = hash >= ppe_num_stats_entries ? hash - PPE_STATS_NUM_ENTRIES 576 : hash; 577 578 return 0; 579 } 580 581 static void airoha_ppe_foe_flow_stat_entry_reset(struct airoha_ppe *ppe, 582 struct airoha_npu *npu, 583 int index) 584 { 585 memset_io(&npu->stats[index], 0, sizeof(*npu->stats)); 586 memset(&ppe->foe_stats[index], 0, sizeof(*ppe->foe_stats)); 587 } 588 589 static void airoha_ppe_foe_flow_stats_reset(struct airoha_ppe *ppe, 590 struct airoha_npu *npu) 591 { 592 int i, ppe_num_stats_entries; 593 594 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 595 if (ppe_num_stats_entries < 0) 596 return; 597 598 for (i = 0; i < ppe_num_stats_entries; i++) 599 airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, i); 600 } 601 602 static void airoha_ppe_foe_flow_stats_update(struct airoha_ppe *ppe, 603 struct airoha_npu *npu, 604 struct airoha_foe_entry *hwe, 605 u32 hash) 606 { 607 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 608 u32 index, pse_port, val, *data, *ib2, *meter; 609 int ppe_num_stats_entries; 610 u8 nbq; 611 612 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 613 if (ppe_num_stats_entries < 0) 614 return; 615 616 if (airoha_ppe_foe_get_flow_stats_index(ppe, hash, &index)) 617 return; 618 619 if (index >= ppe_num_stats_entries) 620 return; 621 622 if (type == PPE_PKT_TYPE_BRIDGE) { 623 data = &hwe->bridge.data; 624 ib2 = &hwe->bridge.ib2; 625 meter = &hwe->bridge.l2.meter; 626 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 627 data = &hwe->ipv6.data; 628 ib2 = &hwe->ipv6.ib2; 629 meter = &hwe->ipv6.meter; 630 } else { 631 data = &hwe->ipv4.data; 632 ib2 = &hwe->ipv4.ib2; 633 meter = &hwe->ipv4.l2.meter; 634 } 635 636 pse_port = FIELD_GET(AIROHA_FOE_IB2_PSE_PORT, *ib2); 637 if (pse_port == FE_PSE_PORT_CDM4) 638 return; 639 640 airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, index); 641 642 val = FIELD_GET(AIROHA_FOE_CHANNEL | AIROHA_FOE_QID, *data); 643 *data = (*data & ~AIROHA_FOE_ACTDP) | 644 FIELD_PREP(AIROHA_FOE_ACTDP, val); 645 646 val = *ib2 & (AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT | 647 AIROHA_FOE_IB2_PSE_QOS | AIROHA_FOE_IB2_FAST_PATH); 648 *meter |= FIELD_PREP(AIROHA_FOE_TUNNEL_MTU, val); 649 650 nbq = pse_port == 1 ? 6 : 5; 651 *ib2 &= ~(AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT | 652 AIROHA_FOE_IB2_PSE_QOS); 653 *ib2 |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, 6) | 654 FIELD_PREP(AIROHA_FOE_IB2_NBQ, nbq); 655 } 656 657 static struct airoha_foe_entry * 658 airoha_ppe_foe_get_entry_locked(struct airoha_ppe *ppe, u32 hash) 659 { 660 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 661 662 lockdep_assert_held(&ppe_lock); 663 664 if (hash < sram_num_entries) { 665 u32 *hwe = ppe->foe + hash * sizeof(struct airoha_foe_entry); 666 bool ppe2 = hash >= PPE_SRAM_NUM_ENTRIES; 667 struct airoha_eth *eth = ppe->eth; 668 u32 val; 669 int i; 670 671 airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2), 672 FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) | 673 PPE_SRAM_CTRL_REQ_MASK); 674 if (read_poll_timeout_atomic(airoha_fe_rr, val, 675 val & PPE_SRAM_CTRL_ACK_MASK, 676 10, 100, false, eth, 677 REG_PPE_RAM_CTRL(ppe2))) 678 return NULL; 679 680 for (i = 0; i < sizeof(struct airoha_foe_entry) / sizeof(*hwe); 681 i++) 682 hwe[i] = airoha_fe_rr(eth, 683 REG_PPE_RAM_ENTRY(ppe2, i)); 684 } 685 686 return ppe->foe + hash * sizeof(struct airoha_foe_entry); 687 } 688 689 struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe, 690 u32 hash) 691 { 692 struct airoha_foe_entry *hwe; 693 694 spin_lock_bh(&ppe_lock); 695 hwe = airoha_ppe_foe_get_entry_locked(ppe, hash); 696 spin_unlock_bh(&ppe_lock); 697 698 return hwe; 699 } 700 701 static bool airoha_ppe_foe_compare_entry(struct airoha_flow_table_entry *e, 702 struct airoha_foe_entry *hwe) 703 { 704 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 705 int len; 706 707 if ((hwe->ib1 ^ e->data.ib1) & AIROHA_FOE_IB1_BIND_UDP) 708 return false; 709 710 if (type > PPE_PKT_TYPE_IPV4_DSLITE) 711 len = offsetof(struct airoha_foe_entry, ipv6.data); 712 else 713 len = offsetof(struct airoha_foe_entry, ipv4.ib2); 714 715 return !memcmp(&e->data.d, &hwe->d, len - sizeof(hwe->ib1)); 716 } 717 718 static int airoha_ppe_foe_commit_sram_entry(struct airoha_ppe *ppe, u32 hash) 719 { 720 struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe); 721 bool ppe2 = hash >= PPE_SRAM_NUM_ENTRIES; 722 u32 *ptr = (u32 *)hwe, val; 723 int i; 724 725 for (i = 0; i < sizeof(*hwe) / sizeof(*ptr); i++) 726 airoha_fe_wr(ppe->eth, REG_PPE_RAM_ENTRY(ppe2, i), ptr[i]); 727 728 wmb(); 729 airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2), 730 FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) | 731 PPE_SRAM_CTRL_WR_MASK | PPE_SRAM_CTRL_REQ_MASK); 732 733 return read_poll_timeout_atomic(airoha_fe_rr, val, 734 val & PPE_SRAM_CTRL_ACK_MASK, 735 10, 100, false, ppe->eth, 736 REG_PPE_RAM_CTRL(ppe2)); 737 } 738 739 static int airoha_ppe_foe_commit_entry(struct airoha_ppe *ppe, 740 struct airoha_foe_entry *e, 741 u32 hash, bool rx_wlan) 742 { 743 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 744 struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe); 745 u32 ts = airoha_ppe_get_timestamp(ppe); 746 struct airoha_eth *eth = ppe->eth; 747 struct airoha_npu *npu; 748 int err = 0; 749 750 memcpy(&hwe->d, &e->d, sizeof(*hwe) - sizeof(hwe->ib1)); 751 wmb(); 752 753 e->ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP; 754 e->ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_TIMESTAMP, ts); 755 hwe->ib1 = e->ib1; 756 757 rcu_read_lock(); 758 759 npu = rcu_dereference(eth->npu); 760 if (!npu) { 761 err = -ENODEV; 762 goto unlock; 763 } 764 765 if (!rx_wlan) 766 airoha_ppe_foe_flow_stats_update(ppe, npu, hwe, hash); 767 768 if (hash < sram_num_entries) 769 err = airoha_ppe_foe_commit_sram_entry(ppe, hash); 770 unlock: 771 rcu_read_unlock(); 772 773 return err; 774 } 775 776 static void airoha_ppe_foe_remove_flow(struct airoha_ppe *ppe, 777 struct airoha_flow_table_entry *e) 778 { 779 lockdep_assert_held(&ppe_lock); 780 781 hlist_del_init(&e->list); 782 if (e->hash != 0xffff) { 783 e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_STATE; 784 e->data.ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, 785 AIROHA_FOE_STATE_INVALID); 786 airoha_ppe_foe_commit_entry(ppe, &e->data, e->hash, false); 787 e->hash = 0xffff; 788 } 789 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 790 hlist_del_init(&e->l2_subflow_node); 791 kfree(e); 792 } 793 } 794 795 static void airoha_ppe_foe_remove_l2_flow(struct airoha_ppe *ppe, 796 struct airoha_flow_table_entry *e) 797 { 798 struct hlist_head *head = &e->l2_flows; 799 struct hlist_node *n; 800 801 lockdep_assert_held(&ppe_lock); 802 803 rhashtable_remove_fast(&ppe->l2_flows, &e->l2_node, 804 airoha_l2_flow_table_params); 805 hlist_for_each_entry_safe(e, n, head, l2_subflow_node) 806 airoha_ppe_foe_remove_flow(ppe, e); 807 } 808 809 static void airoha_ppe_foe_flow_remove_entry(struct airoha_ppe *ppe, 810 struct airoha_flow_table_entry *e) 811 { 812 spin_lock_bh(&ppe_lock); 813 814 if (e->type == FLOW_TYPE_L2) 815 airoha_ppe_foe_remove_l2_flow(ppe, e); 816 else 817 airoha_ppe_foe_remove_flow(ppe, e); 818 819 spin_unlock_bh(&ppe_lock); 820 } 821 822 static int 823 airoha_ppe_foe_commit_subflow_entry(struct airoha_ppe *ppe, 824 struct airoha_flow_table_entry *e, 825 u32 hash, bool rx_wlan) 826 { 827 u32 mask = AIROHA_FOE_IB1_BIND_PACKET_TYPE | AIROHA_FOE_IB1_BIND_UDP; 828 struct airoha_foe_entry *hwe_p, hwe; 829 struct airoha_flow_table_entry *f; 830 int type; 831 832 hwe_p = airoha_ppe_foe_get_entry_locked(ppe, hash); 833 if (!hwe_p) 834 return -EINVAL; 835 836 f = kzalloc_obj(*f, GFP_ATOMIC); 837 if (!f) 838 return -ENOMEM; 839 840 hlist_add_head(&f->l2_subflow_node, &e->l2_flows); 841 f->type = FLOW_TYPE_L2_SUBFLOW; 842 f->hash = hash; 843 844 memcpy(&hwe, hwe_p, sizeof(*hwe_p)); 845 hwe.ib1 = (hwe.ib1 & mask) | (e->data.ib1 & ~mask); 846 847 type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe.ib1); 848 if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 849 memcpy(&hwe.ipv6.l2, &e->data.bridge.l2, sizeof(hwe.ipv6.l2)); 850 hwe.ipv6.ib2 = e->data.bridge.ib2; 851 /* setting smac_id to 0xf instruct the hw to keep original 852 * source mac address 853 */ 854 hwe.ipv6.l2.src_mac_hi = FIELD_PREP(AIROHA_FOE_MAC_SMAC_ID, 855 0xf); 856 } else { 857 memcpy(&hwe.bridge.l2, &e->data.bridge.l2, 858 sizeof(hwe.bridge.l2)); 859 hwe.bridge.ib2 = e->data.bridge.ib2; 860 if (type == PPE_PKT_TYPE_IPV4_HNAPT) 861 memcpy(&hwe.ipv4.new_tuple, &hwe.ipv4.orig_tuple, 862 sizeof(hwe.ipv4.new_tuple)); 863 } 864 865 hwe.bridge.data = e->data.bridge.data; 866 airoha_ppe_foe_commit_entry(ppe, &hwe, hash, rx_wlan); 867 868 return 0; 869 } 870 871 static void airoha_ppe_foe_insert_entry(struct airoha_ppe *ppe, 872 struct sk_buff *skb, 873 u32 hash, bool rx_wlan) 874 { 875 struct airoha_flow_table_entry *e; 876 struct airoha_foe_bridge br = {}; 877 struct airoha_foe_entry *hwe; 878 bool commit_done = false; 879 struct hlist_node *n; 880 u32 index, state; 881 882 spin_lock_bh(&ppe_lock); 883 884 hwe = airoha_ppe_foe_get_entry_locked(ppe, hash); 885 if (!hwe) 886 goto unlock; 887 888 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 889 if (state == AIROHA_FOE_STATE_BIND) 890 goto unlock; 891 892 index = airoha_ppe_foe_get_entry_hash(ppe, hwe); 893 hlist_for_each_entry_safe(e, n, &ppe->foe_flow[index], list) { 894 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 895 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 896 if (state != AIROHA_FOE_STATE_BIND) { 897 e->hash = 0xffff; 898 airoha_ppe_foe_remove_flow(ppe, e); 899 } 900 continue; 901 } 902 903 if (!airoha_ppe_foe_compare_entry(e, hwe)) 904 continue; 905 906 airoha_ppe_foe_commit_entry(ppe, &e->data, hash, rx_wlan); 907 commit_done = true; 908 e->hash = hash; 909 } 910 911 if (commit_done) 912 goto unlock; 913 914 airoha_ppe_foe_set_bridge_addrs(&br, eth_hdr(skb)); 915 e = rhashtable_lookup_fast(&ppe->l2_flows, &br, 916 airoha_l2_flow_table_params); 917 if (e) 918 airoha_ppe_foe_commit_subflow_entry(ppe, e, hash, rx_wlan); 919 unlock: 920 spin_unlock_bh(&ppe_lock); 921 } 922 923 static int 924 airoha_ppe_foe_l2_flow_commit_entry(struct airoha_ppe *ppe, 925 struct airoha_flow_table_entry *e) 926 { 927 struct airoha_flow_table_entry *prev; 928 929 e->type = FLOW_TYPE_L2; 930 prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &e->l2_node, 931 airoha_l2_flow_table_params); 932 if (!prev) 933 return 0; 934 935 if (IS_ERR(prev)) 936 return PTR_ERR(prev); 937 938 return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node, 939 &e->l2_node, 940 airoha_l2_flow_table_params); 941 } 942 943 static int airoha_ppe_foe_flow_commit_entry(struct airoha_ppe *ppe, 944 struct airoha_flow_table_entry *e) 945 { 946 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 947 u32 hash; 948 949 if (type == PPE_PKT_TYPE_BRIDGE) 950 return airoha_ppe_foe_l2_flow_commit_entry(ppe, e); 951 952 hash = airoha_ppe_foe_get_entry_hash(ppe, &e->data); 953 e->type = FLOW_TYPE_L4; 954 e->hash = 0xffff; 955 956 spin_lock_bh(&ppe_lock); 957 hlist_add_head(&e->list, &ppe->foe_flow[hash]); 958 spin_unlock_bh(&ppe_lock); 959 960 return 0; 961 } 962 963 static int airoha_ppe_get_entry_idle_time(struct airoha_ppe *ppe, u32 ib1) 964 { 965 u32 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1); 966 u32 ts, ts_mask, now = airoha_ppe_get_timestamp(ppe); 967 int idle; 968 969 if (state == AIROHA_FOE_STATE_BIND) { 970 ts = FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, ib1); 971 ts_mask = AIROHA_FOE_IB1_BIND_TIMESTAMP; 972 } else { 973 ts = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, ib1); 974 now = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, now); 975 ts_mask = AIROHA_FOE_IB1_UNBIND_TIMESTAMP; 976 } 977 idle = now - ts; 978 979 return idle < 0 ? idle + ts_mask + 1 : idle; 980 } 981 982 static void 983 airoha_ppe_foe_flow_l2_entry_update(struct airoha_ppe *ppe, 984 struct airoha_flow_table_entry *e) 985 { 986 int min_idle = airoha_ppe_get_entry_idle_time(ppe, e->data.ib1); 987 struct airoha_flow_table_entry *iter; 988 struct hlist_node *n; 989 990 lockdep_assert_held(&ppe_lock); 991 992 hlist_for_each_entry_safe(iter, n, &e->l2_flows, l2_subflow_node) { 993 struct airoha_foe_entry *hwe; 994 u32 ib1, state; 995 int idle; 996 997 hwe = airoha_ppe_foe_get_entry_locked(ppe, iter->hash); 998 if (!hwe) 999 continue; 1000 1001 ib1 = READ_ONCE(hwe->ib1); 1002 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1); 1003 if (state != AIROHA_FOE_STATE_BIND) { 1004 iter->hash = 0xffff; 1005 airoha_ppe_foe_remove_flow(ppe, iter); 1006 continue; 1007 } 1008 1009 idle = airoha_ppe_get_entry_idle_time(ppe, ib1); 1010 if (idle >= min_idle) 1011 continue; 1012 1013 min_idle = idle; 1014 e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP; 1015 e->data.ib1 |= ib1 & AIROHA_FOE_IB1_BIND_TIMESTAMP; 1016 } 1017 } 1018 1019 static void airoha_ppe_foe_flow_entry_update(struct airoha_ppe *ppe, 1020 struct airoha_flow_table_entry *e) 1021 { 1022 struct airoha_foe_entry *hwe_p, hwe = {}; 1023 1024 spin_lock_bh(&ppe_lock); 1025 1026 if (e->type == FLOW_TYPE_L2) { 1027 airoha_ppe_foe_flow_l2_entry_update(ppe, e); 1028 goto unlock; 1029 } 1030 1031 if (e->hash == 0xffff) 1032 goto unlock; 1033 1034 hwe_p = airoha_ppe_foe_get_entry_locked(ppe, e->hash); 1035 if (!hwe_p) 1036 goto unlock; 1037 1038 memcpy(&hwe, hwe_p, sizeof(*hwe_p)); 1039 if (!airoha_ppe_foe_compare_entry(e, &hwe)) { 1040 e->hash = 0xffff; 1041 goto unlock; 1042 } 1043 1044 e->data.ib1 = hwe.ib1; 1045 unlock: 1046 spin_unlock_bh(&ppe_lock); 1047 } 1048 1049 static int airoha_ppe_entry_idle_time(struct airoha_ppe *ppe, 1050 struct airoha_flow_table_entry *e) 1051 { 1052 airoha_ppe_foe_flow_entry_update(ppe, e); 1053 1054 return airoha_ppe_get_entry_idle_time(ppe, e->data.ib1); 1055 } 1056 1057 static int airoha_ppe_flow_offload_replace(struct airoha_eth *eth, 1058 struct flow_cls_offload *f) 1059 { 1060 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 1061 struct airoha_flow_table_entry *e; 1062 struct airoha_flow_data data = {}; 1063 struct net_device *odev = NULL; 1064 struct flow_action_entry *act; 1065 struct airoha_foe_entry hwe; 1066 int err, i, offload_type; 1067 u16 addr_type = 0; 1068 u8 l4proto = 0; 1069 1070 if (rhashtable_lookup(ð->flow_table, &f->cookie, 1071 airoha_flow_table_params)) 1072 return -EEXIST; 1073 1074 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) 1075 return -EOPNOTSUPP; 1076 1077 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 1078 struct flow_match_control match; 1079 1080 flow_rule_match_control(rule, &match); 1081 addr_type = match.key->addr_type; 1082 if (flow_rule_has_control_flags(match.mask->flags, 1083 f->common.extack)) 1084 return -EOPNOTSUPP; 1085 } else { 1086 return -EOPNOTSUPP; 1087 } 1088 1089 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 1090 struct flow_match_basic match; 1091 1092 flow_rule_match_basic(rule, &match); 1093 l4proto = match.key->ip_proto; 1094 } else { 1095 return -EOPNOTSUPP; 1096 } 1097 1098 switch (addr_type) { 1099 case 0: 1100 offload_type = PPE_PKT_TYPE_BRIDGE; 1101 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 1102 struct flow_match_eth_addrs match; 1103 1104 flow_rule_match_eth_addrs(rule, &match); 1105 memcpy(data.eth.h_dest, match.key->dst, ETH_ALEN); 1106 memcpy(data.eth.h_source, match.key->src, ETH_ALEN); 1107 } else { 1108 return -EOPNOTSUPP; 1109 } 1110 break; 1111 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1112 offload_type = PPE_PKT_TYPE_IPV4_HNAPT; 1113 break; 1114 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1115 offload_type = PPE_PKT_TYPE_IPV6_ROUTE_5T; 1116 break; 1117 default: 1118 return -EOPNOTSUPP; 1119 } 1120 1121 flow_action_for_each(i, act, &rule->action) { 1122 switch (act->id) { 1123 case FLOW_ACTION_MANGLE: 1124 if (offload_type == PPE_PKT_TYPE_BRIDGE) 1125 return -EOPNOTSUPP; 1126 1127 if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH) 1128 airoha_ppe_flow_mangle_eth(act, &data.eth); 1129 break; 1130 case FLOW_ACTION_REDIRECT: 1131 odev = act->dev; 1132 break; 1133 case FLOW_ACTION_CSUM: 1134 break; 1135 case FLOW_ACTION_VLAN_PUSH: 1136 if (data.vlan.num == 2 || 1137 act->vlan.proto != htons(ETH_P_8021Q)) 1138 return -EOPNOTSUPP; 1139 1140 data.vlan.hdr[data.vlan.num].id = act->vlan.vid; 1141 data.vlan.hdr[data.vlan.num].proto = act->vlan.proto; 1142 data.vlan.num++; 1143 break; 1144 case FLOW_ACTION_VLAN_POP: 1145 break; 1146 case FLOW_ACTION_PPPOE_PUSH: 1147 if (data.pppoe.num == 1 || data.vlan.num == 2) 1148 return -EOPNOTSUPP; 1149 1150 data.pppoe.sid = act->pppoe.sid; 1151 data.pppoe.num++; 1152 break; 1153 default: 1154 return -EOPNOTSUPP; 1155 } 1156 } 1157 1158 if (!is_valid_ether_addr(data.eth.h_source) || 1159 !is_valid_ether_addr(data.eth.h_dest)) 1160 return -EINVAL; 1161 1162 err = airoha_ppe_foe_entry_prepare(eth, &hwe, odev, offload_type, 1163 &data, l4proto); 1164 if (err) 1165 return err; 1166 1167 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 1168 struct flow_match_ports ports; 1169 1170 if (offload_type == PPE_PKT_TYPE_BRIDGE) 1171 return -EOPNOTSUPP; 1172 1173 flow_rule_match_ports(rule, &ports); 1174 data.src_port = ports.key->src; 1175 data.dst_port = ports.key->dst; 1176 } else if (offload_type != PPE_PKT_TYPE_BRIDGE) { 1177 return -EOPNOTSUPP; 1178 } 1179 1180 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1181 struct flow_match_ipv4_addrs addrs; 1182 1183 flow_rule_match_ipv4_addrs(rule, &addrs); 1184 data.v4.src_addr = addrs.key->src; 1185 data.v4.dst_addr = addrs.key->dst; 1186 airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, false); 1187 } 1188 1189 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 1190 struct flow_match_ipv6_addrs addrs; 1191 1192 flow_rule_match_ipv6_addrs(rule, &addrs); 1193 1194 data.v6.src_addr = addrs.key->src; 1195 data.v6.dst_addr = addrs.key->dst; 1196 airoha_ppe_foe_entry_set_ipv6_tuple(&hwe, &data); 1197 } 1198 1199 flow_action_for_each(i, act, &rule->action) { 1200 if (act->id != FLOW_ACTION_MANGLE) 1201 continue; 1202 1203 if (offload_type == PPE_PKT_TYPE_BRIDGE) 1204 return -EOPNOTSUPP; 1205 1206 switch (act->mangle.htype) { 1207 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 1208 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 1209 err = airoha_ppe_flow_mangle_ports(act, &data); 1210 break; 1211 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 1212 err = airoha_ppe_flow_mangle_ipv4(act, &data); 1213 break; 1214 case FLOW_ACT_MANGLE_HDR_TYPE_ETH: 1215 /* handled earlier */ 1216 break; 1217 default: 1218 return -EOPNOTSUPP; 1219 } 1220 1221 if (err) 1222 return err; 1223 } 1224 1225 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1226 err = airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, true); 1227 if (err) 1228 return err; 1229 } 1230 1231 e = kzalloc_obj(*e); 1232 if (!e) 1233 return -ENOMEM; 1234 1235 e->cookie = f->cookie; 1236 memcpy(&e->data, &hwe, sizeof(e->data)); 1237 1238 err = airoha_ppe_foe_flow_commit_entry(eth->ppe, e); 1239 if (err) 1240 goto free_entry; 1241 1242 err = rhashtable_insert_fast(ð->flow_table, &e->node, 1243 airoha_flow_table_params); 1244 if (err < 0) 1245 goto remove_foe_entry; 1246 1247 return 0; 1248 1249 remove_foe_entry: 1250 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 1251 free_entry: 1252 kfree(e); 1253 1254 return err; 1255 } 1256 1257 static int airoha_ppe_flow_offload_destroy(struct airoha_eth *eth, 1258 struct flow_cls_offload *f) 1259 { 1260 struct airoha_flow_table_entry *e; 1261 1262 e = rhashtable_lookup(ð->flow_table, &f->cookie, 1263 airoha_flow_table_params); 1264 if (!e) 1265 return -ENOENT; 1266 1267 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 1268 rhashtable_remove_fast(ð->flow_table, &e->node, 1269 airoha_flow_table_params); 1270 kfree(e); 1271 1272 return 0; 1273 } 1274 1275 void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 hash, 1276 struct airoha_foe_stats64 *stats) 1277 { 1278 struct airoha_eth *eth = ppe->eth; 1279 int ppe_num_stats_entries; 1280 struct airoha_npu *npu; 1281 u32 index; 1282 1283 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 1284 if (ppe_num_stats_entries < 0) 1285 return; 1286 1287 if (airoha_ppe_foe_get_flow_stats_index(ppe, hash, &index)) 1288 return; 1289 1290 if (index >= ppe_num_stats_entries) 1291 return; 1292 1293 rcu_read_lock(); 1294 1295 npu = rcu_dereference(eth->npu); 1296 if (npu) { 1297 u64 packets = ppe->foe_stats[index].packets; 1298 u64 bytes = ppe->foe_stats[index].bytes; 1299 struct airoha_foe_stats npu_stats; 1300 1301 memcpy_fromio(&npu_stats, &npu->stats[index], 1302 sizeof(*npu->stats)); 1303 stats->packets = packets << 32 | npu_stats.packets; 1304 stats->bytes = bytes << 32 | npu_stats.bytes; 1305 } 1306 1307 rcu_read_unlock(); 1308 } 1309 1310 static int airoha_ppe_flow_offload_stats(struct airoha_eth *eth, 1311 struct flow_cls_offload *f) 1312 { 1313 struct airoha_flow_table_entry *e; 1314 u32 idle; 1315 1316 e = rhashtable_lookup(ð->flow_table, &f->cookie, 1317 airoha_flow_table_params); 1318 if (!e) 1319 return -ENOENT; 1320 1321 idle = airoha_ppe_entry_idle_time(eth->ppe, e); 1322 f->stats.lastused = jiffies - idle * HZ; 1323 1324 if (e->hash != 0xffff) { 1325 struct airoha_foe_stats64 stats = {}; 1326 1327 airoha_ppe_foe_entry_get_stats(eth->ppe, e->hash, &stats); 1328 f->stats.pkts += (stats.packets - e->stats.packets); 1329 f->stats.bytes += (stats.bytes - e->stats.bytes); 1330 e->stats = stats; 1331 } 1332 1333 return 0; 1334 } 1335 1336 static int airoha_ppe_flow_offload_cmd(struct airoha_eth *eth, 1337 struct flow_cls_offload *f) 1338 { 1339 switch (f->command) { 1340 case FLOW_CLS_REPLACE: 1341 return airoha_ppe_flow_offload_replace(eth, f); 1342 case FLOW_CLS_DESTROY: 1343 return airoha_ppe_flow_offload_destroy(eth, f); 1344 case FLOW_CLS_STATS: 1345 return airoha_ppe_flow_offload_stats(eth, f); 1346 default: 1347 break; 1348 } 1349 1350 return -EOPNOTSUPP; 1351 } 1352 1353 static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe) 1354 { 1355 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 1356 struct airoha_foe_entry *hwe = ppe->foe; 1357 int i; 1358 1359 for (i = 0; i < sram_num_entries; i++) { 1360 int err; 1361 1362 memset(&hwe[i], 0, sizeof(*hwe)); 1363 err = airoha_ppe_foe_commit_sram_entry(ppe, i); 1364 if (err) 1365 return err; 1366 } 1367 1368 return 0; 1369 } 1370 1371 static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth) 1372 { 1373 struct airoha_npu *npu = airoha_npu_get(eth->dev); 1374 1375 if (IS_ERR(npu)) { 1376 request_module("airoha-npu"); 1377 npu = airoha_npu_get(eth->dev); 1378 } 1379 1380 return npu; 1381 } 1382 1383 static int airoha_ppe_wait_for_npu_init(struct airoha_eth *eth) 1384 { 1385 int err; 1386 u32 val; 1387 1388 /* PPE_FLOW_CFG default register value is 0. Since we reset FE 1389 * during the device probe we can just check the configured value 1390 * is not 0 here. 1391 */ 1392 err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC, 1393 100 * USEC_PER_MSEC, false, eth, 1394 REG_PPE_PPE_FLOW_CFG(0)); 1395 if (err) 1396 return err; 1397 1398 if (airoha_ppe_is_enabled(eth, 1)) 1399 err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC, 1400 100 * USEC_PER_MSEC, false, eth, 1401 REG_PPE_PPE_FLOW_CFG(1)); 1402 1403 return err; 1404 } 1405 1406 static int airoha_ppe_offload_setup(struct airoha_eth *eth) 1407 { 1408 struct airoha_npu *npu = airoha_ppe_npu_get(eth); 1409 struct airoha_ppe *ppe = eth->ppe; 1410 int err, ppe_num_stats_entries; 1411 1412 if (IS_ERR(npu)) 1413 return PTR_ERR(npu); 1414 1415 err = npu->ops.ppe_init(npu); 1416 if (err) 1417 goto error_npu_put; 1418 1419 /* Wait for NPU PPE configuration to complete */ 1420 err = airoha_ppe_wait_for_npu_init(eth); 1421 if (err) 1422 goto error_npu_put; 1423 1424 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 1425 if (ppe_num_stats_entries > 0) { 1426 err = npu->ops.ppe_init_stats(npu, ppe->foe_stats_dma, 1427 ppe_num_stats_entries); 1428 if (err) 1429 goto error_npu_put; 1430 } 1431 1432 airoha_ppe_hw_init(ppe); 1433 airoha_ppe_foe_flow_stats_reset(ppe, npu); 1434 1435 rcu_assign_pointer(eth->npu, npu); 1436 synchronize_rcu(); 1437 1438 return 0; 1439 1440 error_npu_put: 1441 airoha_npu_put(npu); 1442 1443 return err; 1444 } 1445 1446 int airoha_ppe_setup_tc_block_cb(struct airoha_ppe_dev *dev, void *type_data) 1447 { 1448 struct airoha_ppe *ppe = dev->priv; 1449 struct airoha_eth *eth = ppe->eth; 1450 int err = 0; 1451 1452 /* Netfilter flowtable can try to offload flower rules while not all 1453 * the net_devices are registered or initialized. Delay offloading 1454 * until all net_devices are registered in the system. 1455 */ 1456 if (!test_bit(DEV_STATE_REGISTERED, ð->state)) 1457 return -EBUSY; 1458 1459 mutex_lock(&flow_offload_mutex); 1460 1461 if (!eth->npu) 1462 err = airoha_ppe_offload_setup(eth); 1463 if (!err) 1464 err = airoha_ppe_flow_offload_cmd(eth, type_data); 1465 1466 mutex_unlock(&flow_offload_mutex); 1467 1468 return err; 1469 } 1470 1471 void airoha_ppe_check_skb(struct airoha_ppe_dev *dev, struct sk_buff *skb, 1472 u16 hash, bool rx_wlan) 1473 { 1474 struct airoha_ppe *ppe = dev->priv; 1475 u32 ppe_hash_mask = airoha_ppe_get_total_num_entries(ppe) - 1; 1476 u16 now, diff; 1477 1478 if (hash > ppe_hash_mask) 1479 return; 1480 1481 now = (u16)jiffies; 1482 diff = now - ppe->foe_check_time[hash]; 1483 if (diff < HZ / 10) 1484 return; 1485 1486 ppe->foe_check_time[hash] = now; 1487 airoha_ppe_foe_insert_entry(ppe, skb, hash, rx_wlan); 1488 } 1489 1490 void airoha_ppe_init_upd_mem(struct airoha_gdm_dev *dev, const u8 *addr) 1491 { 1492 struct airoha_gdm_port *port = dev->port; 1493 struct airoha_eth *eth = dev->eth; 1494 u32 val; 1495 1496 val = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) | addr[5]; 1497 airoha_fe_wr(eth, REG_UPDMEM_DATA(0), val); 1498 airoha_fe_wr(eth, REG_UPDMEM_CTRL(0), 1499 FIELD_PREP(PPE_UPDMEM_ADDR_MASK, port->id) | 1500 PPE_UPDMEM_WR_MASK | PPE_UPDMEM_REQ_MASK); 1501 1502 val = (addr[0] << 8) | addr[1]; 1503 airoha_fe_wr(eth, REG_UPDMEM_DATA(0), val); 1504 airoha_fe_wr(eth, REG_UPDMEM_CTRL(0), 1505 FIELD_PREP(PPE_UPDMEM_ADDR_MASK, port->id) | 1506 FIELD_PREP(PPE_UPDMEM_OFFSET_MASK, 1) | 1507 PPE_UPDMEM_WR_MASK | PPE_UPDMEM_REQ_MASK); 1508 } 1509 1510 struct airoha_ppe_dev *airoha_ppe_get_dev(struct device *dev) 1511 { 1512 struct platform_device *pdev; 1513 struct device_node *np; 1514 struct airoha_eth *eth; 1515 1516 np = of_parse_phandle(dev->of_node, "airoha,eth", 0); 1517 if (!np) 1518 return ERR_PTR(-ENODEV); 1519 1520 pdev = of_find_device_by_node(np); 1521 if (!pdev) { 1522 dev_err(dev, "cannot find device node %s\n", np->name); 1523 of_node_put(np); 1524 return ERR_PTR(-ENODEV); 1525 } 1526 of_node_put(np); 1527 1528 if (!try_module_get(THIS_MODULE)) { 1529 dev_err(dev, "failed to get the device driver module\n"); 1530 goto error_pdev_put; 1531 } 1532 1533 eth = platform_get_drvdata(pdev); 1534 if (!eth) 1535 goto error_module_put; 1536 1537 if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER)) { 1538 dev_err(&pdev->dev, 1539 "failed to create device link to consumer %s\n", 1540 dev_name(dev)); 1541 goto error_module_put; 1542 } 1543 1544 return ð->ppe->dev; 1545 1546 error_module_put: 1547 module_put(THIS_MODULE); 1548 error_pdev_put: 1549 platform_device_put(pdev); 1550 1551 return ERR_PTR(-ENODEV); 1552 } 1553 EXPORT_SYMBOL_GPL(airoha_ppe_get_dev); 1554 1555 void airoha_ppe_put_dev(struct airoha_ppe_dev *dev) 1556 { 1557 struct airoha_ppe *ppe = dev->priv; 1558 struct airoha_eth *eth = ppe->eth; 1559 1560 module_put(THIS_MODULE); 1561 put_device(eth->dev); 1562 } 1563 EXPORT_SYMBOL_GPL(airoha_ppe_put_dev); 1564 1565 int airoha_ppe_init(struct airoha_eth *eth) 1566 { 1567 int foe_size, err, ppe_num_stats_entries; 1568 u32 ppe_num_entries; 1569 struct airoha_ppe *ppe; 1570 1571 ppe = devm_kzalloc(eth->dev, sizeof(*ppe), GFP_KERNEL); 1572 if (!ppe) 1573 return -ENOMEM; 1574 1575 ppe->dev.ops.setup_tc_block_cb = airoha_ppe_setup_tc_block_cb; 1576 ppe->dev.ops.check_skb = airoha_ppe_check_skb; 1577 ppe->dev.priv = ppe; 1578 ppe->eth = eth; 1579 eth->ppe = ppe; 1580 1581 ppe_num_entries = airoha_ppe_get_total_num_entries(ppe); 1582 foe_size = ppe_num_entries * sizeof(struct airoha_foe_entry); 1583 ppe->foe = dmam_alloc_coherent(eth->dev, foe_size, &ppe->foe_dma, 1584 GFP_KERNEL); 1585 if (!ppe->foe) 1586 return -ENOMEM; 1587 1588 ppe->foe_flow = devm_kzalloc(eth->dev, 1589 ppe_num_entries * sizeof(*ppe->foe_flow), 1590 GFP_KERNEL); 1591 if (!ppe->foe_flow) 1592 return -ENOMEM; 1593 1594 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 1595 if (ppe_num_stats_entries > 0) { 1596 foe_size = ppe_num_stats_entries * sizeof(*ppe->foe_stats); 1597 ppe->foe_stats = dmam_alloc_coherent(eth->dev, foe_size, 1598 &ppe->foe_stats_dma, 1599 GFP_KERNEL); 1600 if (!ppe->foe_stats) 1601 return -ENOMEM; 1602 } 1603 1604 ppe->foe_check_time = devm_kzalloc(eth->dev, ppe_num_entries, 1605 GFP_KERNEL); 1606 if (!ppe->foe_check_time) 1607 return -ENOMEM; 1608 1609 err = airoha_ppe_flush_sram_entries(ppe); 1610 if (err) 1611 return err; 1612 1613 err = rhashtable_init(ð->flow_table, &airoha_flow_table_params); 1614 if (err) 1615 return err; 1616 1617 err = rhashtable_init(&ppe->l2_flows, &airoha_l2_flow_table_params); 1618 if (err) 1619 goto error_flow_table_destroy; 1620 1621 err = airoha_ppe_debugfs_init(ppe); 1622 if (err) 1623 goto error_l2_flow_table_destroy; 1624 1625 return 0; 1626 1627 error_l2_flow_table_destroy: 1628 rhashtable_destroy(&ppe->l2_flows); 1629 error_flow_table_destroy: 1630 rhashtable_destroy(ð->flow_table); 1631 1632 return err; 1633 } 1634 1635 void airoha_ppe_deinit(struct airoha_eth *eth) 1636 { 1637 struct airoha_npu *npu; 1638 1639 mutex_lock(&flow_offload_mutex); 1640 1641 npu = rcu_replace_pointer(eth->npu, NULL, 1642 lockdep_is_held(&flow_offload_mutex)); 1643 if (npu) { 1644 npu->ops.ppe_deinit(npu); 1645 airoha_npu_put(npu); 1646 } 1647 1648 mutex_unlock(&flow_offload_mutex); 1649 1650 rhashtable_destroy(ð->ppe->l2_flows); 1651 rhashtable_destroy(ð->flow_table); 1652 debugfs_remove(eth->ppe->debugfs_dir); 1653 } 1654