1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 7 #include <linux/ip.h> 8 #include <linux/ipv6.h> 9 #include <linux/of_platform.h> 10 #include <linux/platform_device.h> 11 #include <linux/rhashtable.h> 12 #include <net/ipv6.h> 13 #include <net/pkt_cls.h> 14 15 #include "airoha_regs.h" 16 #include "airoha_eth.h" 17 18 /* Serialize airoha_gdm_dev flags, QDMA pointer and PPE CPU port 19 * configuration. 20 */ 21 DEFINE_MUTEX(flow_offload_mutex); 22 static DEFINE_SPINLOCK(ppe_lock); 23 24 static const struct rhashtable_params airoha_flow_table_params = { 25 .head_offset = offsetof(struct airoha_flow_table_entry, node), 26 .key_offset = offsetof(struct airoha_flow_table_entry, cookie), 27 .key_len = sizeof(unsigned long), 28 .automatic_shrinking = true, 29 }; 30 31 static const struct rhashtable_params airoha_l2_flow_table_params = { 32 .head_offset = offsetof(struct airoha_flow_table_entry, l2_node), 33 .key_offset = offsetof(struct airoha_flow_table_entry, data.bridge), 34 .key_len = 2 * ETH_ALEN, 35 .automatic_shrinking = true, 36 }; 37 38 static int airoha_ppe_get_num_stats_entries(struct airoha_ppe *ppe) 39 { 40 if (!IS_ENABLED(CONFIG_NET_AIROHA_FLOW_STATS)) 41 return -EOPNOTSUPP; 42 43 if (airoha_is_7583(ppe->eth)) 44 return -EOPNOTSUPP; 45 46 return PPE_STATS_NUM_ENTRIES; 47 } 48 49 static int airoha_ppe_get_total_num_stats_entries(struct airoha_ppe *ppe) 50 { 51 int num_stats = airoha_ppe_get_num_stats_entries(ppe); 52 53 if (num_stats > 0) { 54 struct airoha_eth *eth = ppe->eth; 55 56 num_stats = num_stats * eth->soc->num_ppe; 57 } 58 59 return num_stats; 60 } 61 62 static u32 airoha_ppe_get_total_sram_num_entries(struct airoha_ppe *ppe) 63 { 64 struct airoha_eth *eth = ppe->eth; 65 66 return PPE_SRAM_NUM_ENTRIES * eth->soc->num_ppe; 67 } 68 69 u32 airoha_ppe_get_total_num_entries(struct airoha_ppe *ppe) 70 { 71 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 72 73 return sram_num_entries + PPE_DRAM_NUM_ENTRIES; 74 } 75 76 bool airoha_ppe_is_enabled(struct airoha_eth *eth, int index) 77 { 78 if (index >= eth->soc->num_ppe) 79 return false; 80 81 return airoha_fe_rr(eth, REG_PPE_GLO_CFG(index)) & PPE_GLO_CFG_EN_MASK; 82 } 83 84 static u32 airoha_ppe_get_timestamp(struct airoha_ppe *ppe) 85 { 86 return airoha_fe_get(ppe->eth, REG_FE_FOE_TS, 87 AIROHA_FOE_IB1_BIND_TIMESTAMP); 88 } 89 90 void airoha_ppe_set_cpu_port(struct airoha_gdm_dev *dev, u8 ppe_id, u8 fport) 91 { 92 struct airoha_qdma *qdma = airoha_qdma_deref(dev); 93 struct airoha_eth *eth = dev->eth; 94 u8 qdma_id = qdma - ð->qdma[0]; 95 u32 fe_cpu_port; 96 97 fe_cpu_port = qdma_id ? FE_PSE_PORT_CDM2 : FE_PSE_PORT_CDM1; 98 airoha_fe_rmw(eth, REG_PPE_DFT_CPORT(ppe_id, fport), 99 DFT_CPORT_MASK(fport), 100 __field_prep(DFT_CPORT_MASK(fport), fe_cpu_port)); 101 } 102 103 void airoha_ppe_set_xmit_frame_size(struct airoha_gdm_dev *dev) 104 { 105 struct airoha_gdm_port *port = dev->port; 106 struct airoha_eth *eth = dev->eth; 107 int i, ppe_id, index; 108 u32 len = 0; 109 110 for (i = 0; i < ARRAY_SIZE(port->devs); i++) { 111 struct airoha_gdm_dev *d = port->devs[i]; 112 struct net_device *netdev; 113 114 if (!d) 115 continue; 116 117 netdev = netdev_from_priv(d); 118 if (netif_running(netdev)) 119 len = max_t(u32, len, netdev->mtu); 120 } 121 len += VLAN_ETH_HLEN; 122 123 ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1); 124 index = port->id == AIROHA_GDM4_IDX ? 7 : port->id; 125 airoha_fe_rmw(eth, REG_PPE_MTU(ppe_id, index), 126 FP_EGRESS_MTU_MASK(index), 127 __field_prep(FP_EGRESS_MTU_MASK(index), len)); 128 } 129 130 static void airoha_ppe_hw_init(struct airoha_ppe *ppe) 131 { 132 u32 sram_ppe_num_data_entries = PPE_SRAM_NUM_ENTRIES, sram_num_entries; 133 u32 sram_tb_size, dram_num_entries; 134 struct airoha_eth *eth = ppe->eth; 135 int i, sram_num_stats_entries; 136 137 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 138 sram_tb_size = sram_num_entries * sizeof(struct airoha_foe_entry); 139 dram_num_entries = PPE_RAM_NUM_ENTRIES_SHIFT(PPE_DRAM_NUM_ENTRIES); 140 141 sram_num_stats_entries = airoha_ppe_get_num_stats_entries(ppe); 142 if (sram_num_stats_entries > 0) 143 sram_ppe_num_data_entries -= sram_num_stats_entries; 144 sram_ppe_num_data_entries = 145 PPE_RAM_NUM_ENTRIES_SHIFT(sram_ppe_num_data_entries); 146 147 for (i = 0; i < eth->soc->num_ppe; i++) { 148 airoha_fe_wr(eth, REG_PPE_TB_BASE(i), 149 ppe->foe_dma + sram_tb_size); 150 151 airoha_fe_rmw(eth, REG_PPE_BND_AGE0(i), 152 PPE_BIND_AGE0_DELTA_NON_L4 | 153 PPE_BIND_AGE0_DELTA_UDP, 154 FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 60) | 155 FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 60)); 156 airoha_fe_rmw(eth, REG_PPE_BND_AGE1(i), 157 PPE_BIND_AGE1_DELTA_TCP_FIN | 158 PPE_BIND_AGE1_DELTA_TCP, 159 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 160 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 60)); 161 162 airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i), 163 PPE_SRAM_TABLE_EN_MASK | 164 PPE_SRAM_HASH1_EN_MASK | 165 PPE_DRAM_TABLE_EN_MASK | 166 PPE_SRAM_HASH0_MODE_MASK | 167 PPE_SRAM_HASH1_MODE_MASK | 168 PPE_DRAM_HASH0_MODE_MASK | 169 PPE_DRAM_HASH1_MODE_MASK, 170 FIELD_PREP(PPE_SRAM_TABLE_EN_MASK, 1) | 171 FIELD_PREP(PPE_SRAM_HASH1_EN_MASK, 1) | 172 FIELD_PREP(PPE_SRAM_HASH1_MODE_MASK, 1) | 173 FIELD_PREP(PPE_DRAM_HASH1_MODE_MASK, 3)); 174 175 airoha_fe_rmw(eth, REG_PPE_TB_CFG(i), 176 PPE_TB_CFG_SEARCH_MISS_MASK | 177 PPE_SRAM_TB_NUM_ENTRY_MASK | 178 PPE_DRAM_TB_NUM_ENTRY_MASK | 179 PPE_TB_CFG_KEEPALIVE_MASK | 180 PPE_TB_ENTRY_SIZE_MASK, 181 FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) | 182 FIELD_PREP(PPE_TB_ENTRY_SIZE_MASK, 0) | 183 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 184 sram_ppe_num_data_entries) | 185 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 186 dram_num_entries)); 187 188 airoha_fe_rmw(eth, REG_PPE_BIND_RATE(i), 189 PPE_BIND_RATE_L2B_BIND_MASK | 190 PPE_BIND_RATE_BIND_MASK, 191 FIELD_PREP(PPE_BIND_RATE_L2B_BIND_MASK, 0x1e) | 192 FIELD_PREP(PPE_BIND_RATE_BIND_MASK, 0x1e)); 193 194 airoha_fe_wr(eth, REG_PPE_HASH_SEED(i), PPE_HASH_SEED); 195 airoha_fe_clear(eth, REG_PPE_PPE_FLOW_CFG(i), 196 PPE_FLOW_CFG_IP6_6RD_MASK); 197 } 198 199 for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { 200 struct airoha_gdm_port *port = eth->ports[i]; 201 int j; 202 203 if (!port) 204 continue; 205 206 for (j = 0; j < ARRAY_SIZE(port->devs); j++) { 207 struct airoha_gdm_dev *dev = port->devs[j]; 208 int ppe_id; 209 u8 fport; 210 211 if (!dev) 212 continue; 213 214 ppe_id = !airoha_is_lan_gdm_dev(dev) && 215 airoha_ppe_is_enabled(eth, 1); 216 fport = airoha_get_fe_port(dev); 217 airoha_ppe_set_cpu_port(dev, ppe_id, fport); 218 airoha_ppe_set_xmit_frame_size(dev); 219 } 220 } 221 } 222 223 static void airoha_ppe_flow_mangle_eth(const struct flow_action_entry *act, void *eth) 224 { 225 void *dest = eth + act->mangle.offset; 226 const void *src = &act->mangle.val; 227 228 if (act->mangle.offset > 8) 229 return; 230 231 if (act->mangle.mask == 0xffff) { 232 src += 2; 233 dest += 2; 234 } 235 236 memcpy(dest, src, act->mangle.mask ? 2 : 4); 237 } 238 239 static int airoha_ppe_flow_mangle_ports(const struct flow_action_entry *act, 240 struct airoha_flow_data *data) 241 { 242 u32 val = be32_to_cpu((__force __be32)act->mangle.val); 243 244 switch (act->mangle.offset) { 245 case 0: 246 if ((__force __be32)act->mangle.mask == ~cpu_to_be32(0xffff)) 247 data->dst_port = cpu_to_be16(val); 248 else 249 data->src_port = cpu_to_be16(val >> 16); 250 break; 251 case 2: 252 data->dst_port = cpu_to_be16(val); 253 break; 254 default: 255 return -EINVAL; 256 } 257 258 return 0; 259 } 260 261 static int airoha_ppe_flow_mangle_ipv4(const struct flow_action_entry *act, 262 struct airoha_flow_data *data) 263 { 264 __be32 *dest; 265 266 switch (act->mangle.offset) { 267 case offsetof(struct iphdr, saddr): 268 dest = &data->v4.src_addr; 269 break; 270 case offsetof(struct iphdr, daddr): 271 dest = &data->v4.dst_addr; 272 break; 273 default: 274 return -EINVAL; 275 } 276 277 memcpy(dest, &act->mangle.val, sizeof(u32)); 278 279 return 0; 280 } 281 282 static int airoha_ppe_get_wdma_info(struct net_device *dev, const u8 *addr, 283 struct airoha_wdma_info *info) 284 { 285 struct net_device_path_stack stack; 286 struct net_device_path_ctx ctx = { 287 .dev = dev, 288 }; 289 struct net_device_path *path; 290 int err; 291 292 if (!dev) 293 return -ENODEV; 294 295 ether_addr_copy(ctx.daddr, addr); 296 297 rcu_read_lock(); 298 err = dev_fill_forward_path(&ctx, &stack); 299 rcu_read_unlock(); 300 if (err) 301 return err; 302 303 path = &stack.path[stack.num_paths - 1]; 304 if (path->type != DEV_PATH_MTK_WDMA) { 305 err = -EINVAL; 306 goto err_out; 307 } 308 309 info->idx = path->mtk_wdma.wdma_idx; 310 info->bss = path->mtk_wdma.bss; 311 info->wcid = path->mtk_wdma.wcid; 312 err_out: 313 dev_fill_forward_path_release(&stack); 314 315 return err; 316 } 317 318 static int airoha_get_dsa_port(struct net_device **dev) 319 { 320 #if IS_ENABLED(CONFIG_NET_DSA) 321 struct dsa_port *dp = dsa_port_from_netdev(*dev); 322 323 if (IS_ERR(dp)) 324 return -ENODEV; 325 326 *dev = dsa_port_to_conduit(dp); 327 return dp->index; 328 #else 329 return -ENODEV; 330 #endif 331 } 332 333 static void airoha_ppe_foe_set_bridge_addrs(struct airoha_foe_bridge *br, 334 struct ethhdr *eh) 335 { 336 br->dest_mac_hi = get_unaligned_be32(eh->h_dest); 337 br->dest_mac_lo = get_unaligned_be16(eh->h_dest + 4); 338 br->src_mac_hi = get_unaligned_be16(eh->h_source); 339 br->src_mac_lo = get_unaligned_be32(eh->h_source + 2); 340 } 341 342 static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth, 343 struct airoha_foe_entry *hwe, 344 struct net_device *netdev, int type, 345 struct airoha_flow_data *data, 346 int l4proto, u8 priority) 347 { 348 u32 qdata = FIELD_PREP(AIROHA_FOE_SHAPER_ID, 0x7f), ports_pad, val; 349 int wlan_etype = -EINVAL, dsa_port = airoha_get_dsa_port(&netdev); 350 struct airoha_foe_mac_info_common *l2; 351 u8 smac_id = 0xf; 352 353 memset(hwe, 0, sizeof(*hwe)); 354 355 val = FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, AIROHA_FOE_STATE_BIND) | 356 FIELD_PREP(AIROHA_FOE_IB1_BIND_PACKET_TYPE, type) | 357 FIELD_PREP(AIROHA_FOE_IB1_BIND_UDP, l4proto == IPPROTO_UDP) | 358 FIELD_PREP(AIROHA_FOE_IB1_BIND_VLAN_LAYER, data->vlan.num) | 359 FIELD_PREP(AIROHA_FOE_IB1_BIND_VPM, data->vlan.num) | 360 FIELD_PREP(AIROHA_FOE_IB1_BIND_PPPOE, data->pppoe.num) | 361 AIROHA_FOE_IB1_BIND_TTL; 362 hwe->ib1 = val; 363 364 val = FIELD_PREP(AIROHA_FOE_IB2_PORT_AG, 0x1f); 365 if (netdev) { 366 struct airoha_wdma_info info = {}; 367 368 if (!airoha_ppe_get_wdma_info(netdev, data->eth.h_dest, 369 &info)) { 370 val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, info.idx) | 371 FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, 372 FE_PSE_PORT_CDM4); 373 qdata |= FIELD_PREP(AIROHA_FOE_ACTDP, info.bss); 374 wlan_etype = FIELD_PREP(AIROHA_FOE_MAC_WDMA_BAND, 375 info.idx) | 376 FIELD_PREP(AIROHA_FOE_MAC_WDMA_WCID, 377 info.wcid); 378 } else { 379 struct airoha_gdm_dev *dev = netdev_priv(netdev); 380 struct airoha_gdm_port *port; 381 u8 pse_port, channel; 382 383 if (!airoha_is_valid_gdm_dev(eth, dev)) 384 return -EINVAL; 385 386 port = dev->port; 387 if (dsa_port >= 0 || airoha_is_lan_gdm_dev(dev)) 388 pse_port = port->id == 4 ? FE_PSE_PORT_GDM4 389 : port->id; 390 else 391 pse_port = 2; /* uplink relies on GDM2 392 * loopback 393 */ 394 395 /* For traffic forwarded to DSA devices select QoS 396 * channel according to the DSA user port index, rely 397 * on port id otherwise. 398 */ 399 channel = dsa_port >= 0 ? dsa_port : port->id; 400 channel = channel % AIROHA_NUM_QOS_CHANNELS; 401 priority = priority % AIROHA_NUM_QOS_QUEUES; 402 qdata |= FIELD_PREP(AIROHA_FOE_CHANNEL, channel) | 403 FIELD_PREP(AIROHA_FOE_QID, priority); 404 405 val |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, pse_port) | 406 AIROHA_FOE_IB2_PSE_QOS; 407 /* For downlink traffic consume SRAM memory for hw 408 * forwarding descriptors queue. 409 */ 410 if (airoha_is_lan_gdm_dev(dev)) 411 val |= AIROHA_FOE_IB2_FAST_PATH; 412 if (dsa_port >= 0) 413 val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, 414 dsa_port); 415 416 smac_id = port->id; 417 } 418 } 419 420 if (is_multicast_ether_addr(data->eth.h_dest)) 421 val |= AIROHA_FOE_IB2_MULTICAST; 422 423 ports_pad = 0xa5a5a500 | (l4proto & 0xff); 424 if (type == PPE_PKT_TYPE_IPV4_ROUTE) 425 hwe->ipv4.orig_tuple.ports = ports_pad; 426 if (type == PPE_PKT_TYPE_IPV6_ROUTE_3T) 427 hwe->ipv6.ports = ports_pad; 428 429 if (type == PPE_PKT_TYPE_BRIDGE) { 430 airoha_ppe_foe_set_bridge_addrs(&hwe->bridge, &data->eth); 431 hwe->bridge.data = qdata; 432 hwe->bridge.ib2 = val; 433 l2 = &hwe->bridge.l2.common; 434 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 435 hwe->ipv6.data = qdata; 436 hwe->ipv6.ib2 = val; 437 l2 = &hwe->ipv6.l2; 438 l2->etype = ETH_P_IPV6; 439 } else { 440 hwe->ipv4.data = qdata; 441 hwe->ipv4.ib2 = val; 442 l2 = &hwe->ipv4.l2.common; 443 l2->etype = ETH_P_IP; 444 } 445 446 l2->dest_mac_hi = get_unaligned_be32(data->eth.h_dest); 447 l2->dest_mac_lo = get_unaligned_be16(data->eth.h_dest + 4); 448 if (type <= PPE_PKT_TYPE_IPV4_DSLITE) { 449 struct airoha_foe_mac_info *mac_info; 450 451 l2->src_mac_hi = get_unaligned_be32(data->eth.h_source); 452 hwe->ipv4.l2.src_mac_lo = 453 get_unaligned_be16(data->eth.h_source + 4); 454 455 mac_info = (struct airoha_foe_mac_info *)l2; 456 mac_info->pppoe_id = data->pppoe.sid; 457 } else { 458 l2->src_mac_hi = FIELD_PREP(AIROHA_FOE_MAC_SMAC_ID, smac_id) | 459 FIELD_PREP(AIROHA_FOE_MAC_PPPOE_ID, 460 data->pppoe.sid); 461 } 462 463 if (data->vlan.num) { 464 l2->vlan1 = data->vlan.hdr[0].id; 465 if (data->vlan.num == 2) 466 l2->vlan2 = data->vlan.hdr[1].id; 467 } 468 469 if (wlan_etype >= 0) { 470 l2->etype = wlan_etype; 471 } else if (dsa_port >= 0) { 472 l2->etype = BIT(dsa_port); 473 l2->etype |= !data->vlan.num ? BIT(15) : 0; 474 } else if (data->pppoe.num) { 475 l2->etype = ETH_P_PPP_SES; 476 } 477 478 return 0; 479 } 480 481 static int airoha_ppe_foe_entry_set_ipv4_tuple(struct airoha_foe_entry *hwe, 482 struct airoha_flow_data *data, 483 bool egress) 484 { 485 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 486 struct airoha_foe_ipv4_tuple *t; 487 488 switch (type) { 489 case PPE_PKT_TYPE_IPV4_HNAPT: 490 if (egress) { 491 t = &hwe->ipv4.new_tuple; 492 break; 493 } 494 fallthrough; 495 case PPE_PKT_TYPE_IPV4_DSLITE: 496 case PPE_PKT_TYPE_IPV4_ROUTE: 497 t = &hwe->ipv4.orig_tuple; 498 break; 499 default: 500 WARN_ON_ONCE(1); 501 return -EINVAL; 502 } 503 504 t->src_ip = be32_to_cpu(data->v4.src_addr); 505 t->dest_ip = be32_to_cpu(data->v4.dst_addr); 506 507 if (type != PPE_PKT_TYPE_IPV4_ROUTE) { 508 t->src_port = be16_to_cpu(data->src_port); 509 t->dest_port = be16_to_cpu(data->dst_port); 510 } 511 512 return 0; 513 } 514 515 static int airoha_ppe_foe_entry_set_ipv6_tuple(struct airoha_foe_entry *hwe, 516 struct airoha_flow_data *data) 517 518 { 519 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 520 u32 *src, *dest; 521 522 switch (type) { 523 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 524 case PPE_PKT_TYPE_IPV6_6RD: 525 hwe->ipv6.src_port = be16_to_cpu(data->src_port); 526 hwe->ipv6.dest_port = be16_to_cpu(data->dst_port); 527 fallthrough; 528 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 529 src = hwe->ipv6.src_ip; 530 dest = hwe->ipv6.dest_ip; 531 break; 532 default: 533 WARN_ON_ONCE(1); 534 return -EINVAL; 535 } 536 537 ipv6_addr_be32_to_cpu(src, data->v6.src_addr.s6_addr32); 538 ipv6_addr_be32_to_cpu(dest, data->v6.dst_addr.s6_addr32); 539 540 return 0; 541 } 542 543 static u32 airoha_ppe_foe_get_entry_hash(struct airoha_ppe *ppe, 544 struct airoha_foe_entry *hwe) 545 { 546 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 547 u32 ppe_hash_mask = airoha_ppe_get_total_num_entries(ppe) - 1; 548 u32 hash, hv1, hv2, hv3; 549 550 switch (type) { 551 case PPE_PKT_TYPE_IPV4_ROUTE: 552 case PPE_PKT_TYPE_IPV4_HNAPT: 553 hv1 = hwe->ipv4.orig_tuple.ports; 554 hv2 = hwe->ipv4.orig_tuple.dest_ip; 555 hv3 = hwe->ipv4.orig_tuple.src_ip; 556 break; 557 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 558 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 559 hv1 = hwe->ipv6.src_ip[3] ^ hwe->ipv6.dest_ip[3]; 560 hv1 ^= hwe->ipv6.ports; 561 562 hv2 = hwe->ipv6.src_ip[2] ^ hwe->ipv6.dest_ip[2]; 563 hv2 ^= hwe->ipv6.dest_ip[0]; 564 565 hv3 = hwe->ipv6.src_ip[1] ^ hwe->ipv6.dest_ip[1]; 566 hv3 ^= hwe->ipv6.src_ip[0]; 567 break; 568 case PPE_PKT_TYPE_BRIDGE: { 569 struct airoha_foe_mac_info *l2 = &hwe->bridge.l2; 570 571 hv1 = l2->common.src_mac_hi & 0xffff; 572 hv1 = hv1 << 16 | l2->src_mac_lo; 573 574 hv2 = l2->common.dest_mac_lo; 575 hv2 = hv2 << 16; 576 hv2 = hv2 | ((l2->common.src_mac_hi & 0xffff0000) >> 16); 577 578 hv3 = l2->common.dest_mac_hi; 579 break; 580 } 581 case PPE_PKT_TYPE_IPV4_DSLITE: 582 case PPE_PKT_TYPE_IPV6_6RD: 583 default: 584 WARN_ON_ONCE(1); 585 return ppe_hash_mask; 586 } 587 588 hash = (hv1 & hv2) | ((~hv1) & hv3); 589 hash = (hash >> 24) | ((hash & 0xffffff) << 8); 590 hash ^= hv1 ^ hv2 ^ hv3; 591 hash ^= hash >> 16; 592 hash &= ppe_hash_mask; 593 594 return hash; 595 } 596 597 static int airoha_ppe_foe_get_flow_stats_index(struct airoha_ppe *ppe, 598 u32 hash, u32 *index) 599 { 600 int ppe_num_stats_entries; 601 602 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 603 if (ppe_num_stats_entries < 0) 604 return ppe_num_stats_entries; 605 606 *index = hash >= ppe_num_stats_entries ? hash - PPE_STATS_NUM_ENTRIES 607 : hash; 608 609 return 0; 610 } 611 612 static void airoha_ppe_foe_flow_stat_entry_reset(struct airoha_ppe *ppe, 613 struct airoha_npu *npu, 614 int index) 615 { 616 memset_io(&npu->stats[index], 0, sizeof(*npu->stats)); 617 memset(&ppe->foe_stats[index], 0, sizeof(*ppe->foe_stats)); 618 } 619 620 static void airoha_ppe_foe_flow_stats_reset(struct airoha_ppe *ppe, 621 struct airoha_npu *npu) 622 { 623 int i, ppe_num_stats_entries; 624 625 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 626 if (ppe_num_stats_entries < 0) 627 return; 628 629 for (i = 0; i < ppe_num_stats_entries; i++) 630 airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, i); 631 } 632 633 static void airoha_ppe_foe_flow_stats_update(struct airoha_ppe *ppe, 634 struct airoha_npu *npu, 635 struct airoha_foe_entry *hwe, 636 u32 hash) 637 { 638 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 639 u32 index, pse_port, val, *data, *ib2, *meter; 640 int ppe_num_stats_entries; 641 u8 nbq; 642 643 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 644 if (ppe_num_stats_entries < 0) 645 return; 646 647 if (airoha_ppe_foe_get_flow_stats_index(ppe, hash, &index)) 648 return; 649 650 if (index >= ppe_num_stats_entries) 651 return; 652 653 if (type == PPE_PKT_TYPE_BRIDGE) { 654 data = &hwe->bridge.data; 655 ib2 = &hwe->bridge.ib2; 656 meter = &hwe->bridge.l2.meter; 657 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 658 data = &hwe->ipv6.data; 659 ib2 = &hwe->ipv6.ib2; 660 meter = &hwe->ipv6.meter; 661 } else { 662 data = &hwe->ipv4.data; 663 ib2 = &hwe->ipv4.ib2; 664 meter = &hwe->ipv4.l2.meter; 665 } 666 667 pse_port = FIELD_GET(AIROHA_FOE_IB2_PSE_PORT, *ib2); 668 if (pse_port == FE_PSE_PORT_CDM4) 669 return; 670 671 airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, index); 672 673 val = FIELD_GET(AIROHA_FOE_CHANNEL | AIROHA_FOE_QID, *data); 674 *data = (*data & ~AIROHA_FOE_ACTDP) | 675 FIELD_PREP(AIROHA_FOE_ACTDP, val); 676 677 val = *ib2 & (AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT | 678 AIROHA_FOE_IB2_PSE_QOS | AIROHA_FOE_IB2_FAST_PATH); 679 *meter |= FIELD_PREP(AIROHA_FOE_TUNNEL_MTU, val); 680 681 nbq = pse_port == 1 ? 6 : 5; 682 *ib2 &= ~(AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT | 683 AIROHA_FOE_IB2_PSE_QOS); 684 *ib2 |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, 6) | 685 FIELD_PREP(AIROHA_FOE_IB2_NBQ, nbq); 686 } 687 688 static struct airoha_foe_entry * 689 airoha_ppe_foe_get_entry_locked(struct airoha_ppe *ppe, u32 hash) 690 { 691 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 692 693 lockdep_assert_held(&ppe_lock); 694 695 if (hash < sram_num_entries) { 696 u32 *hwe = ppe->foe + hash * sizeof(struct airoha_foe_entry); 697 bool ppe2 = hash >= PPE_SRAM_NUM_ENTRIES; 698 struct airoha_eth *eth = ppe->eth; 699 u32 val; 700 int i; 701 702 airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2), 703 FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) | 704 PPE_SRAM_CTRL_REQ_MASK); 705 if (read_poll_timeout_atomic(airoha_fe_rr, val, 706 val & PPE_SRAM_CTRL_ACK_MASK, 707 10, 100, false, eth, 708 REG_PPE_RAM_CTRL(ppe2))) 709 return NULL; 710 711 for (i = 0; i < sizeof(struct airoha_foe_entry) / sizeof(*hwe); 712 i++) 713 hwe[i] = airoha_fe_rr(eth, 714 REG_PPE_RAM_ENTRY(ppe2, i)); 715 } 716 717 return ppe->foe + hash * sizeof(struct airoha_foe_entry); 718 } 719 720 struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe, 721 u32 hash) 722 { 723 struct airoha_foe_entry *hwe; 724 725 spin_lock_bh(&ppe_lock); 726 hwe = airoha_ppe_foe_get_entry_locked(ppe, hash); 727 spin_unlock_bh(&ppe_lock); 728 729 return hwe; 730 } 731 732 static bool airoha_ppe_foe_compare_entry(struct airoha_flow_table_entry *e, 733 struct airoha_foe_entry *hwe) 734 { 735 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 736 int len; 737 738 if ((hwe->ib1 ^ e->data.ib1) & AIROHA_FOE_IB1_BIND_UDP) 739 return false; 740 741 if (type > PPE_PKT_TYPE_IPV4_DSLITE) 742 len = offsetof(struct airoha_foe_entry, ipv6.data); 743 else 744 len = offsetof(struct airoha_foe_entry, ipv4.ib2); 745 746 return !memcmp(&e->data.d, &hwe->d, len - sizeof(hwe->ib1)); 747 } 748 749 static int airoha_ppe_foe_commit_sram_entry(struct airoha_ppe *ppe, u32 hash) 750 { 751 struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe); 752 bool ppe2 = hash >= PPE_SRAM_NUM_ENTRIES; 753 u32 *ptr = (u32 *)hwe, val; 754 int i; 755 756 for (i = 0; i < sizeof(*hwe) / sizeof(*ptr); i++) 757 airoha_fe_wr(ppe->eth, REG_PPE_RAM_ENTRY(ppe2, i), ptr[i]); 758 759 wmb(); 760 airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2), 761 FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) | 762 PPE_SRAM_CTRL_WR_MASK | PPE_SRAM_CTRL_REQ_MASK); 763 764 return read_poll_timeout_atomic(airoha_fe_rr, val, 765 val & PPE_SRAM_CTRL_ACK_MASK, 766 10, 100, false, ppe->eth, 767 REG_PPE_RAM_CTRL(ppe2)); 768 } 769 770 static int airoha_ppe_foe_commit_entry(struct airoha_ppe *ppe, 771 struct airoha_foe_entry *e, 772 u32 hash, bool rx_wlan) 773 { 774 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 775 struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe); 776 u32 ts = airoha_ppe_get_timestamp(ppe); 777 struct airoha_eth *eth = ppe->eth; 778 struct airoha_npu *npu; 779 int err = 0; 780 781 memcpy(&hwe->d, &e->d, sizeof(*hwe) - sizeof(hwe->ib1)); 782 wmb(); 783 784 e->ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP; 785 e->ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_TIMESTAMP, ts); 786 hwe->ib1 = e->ib1; 787 788 rcu_read_lock(); 789 790 npu = rcu_dereference(eth->npu); 791 if (!npu) { 792 err = -ENODEV; 793 goto unlock; 794 } 795 796 if (!rx_wlan) 797 airoha_ppe_foe_flow_stats_update(ppe, npu, hwe, hash); 798 799 if (hash < sram_num_entries) 800 err = airoha_ppe_foe_commit_sram_entry(ppe, hash); 801 unlock: 802 rcu_read_unlock(); 803 804 return err; 805 } 806 807 static void airoha_ppe_foe_remove_flow(struct airoha_ppe *ppe, 808 struct airoha_flow_table_entry *e) 809 { 810 lockdep_assert_held(&ppe_lock); 811 812 hlist_del_init(&e->list); 813 if (e->hash != 0xffff) { 814 e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_STATE; 815 e->data.ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, 816 AIROHA_FOE_STATE_INVALID); 817 airoha_ppe_foe_commit_entry(ppe, &e->data, e->hash, false); 818 e->hash = 0xffff; 819 } 820 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 821 hlist_del_init(&e->l2_subflow_node); 822 kfree(e); 823 } 824 } 825 826 static void airoha_ppe_foe_remove_l2_flow(struct airoha_ppe *ppe, 827 struct airoha_flow_table_entry *e) 828 { 829 struct hlist_head *head = &e->l2_flows; 830 struct hlist_node *n; 831 832 lockdep_assert_held(&ppe_lock); 833 834 rhashtable_remove_fast(&ppe->l2_flows, &e->l2_node, 835 airoha_l2_flow_table_params); 836 hlist_for_each_entry_safe(e, n, head, l2_subflow_node) 837 airoha_ppe_foe_remove_flow(ppe, e); 838 } 839 840 static void airoha_ppe_foe_flow_remove_entry(struct airoha_ppe *ppe, 841 struct airoha_flow_table_entry *e) 842 { 843 spin_lock_bh(&ppe_lock); 844 845 if (e->type == FLOW_TYPE_L2) 846 airoha_ppe_foe_remove_l2_flow(ppe, e); 847 else 848 airoha_ppe_foe_remove_flow(ppe, e); 849 850 spin_unlock_bh(&ppe_lock); 851 } 852 853 static int 854 airoha_ppe_foe_commit_subflow_entry(struct airoha_ppe *ppe, 855 struct airoha_flow_table_entry *e, 856 u32 hash, bool rx_wlan) 857 { 858 u32 mask = AIROHA_FOE_IB1_BIND_PACKET_TYPE | AIROHA_FOE_IB1_BIND_UDP; 859 struct airoha_foe_entry *hwe_p, hwe; 860 struct airoha_flow_table_entry *f; 861 int type; 862 863 hwe_p = airoha_ppe_foe_get_entry_locked(ppe, hash); 864 if (!hwe_p) 865 return -EINVAL; 866 867 f = kzalloc_obj(*f, GFP_ATOMIC); 868 if (!f) 869 return -ENOMEM; 870 871 hlist_add_head(&f->l2_subflow_node, &e->l2_flows); 872 f->type = FLOW_TYPE_L2_SUBFLOW; 873 f->hash = hash; 874 875 memcpy(&hwe, hwe_p, sizeof(*hwe_p)); 876 hwe.ib1 = (hwe.ib1 & mask) | (e->data.ib1 & ~mask); 877 878 type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe.ib1); 879 if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 880 memcpy(&hwe.ipv6.l2, &e->data.bridge.l2, sizeof(hwe.ipv6.l2)); 881 hwe.ipv6.ib2 = e->data.bridge.ib2; 882 /* setting smac_id to 0xf instruct the hw to keep original 883 * source mac address 884 */ 885 hwe.ipv6.l2.src_mac_hi = FIELD_PREP(AIROHA_FOE_MAC_SMAC_ID, 886 0xf); 887 } else { 888 memcpy(&hwe.bridge.l2, &e->data.bridge.l2, 889 sizeof(hwe.bridge.l2)); 890 hwe.bridge.ib2 = e->data.bridge.ib2; 891 if (type == PPE_PKT_TYPE_IPV4_HNAPT) 892 memcpy(&hwe.ipv4.new_tuple, &hwe.ipv4.orig_tuple, 893 sizeof(hwe.ipv4.new_tuple)); 894 } 895 896 hwe.bridge.data = e->data.bridge.data; 897 airoha_ppe_foe_commit_entry(ppe, &hwe, hash, rx_wlan); 898 899 return 0; 900 } 901 902 static void airoha_ppe_foe_insert_entry(struct airoha_ppe *ppe, 903 struct sk_buff *skb, 904 u32 hash, bool rx_wlan) 905 { 906 struct airoha_flow_table_entry *e; 907 struct airoha_foe_bridge br = {}; 908 struct airoha_foe_entry *hwe; 909 bool commit_done = false; 910 struct hlist_node *n; 911 u32 index, state; 912 913 spin_lock_bh(&ppe_lock); 914 915 hwe = airoha_ppe_foe_get_entry_locked(ppe, hash); 916 if (!hwe) 917 goto unlock; 918 919 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 920 if (state == AIROHA_FOE_STATE_BIND) 921 goto unlock; 922 923 index = airoha_ppe_foe_get_entry_hash(ppe, hwe); 924 hlist_for_each_entry_safe(e, n, &ppe->foe_flow[index], list) { 925 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 926 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 927 if (state != AIROHA_FOE_STATE_BIND) { 928 e->hash = 0xffff; 929 airoha_ppe_foe_remove_flow(ppe, e); 930 } 931 continue; 932 } 933 934 if (!airoha_ppe_foe_compare_entry(e, hwe)) 935 continue; 936 937 airoha_ppe_foe_commit_entry(ppe, &e->data, hash, rx_wlan); 938 commit_done = true; 939 e->hash = hash; 940 } 941 942 if (commit_done) 943 goto unlock; 944 945 airoha_ppe_foe_set_bridge_addrs(&br, eth_hdr(skb)); 946 e = rhashtable_lookup_fast(&ppe->l2_flows, &br, 947 airoha_l2_flow_table_params); 948 if (e) 949 airoha_ppe_foe_commit_subflow_entry(ppe, e, hash, rx_wlan); 950 unlock: 951 spin_unlock_bh(&ppe_lock); 952 } 953 954 static int 955 airoha_ppe_foe_l2_flow_commit_entry(struct airoha_ppe *ppe, 956 struct airoha_flow_table_entry *e) 957 { 958 struct airoha_flow_table_entry *prev; 959 960 e->type = FLOW_TYPE_L2; 961 prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &e->l2_node, 962 airoha_l2_flow_table_params); 963 if (!prev) 964 return 0; 965 966 if (IS_ERR(prev)) 967 return PTR_ERR(prev); 968 969 return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node, 970 &e->l2_node, 971 airoha_l2_flow_table_params); 972 } 973 974 static int airoha_ppe_foe_flow_commit_entry(struct airoha_ppe *ppe, 975 struct airoha_flow_table_entry *e) 976 { 977 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 978 u32 hash; 979 980 if (type == PPE_PKT_TYPE_BRIDGE) 981 return airoha_ppe_foe_l2_flow_commit_entry(ppe, e); 982 983 hash = airoha_ppe_foe_get_entry_hash(ppe, &e->data); 984 e->type = FLOW_TYPE_L4; 985 e->hash = 0xffff; 986 987 spin_lock_bh(&ppe_lock); 988 hlist_add_head(&e->list, &ppe->foe_flow[hash]); 989 spin_unlock_bh(&ppe_lock); 990 991 return 0; 992 } 993 994 static int airoha_ppe_get_entry_idle_time(struct airoha_ppe *ppe, u32 ib1) 995 { 996 u32 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1); 997 u32 ts, ts_mask, now = airoha_ppe_get_timestamp(ppe); 998 int idle; 999 1000 if (state == AIROHA_FOE_STATE_BIND) { 1001 ts = FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, ib1); 1002 ts_mask = AIROHA_FOE_IB1_BIND_TIMESTAMP; 1003 } else { 1004 ts = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, ib1); 1005 now = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, now); 1006 ts_mask = AIROHA_FOE_IB1_UNBIND_TIMESTAMP; 1007 } 1008 idle = now - ts; 1009 1010 return idle < 0 ? idle + ts_mask + 1 : idle; 1011 } 1012 1013 static void 1014 airoha_ppe_foe_flow_l2_entry_update(struct airoha_ppe *ppe, 1015 struct airoha_flow_table_entry *e) 1016 { 1017 int min_idle = airoha_ppe_get_entry_idle_time(ppe, e->data.ib1); 1018 struct airoha_flow_table_entry *iter; 1019 struct hlist_node *n; 1020 1021 lockdep_assert_held(&ppe_lock); 1022 1023 hlist_for_each_entry_safe(iter, n, &e->l2_flows, l2_subflow_node) { 1024 struct airoha_foe_entry *hwe; 1025 u32 ib1, state; 1026 int idle; 1027 1028 hwe = airoha_ppe_foe_get_entry_locked(ppe, iter->hash); 1029 if (!hwe) 1030 continue; 1031 1032 ib1 = READ_ONCE(hwe->ib1); 1033 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1); 1034 if (state != AIROHA_FOE_STATE_BIND) { 1035 iter->hash = 0xffff; 1036 airoha_ppe_foe_remove_flow(ppe, iter); 1037 continue; 1038 } 1039 1040 idle = airoha_ppe_get_entry_idle_time(ppe, ib1); 1041 if (idle >= min_idle) 1042 continue; 1043 1044 min_idle = idle; 1045 e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP; 1046 e->data.ib1 |= ib1 & AIROHA_FOE_IB1_BIND_TIMESTAMP; 1047 } 1048 } 1049 1050 static void airoha_ppe_foe_flow_entry_update(struct airoha_ppe *ppe, 1051 struct airoha_flow_table_entry *e) 1052 { 1053 struct airoha_foe_entry *hwe_p, hwe = {}; 1054 1055 spin_lock_bh(&ppe_lock); 1056 1057 if (e->type == FLOW_TYPE_L2) { 1058 airoha_ppe_foe_flow_l2_entry_update(ppe, e); 1059 goto unlock; 1060 } 1061 1062 if (e->hash == 0xffff) 1063 goto unlock; 1064 1065 hwe_p = airoha_ppe_foe_get_entry_locked(ppe, e->hash); 1066 if (!hwe_p) 1067 goto unlock; 1068 1069 memcpy(&hwe, hwe_p, sizeof(*hwe_p)); 1070 if (!airoha_ppe_foe_compare_entry(e, &hwe)) { 1071 e->hash = 0xffff; 1072 goto unlock; 1073 } 1074 1075 e->data.ib1 = hwe.ib1; 1076 unlock: 1077 spin_unlock_bh(&ppe_lock); 1078 } 1079 1080 static int airoha_ppe_entry_idle_time(struct airoha_ppe *ppe, 1081 struct airoha_flow_table_entry *e) 1082 { 1083 airoha_ppe_foe_flow_entry_update(ppe, e); 1084 1085 return airoha_ppe_get_entry_idle_time(ppe, e->data.ib1); 1086 } 1087 1088 static int airoha_ppe_flow_offload_replace(struct airoha_eth *eth, 1089 struct flow_cls_offload *f) 1090 { 1091 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 1092 struct airoha_flow_table_entry *e; 1093 struct airoha_flow_data data = {}; 1094 struct net_device *odev = NULL; 1095 struct flow_action_entry *act; 1096 u8 l4proto = 0, priority = 0; 1097 struct airoha_foe_entry hwe; 1098 int err, i, offload_type; 1099 u16 addr_type = 0; 1100 1101 if (rhashtable_lookup(ð->flow_table, &f->cookie, 1102 airoha_flow_table_params)) 1103 return -EEXIST; 1104 1105 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) 1106 return -EOPNOTSUPP; 1107 1108 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 1109 struct flow_match_control match; 1110 1111 flow_rule_match_control(rule, &match); 1112 addr_type = match.key->addr_type; 1113 if (flow_rule_has_control_flags(match.mask->flags, 1114 f->common.extack)) 1115 return -EOPNOTSUPP; 1116 } else { 1117 return -EOPNOTSUPP; 1118 } 1119 1120 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 1121 struct flow_match_basic match; 1122 1123 flow_rule_match_basic(rule, &match); 1124 l4proto = match.key->ip_proto; 1125 } else { 1126 return -EOPNOTSUPP; 1127 } 1128 1129 switch (addr_type) { 1130 case 0: 1131 offload_type = PPE_PKT_TYPE_BRIDGE; 1132 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 1133 struct flow_match_eth_addrs match; 1134 1135 flow_rule_match_eth_addrs(rule, &match); 1136 memcpy(data.eth.h_dest, match.key->dst, ETH_ALEN); 1137 memcpy(data.eth.h_source, match.key->src, ETH_ALEN); 1138 } else { 1139 return -EOPNOTSUPP; 1140 } 1141 break; 1142 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1143 offload_type = PPE_PKT_TYPE_IPV4_HNAPT; 1144 break; 1145 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1146 offload_type = PPE_PKT_TYPE_IPV6_ROUTE_5T; 1147 break; 1148 default: 1149 return -EOPNOTSUPP; 1150 } 1151 1152 flow_action_for_each(i, act, &rule->action) { 1153 switch (act->id) { 1154 case FLOW_ACTION_MANGLE: 1155 if (offload_type == PPE_PKT_TYPE_BRIDGE) 1156 return -EOPNOTSUPP; 1157 1158 if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH) 1159 airoha_ppe_flow_mangle_eth(act, &data.eth); 1160 break; 1161 case FLOW_ACTION_REDIRECT: 1162 odev = act->dev; 1163 break; 1164 case FLOW_ACTION_CSUM: 1165 break; 1166 case FLOW_ACTION_VLAN_PUSH: 1167 if (data.vlan.num == 2 || 1168 act->vlan.proto != htons(ETH_P_8021Q)) 1169 return -EOPNOTSUPP; 1170 1171 data.vlan.hdr[data.vlan.num].id = act->vlan.vid; 1172 data.vlan.hdr[data.vlan.num].proto = act->vlan.proto; 1173 data.vlan.num++; 1174 break; 1175 case FLOW_ACTION_VLAN_POP: 1176 break; 1177 case FLOW_ACTION_PPPOE_PUSH: 1178 if (data.pppoe.num == 1 || data.vlan.num == 2) 1179 return -EOPNOTSUPP; 1180 1181 data.pppoe.sid = act->pppoe.sid; 1182 data.pppoe.num++; 1183 break; 1184 default: 1185 return -EOPNOTSUPP; 1186 } 1187 } 1188 1189 if (!is_valid_ether_addr(data.eth.h_source) || 1190 !is_valid_ether_addr(data.eth.h_dest)) 1191 return -EINVAL; 1192 1193 err = airoha_ppe_foe_entry_prepare(eth, &hwe, odev, offload_type, 1194 &data, l4proto, priority); 1195 if (err) 1196 return err; 1197 1198 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 1199 struct flow_match_ports ports; 1200 1201 if (offload_type == PPE_PKT_TYPE_BRIDGE) 1202 return -EOPNOTSUPP; 1203 1204 flow_rule_match_ports(rule, &ports); 1205 data.src_port = ports.key->src; 1206 data.dst_port = ports.key->dst; 1207 } else if (offload_type != PPE_PKT_TYPE_BRIDGE) { 1208 return -EOPNOTSUPP; 1209 } 1210 1211 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1212 struct flow_match_ipv4_addrs addrs; 1213 1214 flow_rule_match_ipv4_addrs(rule, &addrs); 1215 data.v4.src_addr = addrs.key->src; 1216 data.v4.dst_addr = addrs.key->dst; 1217 airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, false); 1218 } 1219 1220 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 1221 struct flow_match_ipv6_addrs addrs; 1222 1223 flow_rule_match_ipv6_addrs(rule, &addrs); 1224 1225 data.v6.src_addr = addrs.key->src; 1226 data.v6.dst_addr = addrs.key->dst; 1227 airoha_ppe_foe_entry_set_ipv6_tuple(&hwe, &data); 1228 } 1229 1230 flow_action_for_each(i, act, &rule->action) { 1231 if (act->id != FLOW_ACTION_MANGLE) 1232 continue; 1233 1234 if (offload_type == PPE_PKT_TYPE_BRIDGE) 1235 return -EOPNOTSUPP; 1236 1237 switch (act->mangle.htype) { 1238 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 1239 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 1240 err = airoha_ppe_flow_mangle_ports(act, &data); 1241 break; 1242 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 1243 err = airoha_ppe_flow_mangle_ipv4(act, &data); 1244 break; 1245 case FLOW_ACT_MANGLE_HDR_TYPE_ETH: 1246 /* handled earlier */ 1247 break; 1248 default: 1249 return -EOPNOTSUPP; 1250 } 1251 1252 if (err) 1253 return err; 1254 } 1255 1256 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1257 err = airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, true); 1258 if (err) 1259 return err; 1260 } 1261 1262 e = kzalloc_obj(*e); 1263 if (!e) 1264 return -ENOMEM; 1265 1266 e->cookie = f->cookie; 1267 memcpy(&e->data, &hwe, sizeof(e->data)); 1268 1269 err = airoha_ppe_foe_flow_commit_entry(eth->ppe, e); 1270 if (err) 1271 goto free_entry; 1272 1273 err = rhashtable_insert_fast(ð->flow_table, &e->node, 1274 airoha_flow_table_params); 1275 if (err < 0) 1276 goto remove_foe_entry; 1277 1278 return 0; 1279 1280 remove_foe_entry: 1281 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 1282 free_entry: 1283 kfree(e); 1284 1285 return err; 1286 } 1287 1288 static int airoha_ppe_flow_offload_destroy(struct airoha_eth *eth, 1289 struct flow_cls_offload *f) 1290 { 1291 struct airoha_flow_table_entry *e; 1292 1293 e = rhashtable_lookup(ð->flow_table, &f->cookie, 1294 airoha_flow_table_params); 1295 if (!e) 1296 return -ENOENT; 1297 1298 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 1299 rhashtable_remove_fast(ð->flow_table, &e->node, 1300 airoha_flow_table_params); 1301 kfree(e); 1302 1303 return 0; 1304 } 1305 1306 void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 hash, 1307 struct airoha_foe_stats64 *stats) 1308 { 1309 struct airoha_eth *eth = ppe->eth; 1310 int ppe_num_stats_entries; 1311 struct airoha_npu *npu; 1312 u32 index; 1313 1314 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 1315 if (ppe_num_stats_entries < 0) 1316 return; 1317 1318 if (airoha_ppe_foe_get_flow_stats_index(ppe, hash, &index)) 1319 return; 1320 1321 if (index >= ppe_num_stats_entries) 1322 return; 1323 1324 rcu_read_lock(); 1325 1326 npu = rcu_dereference(eth->npu); 1327 if (npu) { 1328 u64 packets = ppe->foe_stats[index].packets; 1329 u64 bytes = ppe->foe_stats[index].bytes; 1330 struct airoha_foe_stats npu_stats; 1331 1332 memcpy_fromio(&npu_stats, &npu->stats[index], 1333 sizeof(*npu->stats)); 1334 stats->packets = packets << 32 | npu_stats.packets; 1335 stats->bytes = bytes << 32 | npu_stats.bytes; 1336 } 1337 1338 rcu_read_unlock(); 1339 } 1340 1341 static int airoha_ppe_flow_offload_stats(struct airoha_eth *eth, 1342 struct flow_cls_offload *f) 1343 { 1344 struct airoha_flow_table_entry *e; 1345 u32 idle; 1346 1347 e = rhashtable_lookup(ð->flow_table, &f->cookie, 1348 airoha_flow_table_params); 1349 if (!e) 1350 return -ENOENT; 1351 1352 idle = airoha_ppe_entry_idle_time(eth->ppe, e); 1353 f->stats.lastused = jiffies - idle * HZ; 1354 1355 if (e->hash != 0xffff) { 1356 struct airoha_foe_stats64 stats = {}; 1357 1358 airoha_ppe_foe_entry_get_stats(eth->ppe, e->hash, &stats); 1359 f->stats.pkts += (stats.packets - e->stats.packets); 1360 f->stats.bytes += (stats.bytes - e->stats.bytes); 1361 e->stats = stats; 1362 } 1363 1364 return 0; 1365 } 1366 1367 static int airoha_ppe_flow_offload_cmd(struct airoha_eth *eth, 1368 struct flow_cls_offload *f) 1369 { 1370 switch (f->command) { 1371 case FLOW_CLS_REPLACE: 1372 return airoha_ppe_flow_offload_replace(eth, f); 1373 case FLOW_CLS_DESTROY: 1374 return airoha_ppe_flow_offload_destroy(eth, f); 1375 case FLOW_CLS_STATS: 1376 return airoha_ppe_flow_offload_stats(eth, f); 1377 default: 1378 break; 1379 } 1380 1381 return -EOPNOTSUPP; 1382 } 1383 1384 static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe) 1385 { 1386 u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe); 1387 struct airoha_foe_entry *hwe = ppe->foe; 1388 int i; 1389 1390 for (i = 0; i < sram_num_entries; i++) { 1391 int err; 1392 1393 memset(&hwe[i], 0, sizeof(*hwe)); 1394 err = airoha_ppe_foe_commit_sram_entry(ppe, i); 1395 if (err) 1396 return err; 1397 } 1398 1399 return 0; 1400 } 1401 1402 static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth) 1403 { 1404 struct airoha_npu *npu = airoha_npu_get(eth->dev); 1405 1406 if (IS_ERR(npu)) { 1407 request_module("airoha-npu"); 1408 npu = airoha_npu_get(eth->dev); 1409 } 1410 1411 return npu; 1412 } 1413 1414 static int airoha_ppe_wait_for_npu_init(struct airoha_eth *eth) 1415 { 1416 int err; 1417 u32 val; 1418 1419 /* PPE_FLOW_CFG default register value is 0. Since we reset FE 1420 * during the device probe we can just check the configured value 1421 * is not 0 here. 1422 */ 1423 err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC, 1424 100 * USEC_PER_MSEC, false, eth, 1425 REG_PPE_PPE_FLOW_CFG(0)); 1426 if (err) 1427 return err; 1428 1429 if (airoha_ppe_is_enabled(eth, 1)) 1430 err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC, 1431 100 * USEC_PER_MSEC, false, eth, 1432 REG_PPE_PPE_FLOW_CFG(1)); 1433 1434 return err; 1435 } 1436 1437 static int airoha_ppe_offload_setup(struct airoha_eth *eth) 1438 { 1439 struct airoha_npu *npu = airoha_ppe_npu_get(eth); 1440 struct airoha_ppe *ppe = eth->ppe; 1441 int err, ppe_num_stats_entries; 1442 1443 if (IS_ERR(npu)) 1444 return PTR_ERR(npu); 1445 1446 err = npu->ops.ppe_init(npu); 1447 if (err) 1448 goto error_npu_put; 1449 1450 /* Wait for NPU PPE configuration to complete */ 1451 err = airoha_ppe_wait_for_npu_init(eth); 1452 if (err) 1453 goto error_npu_put; 1454 1455 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 1456 if (ppe_num_stats_entries > 0) { 1457 err = npu->ops.ppe_init_stats(npu, ppe->foe_stats_dma, 1458 ppe_num_stats_entries); 1459 if (err) 1460 goto error_npu_put; 1461 } 1462 1463 airoha_ppe_hw_init(ppe); 1464 airoha_ppe_foe_flow_stats_reset(ppe, npu); 1465 1466 rcu_assign_pointer(eth->npu, npu); 1467 synchronize_rcu(); 1468 1469 return 0; 1470 1471 error_npu_put: 1472 airoha_npu_put(npu); 1473 1474 return err; 1475 } 1476 1477 int airoha_ppe_setup_tc_block_cb(struct airoha_ppe_dev *dev, void *type_data) 1478 { 1479 struct airoha_ppe *ppe = dev->priv; 1480 struct airoha_eth *eth = ppe->eth; 1481 int err = 0; 1482 1483 /* Netfilter flowtable can try to offload flower rules while not all 1484 * the net_devices are registered or initialized. Delay offloading 1485 * until all net_devices are registered in the system. 1486 */ 1487 if (!test_bit(DEV_STATE_REGISTERED, ð->state)) 1488 return -EBUSY; 1489 1490 mutex_lock(&flow_offload_mutex); 1491 1492 if (!eth->npu) 1493 err = airoha_ppe_offload_setup(eth); 1494 if (!err) 1495 err = airoha_ppe_flow_offload_cmd(eth, type_data); 1496 1497 mutex_unlock(&flow_offload_mutex); 1498 1499 return err; 1500 } 1501 1502 void airoha_ppe_check_skb(struct airoha_ppe_dev *dev, struct sk_buff *skb, 1503 u16 hash, bool rx_wlan) 1504 { 1505 struct airoha_ppe *ppe = dev->priv; 1506 u32 ppe_hash_mask = airoha_ppe_get_total_num_entries(ppe) - 1; 1507 u16 now, diff; 1508 1509 if (hash > ppe_hash_mask) 1510 return; 1511 1512 now = (u16)jiffies; 1513 diff = now - ppe->foe_check_time[hash]; 1514 if (diff < HZ / 10) 1515 return; 1516 1517 ppe->foe_check_time[hash] = now; 1518 airoha_ppe_foe_insert_entry(ppe, skb, hash, rx_wlan); 1519 } 1520 1521 void airoha_ppe_init_upd_mem(struct airoha_gdm_dev *dev, const u8 *addr) 1522 { 1523 struct airoha_gdm_port *port = dev->port; 1524 struct airoha_eth *eth = dev->eth; 1525 u32 val; 1526 1527 val = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) | addr[5]; 1528 airoha_fe_wr(eth, REG_UPDMEM_DATA(0), val); 1529 airoha_fe_wr(eth, REG_UPDMEM_CTRL(0), 1530 FIELD_PREP(PPE_UPDMEM_ADDR_MASK, port->id) | 1531 PPE_UPDMEM_WR_MASK | PPE_UPDMEM_REQ_MASK); 1532 1533 val = (addr[0] << 8) | addr[1]; 1534 airoha_fe_wr(eth, REG_UPDMEM_DATA(0), val); 1535 airoha_fe_wr(eth, REG_UPDMEM_CTRL(0), 1536 FIELD_PREP(PPE_UPDMEM_ADDR_MASK, port->id) | 1537 FIELD_PREP(PPE_UPDMEM_OFFSET_MASK, 1) | 1538 PPE_UPDMEM_WR_MASK | PPE_UPDMEM_REQ_MASK); 1539 } 1540 1541 struct airoha_ppe_dev *airoha_ppe_get_dev(struct device *dev) 1542 { 1543 struct platform_device *pdev; 1544 struct device_node *np; 1545 struct airoha_eth *eth; 1546 1547 np = of_parse_phandle(dev->of_node, "airoha,eth", 0); 1548 if (!np) 1549 return ERR_PTR(-ENODEV); 1550 1551 pdev = of_find_device_by_node(np); 1552 if (!pdev) { 1553 dev_err(dev, "cannot find device node %s\n", np->name); 1554 of_node_put(np); 1555 return ERR_PTR(-ENODEV); 1556 } 1557 of_node_put(np); 1558 1559 if (!try_module_get(THIS_MODULE)) { 1560 dev_err(dev, "failed to get the device driver module\n"); 1561 goto error_pdev_put; 1562 } 1563 1564 eth = platform_get_drvdata(pdev); 1565 if (!eth) 1566 goto error_module_put; 1567 1568 if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER)) { 1569 dev_err(&pdev->dev, 1570 "failed to create device link to consumer %s\n", 1571 dev_name(dev)); 1572 goto error_module_put; 1573 } 1574 1575 return ð->ppe->dev; 1576 1577 error_module_put: 1578 module_put(THIS_MODULE); 1579 error_pdev_put: 1580 platform_device_put(pdev); 1581 1582 return ERR_PTR(-ENODEV); 1583 } 1584 EXPORT_SYMBOL_GPL(airoha_ppe_get_dev); 1585 1586 void airoha_ppe_put_dev(struct airoha_ppe_dev *dev) 1587 { 1588 struct airoha_ppe *ppe = dev->priv; 1589 struct airoha_eth *eth = ppe->eth; 1590 1591 module_put(THIS_MODULE); 1592 put_device(eth->dev); 1593 } 1594 EXPORT_SYMBOL_GPL(airoha_ppe_put_dev); 1595 1596 int airoha_ppe_init(struct airoha_eth *eth) 1597 { 1598 int foe_size, err, ppe_num_stats_entries; 1599 u32 ppe_num_entries; 1600 struct airoha_ppe *ppe; 1601 1602 ppe = devm_kzalloc(eth->dev, sizeof(*ppe), GFP_KERNEL); 1603 if (!ppe) 1604 return -ENOMEM; 1605 1606 ppe->dev.ops.setup_tc_block_cb = airoha_ppe_setup_tc_block_cb; 1607 ppe->dev.ops.check_skb = airoha_ppe_check_skb; 1608 ppe->dev.priv = ppe; 1609 ppe->eth = eth; 1610 eth->ppe = ppe; 1611 1612 ppe_num_entries = airoha_ppe_get_total_num_entries(ppe); 1613 foe_size = ppe_num_entries * sizeof(struct airoha_foe_entry); 1614 ppe->foe = dmam_alloc_coherent(eth->dev, foe_size, &ppe->foe_dma, 1615 GFP_KERNEL); 1616 if (!ppe->foe) 1617 return -ENOMEM; 1618 1619 ppe->foe_flow = devm_kzalloc(eth->dev, 1620 ppe_num_entries * sizeof(*ppe->foe_flow), 1621 GFP_KERNEL); 1622 if (!ppe->foe_flow) 1623 return -ENOMEM; 1624 1625 ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); 1626 if (ppe_num_stats_entries > 0) { 1627 foe_size = ppe_num_stats_entries * sizeof(*ppe->foe_stats); 1628 ppe->foe_stats = dmam_alloc_coherent(eth->dev, foe_size, 1629 &ppe->foe_stats_dma, 1630 GFP_KERNEL); 1631 if (!ppe->foe_stats) 1632 return -ENOMEM; 1633 } 1634 1635 ppe->foe_check_time = devm_kzalloc(eth->dev, 1636 ppe_num_entries * sizeof(*ppe->foe_check_time), 1637 GFP_KERNEL); 1638 if (!ppe->foe_check_time) 1639 return -ENOMEM; 1640 1641 err = airoha_ppe_flush_sram_entries(ppe); 1642 if (err) 1643 return err; 1644 1645 err = rhashtable_init(ð->flow_table, &airoha_flow_table_params); 1646 if (err) 1647 return err; 1648 1649 err = rhashtable_init(&ppe->l2_flows, &airoha_l2_flow_table_params); 1650 if (err) 1651 goto error_flow_table_destroy; 1652 1653 err = airoha_ppe_debugfs_init(ppe); 1654 if (err) 1655 goto error_l2_flow_table_destroy; 1656 1657 return 0; 1658 1659 error_l2_flow_table_destroy: 1660 rhashtable_destroy(&ppe->l2_flows); 1661 error_flow_table_destroy: 1662 rhashtable_destroy(ð->flow_table); 1663 1664 return err; 1665 } 1666 1667 void airoha_ppe_deinit(struct airoha_eth *eth) 1668 { 1669 struct airoha_npu *npu; 1670 1671 mutex_lock(&flow_offload_mutex); 1672 1673 npu = rcu_replace_pointer(eth->npu, NULL, 1674 lockdep_is_held(&flow_offload_mutex)); 1675 if (npu) { 1676 synchronize_rcu(); 1677 npu->ops.ppe_deinit(npu); 1678 airoha_npu_put(npu); 1679 } 1680 1681 mutex_unlock(&flow_offload_mutex); 1682 1683 rhashtable_destroy(ð->ppe->l2_flows); 1684 rhashtable_destroy(ð->flow_table); 1685 debugfs_remove(eth->ppe->debugfs_dir); 1686 } 1687