1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 7 #include <linux/ip.h> 8 #include <linux/ipv6.h> 9 #include <linux/rhashtable.h> 10 #include <net/ipv6.h> 11 #include <net/pkt_cls.h> 12 13 #include "airoha_npu.h" 14 #include "airoha_regs.h" 15 #include "airoha_eth.h" 16 17 static DEFINE_MUTEX(flow_offload_mutex); 18 static DEFINE_SPINLOCK(ppe_lock); 19 20 static const struct rhashtable_params airoha_flow_table_params = { 21 .head_offset = offsetof(struct airoha_flow_table_entry, node), 22 .key_offset = offsetof(struct airoha_flow_table_entry, cookie), 23 .key_len = sizeof(unsigned long), 24 .automatic_shrinking = true, 25 }; 26 27 static const struct rhashtable_params airoha_l2_flow_table_params = { 28 .head_offset = offsetof(struct airoha_flow_table_entry, l2_node), 29 .key_offset = offsetof(struct airoha_flow_table_entry, data.bridge), 30 .key_len = 2 * ETH_ALEN, 31 .automatic_shrinking = true, 32 }; 33 34 static bool airoha_ppe2_is_enabled(struct airoha_eth *eth) 35 { 36 return airoha_fe_rr(eth, REG_PPE_GLO_CFG(1)) & PPE_GLO_CFG_EN_MASK; 37 } 38 39 static u32 airoha_ppe_get_timestamp(struct airoha_ppe *ppe) 40 { 41 u16 timestamp = airoha_fe_rr(ppe->eth, REG_FE_FOE_TS); 42 43 return FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, timestamp); 44 } 45 46 static void airoha_ppe_hw_init(struct airoha_ppe *ppe) 47 { 48 u32 sram_tb_size, sram_num_entries, dram_num_entries; 49 struct airoha_eth *eth = ppe->eth; 50 int i; 51 52 sram_tb_size = PPE_SRAM_NUM_ENTRIES * sizeof(struct airoha_foe_entry); 53 dram_num_entries = PPE_RAM_NUM_ENTRIES_SHIFT(PPE_DRAM_NUM_ENTRIES); 54 55 for (i = 0; i < PPE_NUM; i++) { 56 int p; 57 58 airoha_fe_wr(eth, REG_PPE_TB_BASE(i), 59 ppe->foe_dma + sram_tb_size); 60 61 airoha_fe_rmw(eth, REG_PPE_BND_AGE0(i), 62 PPE_BIND_AGE0_DELTA_NON_L4 | 63 PPE_BIND_AGE0_DELTA_UDP, 64 FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 1) | 65 FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 12)); 66 airoha_fe_rmw(eth, REG_PPE_BND_AGE1(i), 67 PPE_BIND_AGE1_DELTA_TCP_FIN | 68 PPE_BIND_AGE1_DELTA_TCP, 69 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 70 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 7)); 71 72 airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i), 73 PPE_SRAM_TABLE_EN_MASK | 74 PPE_SRAM_HASH1_EN_MASK | 75 PPE_DRAM_TABLE_EN_MASK | 76 PPE_SRAM_HASH0_MODE_MASK | 77 PPE_SRAM_HASH1_MODE_MASK | 78 PPE_DRAM_HASH0_MODE_MASK | 79 PPE_DRAM_HASH1_MODE_MASK, 80 FIELD_PREP(PPE_SRAM_TABLE_EN_MASK, 1) | 81 FIELD_PREP(PPE_SRAM_HASH1_EN_MASK, 1) | 82 FIELD_PREP(PPE_SRAM_HASH1_MODE_MASK, 1) | 83 FIELD_PREP(PPE_DRAM_HASH1_MODE_MASK, 3)); 84 85 airoha_fe_rmw(eth, REG_PPE_TB_CFG(i), 86 PPE_TB_CFG_SEARCH_MISS_MASK | 87 PPE_TB_ENTRY_SIZE_MASK, 88 FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) | 89 FIELD_PREP(PPE_TB_ENTRY_SIZE_MASK, 0)); 90 91 airoha_fe_wr(eth, REG_PPE_HASH_SEED(i), PPE_HASH_SEED); 92 93 for (p = 0; p < ARRAY_SIZE(eth->ports); p++) 94 airoha_fe_rmw(eth, REG_PPE_MTU(i, p), 95 FP0_EGRESS_MTU_MASK | 96 FP1_EGRESS_MTU_MASK, 97 FIELD_PREP(FP0_EGRESS_MTU_MASK, 98 AIROHA_MAX_MTU) | 99 FIELD_PREP(FP1_EGRESS_MTU_MASK, 100 AIROHA_MAX_MTU)); 101 } 102 103 if (airoha_ppe2_is_enabled(eth)) { 104 sram_num_entries = 105 PPE_RAM_NUM_ENTRIES_SHIFT(PPE1_SRAM_NUM_ENTRIES); 106 airoha_fe_rmw(eth, REG_PPE_TB_CFG(0), 107 PPE_SRAM_TB_NUM_ENTRY_MASK | 108 PPE_DRAM_TB_NUM_ENTRY_MASK, 109 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 110 sram_num_entries) | 111 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 112 dram_num_entries)); 113 airoha_fe_rmw(eth, REG_PPE_TB_CFG(1), 114 PPE_SRAM_TB_NUM_ENTRY_MASK | 115 PPE_DRAM_TB_NUM_ENTRY_MASK, 116 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 117 sram_num_entries) | 118 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 119 dram_num_entries)); 120 } else { 121 sram_num_entries = 122 PPE_RAM_NUM_ENTRIES_SHIFT(PPE_SRAM_NUM_ENTRIES); 123 airoha_fe_rmw(eth, REG_PPE_TB_CFG(0), 124 PPE_SRAM_TB_NUM_ENTRY_MASK | 125 PPE_DRAM_TB_NUM_ENTRY_MASK, 126 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 127 sram_num_entries) | 128 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 129 dram_num_entries)); 130 } 131 } 132 133 static void airoha_ppe_flow_mangle_eth(const struct flow_action_entry *act, void *eth) 134 { 135 void *dest = eth + act->mangle.offset; 136 const void *src = &act->mangle.val; 137 138 if (act->mangle.offset > 8) 139 return; 140 141 if (act->mangle.mask == 0xffff) { 142 src += 2; 143 dest += 2; 144 } 145 146 memcpy(dest, src, act->mangle.mask ? 2 : 4); 147 } 148 149 static int airoha_ppe_flow_mangle_ports(const struct flow_action_entry *act, 150 struct airoha_flow_data *data) 151 { 152 u32 val = be32_to_cpu((__force __be32)act->mangle.val); 153 154 switch (act->mangle.offset) { 155 case 0: 156 if ((__force __be32)act->mangle.mask == ~cpu_to_be32(0xffff)) 157 data->dst_port = cpu_to_be16(val); 158 else 159 data->src_port = cpu_to_be16(val >> 16); 160 break; 161 case 2: 162 data->dst_port = cpu_to_be16(val); 163 break; 164 default: 165 return -EINVAL; 166 } 167 168 return 0; 169 } 170 171 static int airoha_ppe_flow_mangle_ipv4(const struct flow_action_entry *act, 172 struct airoha_flow_data *data) 173 { 174 __be32 *dest; 175 176 switch (act->mangle.offset) { 177 case offsetof(struct iphdr, saddr): 178 dest = &data->v4.src_addr; 179 break; 180 case offsetof(struct iphdr, daddr): 181 dest = &data->v4.dst_addr; 182 break; 183 default: 184 return -EINVAL; 185 } 186 187 memcpy(dest, &act->mangle.val, sizeof(u32)); 188 189 return 0; 190 } 191 192 static int airoha_get_dsa_port(struct net_device **dev) 193 { 194 #if IS_ENABLED(CONFIG_NET_DSA) 195 struct dsa_port *dp = dsa_port_from_netdev(*dev); 196 197 if (IS_ERR(dp)) 198 return -ENODEV; 199 200 *dev = dsa_port_to_conduit(dp); 201 return dp->index; 202 #else 203 return -ENODEV; 204 #endif 205 } 206 207 static void airoha_ppe_foe_set_bridge_addrs(struct airoha_foe_bridge *br, 208 struct ethhdr *eh) 209 { 210 br->dest_mac_hi = get_unaligned_be32(eh->h_dest); 211 br->dest_mac_lo = get_unaligned_be16(eh->h_dest + 4); 212 br->src_mac_hi = get_unaligned_be16(eh->h_source); 213 br->src_mac_lo = get_unaligned_be32(eh->h_source + 2); 214 } 215 216 static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth, 217 struct airoha_foe_entry *hwe, 218 struct net_device *dev, int type, 219 struct airoha_flow_data *data, 220 int l4proto) 221 { 222 int dsa_port = airoha_get_dsa_port(&dev); 223 struct airoha_foe_mac_info_common *l2; 224 u32 qdata, ports_pad, val; 225 226 memset(hwe, 0, sizeof(*hwe)); 227 228 val = FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, AIROHA_FOE_STATE_BIND) | 229 FIELD_PREP(AIROHA_FOE_IB1_BIND_PACKET_TYPE, type) | 230 FIELD_PREP(AIROHA_FOE_IB1_BIND_UDP, l4proto == IPPROTO_UDP) | 231 FIELD_PREP(AIROHA_FOE_IB1_BIND_VLAN_LAYER, data->vlan.num) | 232 FIELD_PREP(AIROHA_FOE_IB1_BIND_VPM, data->vlan.num) | 233 AIROHA_FOE_IB1_BIND_TTL; 234 hwe->ib1 = val; 235 236 val = FIELD_PREP(AIROHA_FOE_IB2_PORT_AG, 0x1f) | 237 AIROHA_FOE_IB2_PSE_QOS; 238 if (dsa_port >= 0) 239 val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, dsa_port); 240 241 if (dev) { 242 struct airoha_gdm_port *port = netdev_priv(dev); 243 u8 pse_port; 244 245 if (!airoha_is_valid_gdm_port(eth, port)) 246 return -EINVAL; 247 248 if (dsa_port >= 0) 249 pse_port = port->id == 4 ? FE_PSE_PORT_GDM4 : port->id; 250 else 251 pse_port = 2; /* uplink relies on GDM2 loopback */ 252 val |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, pse_port); 253 } 254 255 if (is_multicast_ether_addr(data->eth.h_dest)) 256 val |= AIROHA_FOE_IB2_MULTICAST; 257 258 ports_pad = 0xa5a5a500 | (l4proto & 0xff); 259 if (type == PPE_PKT_TYPE_IPV4_ROUTE) 260 hwe->ipv4.orig_tuple.ports = ports_pad; 261 if (type == PPE_PKT_TYPE_IPV6_ROUTE_3T) 262 hwe->ipv6.ports = ports_pad; 263 264 qdata = FIELD_PREP(AIROHA_FOE_SHAPER_ID, 0x7f); 265 if (type == PPE_PKT_TYPE_BRIDGE) { 266 airoha_ppe_foe_set_bridge_addrs(&hwe->bridge, &data->eth); 267 hwe->bridge.data = qdata; 268 hwe->bridge.ib2 = val; 269 l2 = &hwe->bridge.l2.common; 270 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 271 hwe->ipv6.data = qdata; 272 hwe->ipv6.ib2 = val; 273 l2 = &hwe->ipv6.l2; 274 } else { 275 hwe->ipv4.data = qdata; 276 hwe->ipv4.ib2 = val; 277 l2 = &hwe->ipv4.l2.common; 278 } 279 280 l2->dest_mac_hi = get_unaligned_be32(data->eth.h_dest); 281 l2->dest_mac_lo = get_unaligned_be16(data->eth.h_dest + 4); 282 if (type <= PPE_PKT_TYPE_IPV4_DSLITE) { 283 l2->src_mac_hi = get_unaligned_be32(data->eth.h_source); 284 hwe->ipv4.l2.src_mac_lo = 285 get_unaligned_be16(data->eth.h_source + 4); 286 } else { 287 l2->src_mac_hi = FIELD_PREP(AIROHA_FOE_MAC_SMAC_ID, 0xf); 288 } 289 290 if (data->vlan.num) { 291 l2->etype = dsa_port >= 0 ? BIT(dsa_port) : 0; 292 l2->vlan1 = data->vlan.hdr[0].id; 293 if (data->vlan.num == 2) 294 l2->vlan2 = data->vlan.hdr[1].id; 295 } else if (dsa_port >= 0) { 296 l2->etype = BIT(15) | BIT(dsa_port); 297 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 298 l2->etype = ETH_P_IPV6; 299 } else { 300 l2->etype = ETH_P_IP; 301 } 302 303 return 0; 304 } 305 306 static int airoha_ppe_foe_entry_set_ipv4_tuple(struct airoha_foe_entry *hwe, 307 struct airoha_flow_data *data, 308 bool egress) 309 { 310 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 311 struct airoha_foe_ipv4_tuple *t; 312 313 switch (type) { 314 case PPE_PKT_TYPE_IPV4_HNAPT: 315 if (egress) { 316 t = &hwe->ipv4.new_tuple; 317 break; 318 } 319 fallthrough; 320 case PPE_PKT_TYPE_IPV4_DSLITE: 321 case PPE_PKT_TYPE_IPV4_ROUTE: 322 t = &hwe->ipv4.orig_tuple; 323 break; 324 default: 325 WARN_ON_ONCE(1); 326 return -EINVAL; 327 } 328 329 t->src_ip = be32_to_cpu(data->v4.src_addr); 330 t->dest_ip = be32_to_cpu(data->v4.dst_addr); 331 332 if (type != PPE_PKT_TYPE_IPV4_ROUTE) { 333 t->src_port = be16_to_cpu(data->src_port); 334 t->dest_port = be16_to_cpu(data->dst_port); 335 } 336 337 return 0; 338 } 339 340 static int airoha_ppe_foe_entry_set_ipv6_tuple(struct airoha_foe_entry *hwe, 341 struct airoha_flow_data *data) 342 343 { 344 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 345 u32 *src, *dest; 346 347 switch (type) { 348 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 349 case PPE_PKT_TYPE_IPV6_6RD: 350 hwe->ipv6.src_port = be16_to_cpu(data->src_port); 351 hwe->ipv6.dest_port = be16_to_cpu(data->dst_port); 352 fallthrough; 353 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 354 src = hwe->ipv6.src_ip; 355 dest = hwe->ipv6.dest_ip; 356 break; 357 default: 358 WARN_ON_ONCE(1); 359 return -EINVAL; 360 } 361 362 ipv6_addr_be32_to_cpu(src, data->v6.src_addr.s6_addr32); 363 ipv6_addr_be32_to_cpu(dest, data->v6.dst_addr.s6_addr32); 364 365 return 0; 366 } 367 368 static u32 airoha_ppe_foe_get_entry_hash(struct airoha_foe_entry *hwe) 369 { 370 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 371 u32 hash, hv1, hv2, hv3; 372 373 switch (type) { 374 case PPE_PKT_TYPE_IPV4_ROUTE: 375 case PPE_PKT_TYPE_IPV4_HNAPT: 376 hv1 = hwe->ipv4.orig_tuple.ports; 377 hv2 = hwe->ipv4.orig_tuple.dest_ip; 378 hv3 = hwe->ipv4.orig_tuple.src_ip; 379 break; 380 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 381 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 382 hv1 = hwe->ipv6.src_ip[3] ^ hwe->ipv6.dest_ip[3]; 383 hv1 ^= hwe->ipv6.ports; 384 385 hv2 = hwe->ipv6.src_ip[2] ^ hwe->ipv6.dest_ip[2]; 386 hv2 ^= hwe->ipv6.dest_ip[0]; 387 388 hv3 = hwe->ipv6.src_ip[1] ^ hwe->ipv6.dest_ip[1]; 389 hv3 ^= hwe->ipv6.src_ip[0]; 390 break; 391 case PPE_PKT_TYPE_BRIDGE: { 392 struct airoha_foe_mac_info *l2 = &hwe->bridge.l2; 393 394 hv1 = l2->common.src_mac_hi & 0xffff; 395 hv1 = hv1 << 16 | l2->src_mac_lo; 396 397 hv2 = l2->common.dest_mac_lo; 398 hv2 = hv2 << 16; 399 hv2 = hv2 | ((l2->common.src_mac_hi & 0xffff0000) >> 16); 400 401 hv3 = l2->common.dest_mac_hi; 402 break; 403 } 404 case PPE_PKT_TYPE_IPV4_DSLITE: 405 case PPE_PKT_TYPE_IPV6_6RD: 406 default: 407 WARN_ON_ONCE(1); 408 return PPE_HASH_MASK; 409 } 410 411 hash = (hv1 & hv2) | ((~hv1) & hv3); 412 hash = (hash >> 24) | ((hash & 0xffffff) << 8); 413 hash ^= hv1 ^ hv2 ^ hv3; 414 hash ^= hash >> 16; 415 hash &= PPE_NUM_ENTRIES - 1; 416 417 return hash; 418 } 419 420 struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe, 421 u32 hash) 422 { 423 if (hash < PPE_SRAM_NUM_ENTRIES) { 424 u32 *hwe = ppe->foe + hash * sizeof(struct airoha_foe_entry); 425 struct airoha_eth *eth = ppe->eth; 426 bool ppe2; 427 u32 val; 428 int i; 429 430 ppe2 = airoha_ppe2_is_enabled(ppe->eth) && 431 hash >= PPE1_SRAM_NUM_ENTRIES; 432 airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2), 433 FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) | 434 PPE_SRAM_CTRL_REQ_MASK); 435 if (read_poll_timeout_atomic(airoha_fe_rr, val, 436 val & PPE_SRAM_CTRL_ACK_MASK, 437 10, 100, false, eth, 438 REG_PPE_RAM_CTRL(ppe2))) 439 return NULL; 440 441 for (i = 0; i < sizeof(struct airoha_foe_entry) / 4; i++) 442 hwe[i] = airoha_fe_rr(eth, 443 REG_PPE_RAM_ENTRY(ppe2, i)); 444 } 445 446 return ppe->foe + hash * sizeof(struct airoha_foe_entry); 447 } 448 449 static bool airoha_ppe_foe_compare_entry(struct airoha_flow_table_entry *e, 450 struct airoha_foe_entry *hwe) 451 { 452 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 453 int len; 454 455 if ((hwe->ib1 ^ e->data.ib1) & AIROHA_FOE_IB1_BIND_UDP) 456 return false; 457 458 if (type > PPE_PKT_TYPE_IPV4_DSLITE) 459 len = offsetof(struct airoha_foe_entry, ipv6.data); 460 else 461 len = offsetof(struct airoha_foe_entry, ipv4.ib2); 462 463 return !memcmp(&e->data.d, &hwe->d, len - sizeof(hwe->ib1)); 464 } 465 466 static int airoha_ppe_foe_commit_entry(struct airoha_ppe *ppe, 467 struct airoha_foe_entry *e, 468 u32 hash) 469 { 470 struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe); 471 u32 ts = airoha_ppe_get_timestamp(ppe); 472 struct airoha_eth *eth = ppe->eth; 473 474 memcpy(&hwe->d, &e->d, sizeof(*hwe) - sizeof(hwe->ib1)); 475 wmb(); 476 477 e->ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP; 478 e->ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_TIMESTAMP, ts); 479 hwe->ib1 = e->ib1; 480 481 if (hash < PPE_SRAM_NUM_ENTRIES) { 482 dma_addr_t addr = ppe->foe_dma + hash * sizeof(*hwe); 483 bool ppe2 = airoha_ppe2_is_enabled(eth) && 484 hash >= PPE1_SRAM_NUM_ENTRIES; 485 struct airoha_npu *npu; 486 int err = -ENODEV; 487 488 rcu_read_lock(); 489 npu = rcu_dereference(eth->npu); 490 if (npu) 491 err = npu->ops.ppe_foe_commit_entry(npu, addr, 492 sizeof(*hwe), hash, 493 ppe2); 494 rcu_read_unlock(); 495 496 return err; 497 } 498 499 return 0; 500 } 501 502 static void airoha_ppe_foe_remove_flow(struct airoha_ppe *ppe, 503 struct airoha_flow_table_entry *e) 504 { 505 lockdep_assert_held(&ppe_lock); 506 507 hlist_del_init(&e->list); 508 if (e->hash != 0xffff) { 509 e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_STATE; 510 e->data.ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, 511 AIROHA_FOE_STATE_INVALID); 512 airoha_ppe_foe_commit_entry(ppe, &e->data, e->hash); 513 e->hash = 0xffff; 514 } 515 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 516 hlist_del_init(&e->l2_subflow_node); 517 kfree(e); 518 } 519 } 520 521 static void airoha_ppe_foe_remove_l2_flow(struct airoha_ppe *ppe, 522 struct airoha_flow_table_entry *e) 523 { 524 struct hlist_head *head = &e->l2_flows; 525 struct hlist_node *n; 526 527 lockdep_assert_held(&ppe_lock); 528 529 rhashtable_remove_fast(&ppe->l2_flows, &e->l2_node, 530 airoha_l2_flow_table_params); 531 hlist_for_each_entry_safe(e, n, head, l2_subflow_node) 532 airoha_ppe_foe_remove_flow(ppe, e); 533 } 534 535 static void airoha_ppe_foe_flow_remove_entry(struct airoha_ppe *ppe, 536 struct airoha_flow_table_entry *e) 537 { 538 spin_lock_bh(&ppe_lock); 539 540 if (e->type == FLOW_TYPE_L2) 541 airoha_ppe_foe_remove_l2_flow(ppe, e); 542 else 543 airoha_ppe_foe_remove_flow(ppe, e); 544 545 spin_unlock_bh(&ppe_lock); 546 } 547 548 static int 549 airoha_ppe_foe_commit_subflow_entry(struct airoha_ppe *ppe, 550 struct airoha_flow_table_entry *e, 551 u32 hash) 552 { 553 u32 mask = AIROHA_FOE_IB1_BIND_PACKET_TYPE | AIROHA_FOE_IB1_BIND_UDP; 554 struct airoha_foe_entry *hwe_p, hwe; 555 struct airoha_flow_table_entry *f; 556 struct airoha_foe_mac_info *l2; 557 int type; 558 559 hwe_p = airoha_ppe_foe_get_entry(ppe, hash); 560 if (!hwe_p) 561 return -EINVAL; 562 563 f = kzalloc(sizeof(*f), GFP_ATOMIC); 564 if (!f) 565 return -ENOMEM; 566 567 hlist_add_head(&f->l2_subflow_node, &e->l2_flows); 568 f->type = FLOW_TYPE_L2_SUBFLOW; 569 f->hash = hash; 570 571 memcpy(&hwe, hwe_p, sizeof(*hwe_p)); 572 hwe.ib1 = (hwe.ib1 & mask) | (e->data.ib1 & ~mask); 573 l2 = &hwe.bridge.l2; 574 memcpy(l2, &e->data.bridge.l2, sizeof(*l2)); 575 576 type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe.ib1); 577 if (type == PPE_PKT_TYPE_IPV4_HNAPT) 578 memcpy(&hwe.ipv4.new_tuple, &hwe.ipv4.orig_tuple, 579 sizeof(hwe.ipv4.new_tuple)); 580 else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T && 581 l2->common.etype == ETH_P_IP) 582 l2->common.etype = ETH_P_IPV6; 583 584 hwe.bridge.ib2 = e->data.bridge.ib2; 585 airoha_ppe_foe_commit_entry(ppe, &hwe, hash); 586 587 return 0; 588 } 589 590 static void airoha_ppe_foe_insert_entry(struct airoha_ppe *ppe, 591 struct sk_buff *skb, 592 u32 hash) 593 { 594 struct airoha_flow_table_entry *e; 595 struct airoha_foe_bridge br = {}; 596 struct airoha_foe_entry *hwe; 597 bool commit_done = false; 598 struct hlist_node *n; 599 u32 index, state; 600 601 spin_lock_bh(&ppe_lock); 602 603 hwe = airoha_ppe_foe_get_entry(ppe, hash); 604 if (!hwe) 605 goto unlock; 606 607 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 608 if (state == AIROHA_FOE_STATE_BIND) 609 goto unlock; 610 611 index = airoha_ppe_foe_get_entry_hash(hwe); 612 hlist_for_each_entry_safe(e, n, &ppe->foe_flow[index], list) { 613 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 614 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 615 if (state != AIROHA_FOE_STATE_BIND) { 616 e->hash = 0xffff; 617 airoha_ppe_foe_remove_flow(ppe, e); 618 } 619 continue; 620 } 621 622 if (commit_done || !airoha_ppe_foe_compare_entry(e, hwe)) { 623 e->hash = 0xffff; 624 continue; 625 } 626 627 airoha_ppe_foe_commit_entry(ppe, &e->data, hash); 628 commit_done = true; 629 e->hash = hash; 630 } 631 632 if (commit_done) 633 goto unlock; 634 635 airoha_ppe_foe_set_bridge_addrs(&br, eth_hdr(skb)); 636 e = rhashtable_lookup_fast(&ppe->l2_flows, &br, 637 airoha_l2_flow_table_params); 638 if (e) 639 airoha_ppe_foe_commit_subflow_entry(ppe, e, hash); 640 unlock: 641 spin_unlock_bh(&ppe_lock); 642 } 643 644 static int 645 airoha_ppe_foe_l2_flow_commit_entry(struct airoha_ppe *ppe, 646 struct airoha_flow_table_entry *e) 647 { 648 struct airoha_flow_table_entry *prev; 649 650 e->type = FLOW_TYPE_L2; 651 prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &e->l2_node, 652 airoha_l2_flow_table_params); 653 if (!prev) 654 return 0; 655 656 if (IS_ERR(prev)) 657 return PTR_ERR(prev); 658 659 return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node, 660 &e->l2_node, 661 airoha_l2_flow_table_params); 662 } 663 664 static int airoha_ppe_foe_flow_commit_entry(struct airoha_ppe *ppe, 665 struct airoha_flow_table_entry *e) 666 { 667 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 668 u32 hash; 669 670 if (type == PPE_PKT_TYPE_BRIDGE) 671 return airoha_ppe_foe_l2_flow_commit_entry(ppe, e); 672 673 hash = airoha_ppe_foe_get_entry_hash(&e->data); 674 e->type = FLOW_TYPE_L4; 675 e->hash = 0xffff; 676 677 spin_lock_bh(&ppe_lock); 678 hlist_add_head(&e->list, &ppe->foe_flow[hash]); 679 spin_unlock_bh(&ppe_lock); 680 681 return 0; 682 } 683 684 static int airoha_ppe_flow_offload_replace(struct airoha_gdm_port *port, 685 struct flow_cls_offload *f) 686 { 687 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 688 struct airoha_eth *eth = port->qdma->eth; 689 struct airoha_flow_table_entry *e; 690 struct airoha_flow_data data = {}; 691 struct net_device *odev = NULL; 692 struct flow_action_entry *act; 693 struct airoha_foe_entry hwe; 694 int err, i, offload_type; 695 u16 addr_type = 0; 696 u8 l4proto = 0; 697 698 if (rhashtable_lookup(ð->flow_table, &f->cookie, 699 airoha_flow_table_params)) 700 return -EEXIST; 701 702 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) 703 return -EOPNOTSUPP; 704 705 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 706 struct flow_match_control match; 707 708 flow_rule_match_control(rule, &match); 709 addr_type = match.key->addr_type; 710 if (flow_rule_has_control_flags(match.mask->flags, 711 f->common.extack)) 712 return -EOPNOTSUPP; 713 } else { 714 return -EOPNOTSUPP; 715 } 716 717 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 718 struct flow_match_basic match; 719 720 flow_rule_match_basic(rule, &match); 721 l4proto = match.key->ip_proto; 722 } else { 723 return -EOPNOTSUPP; 724 } 725 726 switch (addr_type) { 727 case 0: 728 offload_type = PPE_PKT_TYPE_BRIDGE; 729 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 730 struct flow_match_eth_addrs match; 731 732 flow_rule_match_eth_addrs(rule, &match); 733 memcpy(data.eth.h_dest, match.key->dst, ETH_ALEN); 734 memcpy(data.eth.h_source, match.key->src, ETH_ALEN); 735 } else { 736 return -EOPNOTSUPP; 737 } 738 break; 739 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 740 offload_type = PPE_PKT_TYPE_IPV4_HNAPT; 741 break; 742 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 743 offload_type = PPE_PKT_TYPE_IPV6_ROUTE_5T; 744 break; 745 default: 746 return -EOPNOTSUPP; 747 } 748 749 flow_action_for_each(i, act, &rule->action) { 750 switch (act->id) { 751 case FLOW_ACTION_MANGLE: 752 if (offload_type == PPE_PKT_TYPE_BRIDGE) 753 return -EOPNOTSUPP; 754 755 if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH) 756 airoha_ppe_flow_mangle_eth(act, &data.eth); 757 break; 758 case FLOW_ACTION_REDIRECT: 759 odev = act->dev; 760 break; 761 case FLOW_ACTION_CSUM: 762 break; 763 case FLOW_ACTION_VLAN_PUSH: 764 if (data.vlan.num == 2 || 765 act->vlan.proto != htons(ETH_P_8021Q)) 766 return -EOPNOTSUPP; 767 768 data.vlan.hdr[data.vlan.num].id = act->vlan.vid; 769 data.vlan.hdr[data.vlan.num].proto = act->vlan.proto; 770 data.vlan.num++; 771 break; 772 case FLOW_ACTION_VLAN_POP: 773 break; 774 case FLOW_ACTION_PPPOE_PUSH: 775 break; 776 default: 777 return -EOPNOTSUPP; 778 } 779 } 780 781 if (!is_valid_ether_addr(data.eth.h_source) || 782 !is_valid_ether_addr(data.eth.h_dest)) 783 return -EINVAL; 784 785 err = airoha_ppe_foe_entry_prepare(eth, &hwe, odev, offload_type, 786 &data, l4proto); 787 if (err) 788 return err; 789 790 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 791 struct flow_match_ports ports; 792 793 if (offload_type == PPE_PKT_TYPE_BRIDGE) 794 return -EOPNOTSUPP; 795 796 flow_rule_match_ports(rule, &ports); 797 data.src_port = ports.key->src; 798 data.dst_port = ports.key->dst; 799 } else if (offload_type != PPE_PKT_TYPE_BRIDGE) { 800 return -EOPNOTSUPP; 801 } 802 803 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 804 struct flow_match_ipv4_addrs addrs; 805 806 flow_rule_match_ipv4_addrs(rule, &addrs); 807 data.v4.src_addr = addrs.key->src; 808 data.v4.dst_addr = addrs.key->dst; 809 airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, false); 810 } 811 812 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 813 struct flow_match_ipv6_addrs addrs; 814 815 flow_rule_match_ipv6_addrs(rule, &addrs); 816 817 data.v6.src_addr = addrs.key->src; 818 data.v6.dst_addr = addrs.key->dst; 819 airoha_ppe_foe_entry_set_ipv6_tuple(&hwe, &data); 820 } 821 822 flow_action_for_each(i, act, &rule->action) { 823 if (act->id != FLOW_ACTION_MANGLE) 824 continue; 825 826 if (offload_type == PPE_PKT_TYPE_BRIDGE) 827 return -EOPNOTSUPP; 828 829 switch (act->mangle.htype) { 830 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 831 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 832 err = airoha_ppe_flow_mangle_ports(act, &data); 833 break; 834 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 835 err = airoha_ppe_flow_mangle_ipv4(act, &data); 836 break; 837 case FLOW_ACT_MANGLE_HDR_TYPE_ETH: 838 /* handled earlier */ 839 break; 840 default: 841 return -EOPNOTSUPP; 842 } 843 844 if (err) 845 return err; 846 } 847 848 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 849 err = airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, true); 850 if (err) 851 return err; 852 } 853 854 e = kzalloc(sizeof(*e), GFP_KERNEL); 855 if (!e) 856 return -ENOMEM; 857 858 e->cookie = f->cookie; 859 memcpy(&e->data, &hwe, sizeof(e->data)); 860 861 err = airoha_ppe_foe_flow_commit_entry(eth->ppe, e); 862 if (err) 863 goto free_entry; 864 865 err = rhashtable_insert_fast(ð->flow_table, &e->node, 866 airoha_flow_table_params); 867 if (err < 0) 868 goto remove_foe_entry; 869 870 return 0; 871 872 remove_foe_entry: 873 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 874 free_entry: 875 kfree(e); 876 877 return err; 878 } 879 880 static int airoha_ppe_flow_offload_destroy(struct airoha_gdm_port *port, 881 struct flow_cls_offload *f) 882 { 883 struct airoha_eth *eth = port->qdma->eth; 884 struct airoha_flow_table_entry *e; 885 886 e = rhashtable_lookup(ð->flow_table, &f->cookie, 887 airoha_flow_table_params); 888 if (!e) 889 return -ENOENT; 890 891 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 892 rhashtable_remove_fast(ð->flow_table, &e->node, 893 airoha_flow_table_params); 894 kfree(e); 895 896 return 0; 897 } 898 899 static int airoha_ppe_flow_offload_cmd(struct airoha_gdm_port *port, 900 struct flow_cls_offload *f) 901 { 902 switch (f->command) { 903 case FLOW_CLS_REPLACE: 904 return airoha_ppe_flow_offload_replace(port, f); 905 case FLOW_CLS_DESTROY: 906 return airoha_ppe_flow_offload_destroy(port, f); 907 default: 908 break; 909 } 910 911 return -EOPNOTSUPP; 912 } 913 914 static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe, 915 struct airoha_npu *npu) 916 { 917 int i, sram_num_entries = PPE_SRAM_NUM_ENTRIES; 918 struct airoha_foe_entry *hwe = ppe->foe; 919 920 if (airoha_ppe2_is_enabled(ppe->eth)) 921 sram_num_entries = sram_num_entries / 2; 922 923 for (i = 0; i < sram_num_entries; i++) 924 memset(&hwe[i], 0, sizeof(*hwe)); 925 926 return npu->ops.ppe_flush_sram_entries(npu, ppe->foe_dma, 927 PPE_SRAM_NUM_ENTRIES); 928 } 929 930 static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth) 931 { 932 struct airoha_npu *npu = airoha_npu_get(eth->dev); 933 934 if (IS_ERR(npu)) { 935 request_module("airoha-npu"); 936 npu = airoha_npu_get(eth->dev); 937 } 938 939 return npu; 940 } 941 942 static int airoha_ppe_offload_setup(struct airoha_eth *eth) 943 { 944 struct airoha_npu *npu = airoha_ppe_npu_get(eth); 945 int err; 946 947 if (IS_ERR(npu)) 948 return PTR_ERR(npu); 949 950 err = npu->ops.ppe_init(npu); 951 if (err) 952 goto error_npu_put; 953 954 airoha_ppe_hw_init(eth->ppe); 955 err = airoha_ppe_flush_sram_entries(eth->ppe, npu); 956 if (err) 957 goto error_npu_put; 958 959 rcu_assign_pointer(eth->npu, npu); 960 synchronize_rcu(); 961 962 return 0; 963 964 error_npu_put: 965 airoha_npu_put(npu); 966 967 return err; 968 } 969 970 int airoha_ppe_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 971 void *cb_priv) 972 { 973 struct flow_cls_offload *cls = type_data; 974 struct net_device *dev = cb_priv; 975 struct airoha_gdm_port *port = netdev_priv(dev); 976 struct airoha_eth *eth = port->qdma->eth; 977 int err = 0; 978 979 if (!tc_can_offload(dev) || type != TC_SETUP_CLSFLOWER) 980 return -EOPNOTSUPP; 981 982 mutex_lock(&flow_offload_mutex); 983 984 if (!eth->npu) 985 err = airoha_ppe_offload_setup(eth); 986 if (!err) 987 err = airoha_ppe_flow_offload_cmd(port, cls); 988 989 mutex_unlock(&flow_offload_mutex); 990 991 return err; 992 } 993 994 void airoha_ppe_check_skb(struct airoha_ppe *ppe, struct sk_buff *skb, 995 u16 hash) 996 { 997 u16 now, diff; 998 999 if (hash > PPE_HASH_MASK) 1000 return; 1001 1002 now = (u16)jiffies; 1003 diff = now - ppe->foe_check_time[hash]; 1004 if (diff < HZ / 10) 1005 return; 1006 1007 ppe->foe_check_time[hash] = now; 1008 airoha_ppe_foe_insert_entry(ppe, skb, hash); 1009 } 1010 1011 int airoha_ppe_init(struct airoha_eth *eth) 1012 { 1013 struct airoha_ppe *ppe; 1014 int foe_size, err; 1015 1016 ppe = devm_kzalloc(eth->dev, sizeof(*ppe), GFP_KERNEL); 1017 if (!ppe) 1018 return -ENOMEM; 1019 1020 foe_size = PPE_NUM_ENTRIES * sizeof(struct airoha_foe_entry); 1021 ppe->foe = dmam_alloc_coherent(eth->dev, foe_size, &ppe->foe_dma, 1022 GFP_KERNEL); 1023 if (!ppe->foe) 1024 return -ENOMEM; 1025 1026 ppe->eth = eth; 1027 eth->ppe = ppe; 1028 1029 ppe->foe_flow = devm_kzalloc(eth->dev, 1030 PPE_NUM_ENTRIES * sizeof(*ppe->foe_flow), 1031 GFP_KERNEL); 1032 if (!ppe->foe_flow) 1033 return -ENOMEM; 1034 1035 err = rhashtable_init(ð->flow_table, &airoha_flow_table_params); 1036 if (err) 1037 return err; 1038 1039 err = rhashtable_init(&ppe->l2_flows, &airoha_l2_flow_table_params); 1040 if (err) 1041 goto error_flow_table_destroy; 1042 1043 err = airoha_ppe_debugfs_init(ppe); 1044 if (err) 1045 goto error_l2_flow_table_destroy; 1046 1047 return 0; 1048 1049 error_l2_flow_table_destroy: 1050 rhashtable_destroy(&ppe->l2_flows); 1051 error_flow_table_destroy: 1052 rhashtable_destroy(ð->flow_table); 1053 1054 return err; 1055 } 1056 1057 void airoha_ppe_deinit(struct airoha_eth *eth) 1058 { 1059 struct airoha_npu *npu; 1060 1061 rcu_read_lock(); 1062 npu = rcu_dereference(eth->npu); 1063 if (npu) { 1064 npu->ops.ppe_deinit(npu); 1065 airoha_npu_put(npu); 1066 } 1067 rcu_read_unlock(); 1068 1069 rhashtable_destroy(ð->ppe->l2_flows); 1070 rhashtable_destroy(ð->flow_table); 1071 debugfs_remove(eth->ppe->debugfs_dir); 1072 } 1073