1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 7 #include <linux/ip.h> 8 #include <linux/ipv6.h> 9 #include <linux/rhashtable.h> 10 #include <net/ipv6.h> 11 #include <net/pkt_cls.h> 12 13 #include "airoha_npu.h" 14 #include "airoha_regs.h" 15 #include "airoha_eth.h" 16 17 static DEFINE_MUTEX(flow_offload_mutex); 18 static DEFINE_SPINLOCK(ppe_lock); 19 20 static const struct rhashtable_params airoha_flow_table_params = { 21 .head_offset = offsetof(struct airoha_flow_table_entry, node), 22 .key_offset = offsetof(struct airoha_flow_table_entry, cookie), 23 .key_len = sizeof(unsigned long), 24 .automatic_shrinking = true, 25 }; 26 27 static const struct rhashtable_params airoha_l2_flow_table_params = { 28 .head_offset = offsetof(struct airoha_flow_table_entry, l2_node), 29 .key_offset = offsetof(struct airoha_flow_table_entry, data.bridge), 30 .key_len = 2 * ETH_ALEN, 31 .automatic_shrinking = true, 32 }; 33 34 static bool airoha_ppe2_is_enabled(struct airoha_eth *eth) 35 { 36 return airoha_fe_rr(eth, REG_PPE_GLO_CFG(1)) & PPE_GLO_CFG_EN_MASK; 37 } 38 39 static u32 airoha_ppe_get_timestamp(struct airoha_ppe *ppe) 40 { 41 u16 timestamp = airoha_fe_rr(ppe->eth, REG_FE_FOE_TS); 42 43 return FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, timestamp); 44 } 45 46 static void airoha_ppe_hw_init(struct airoha_ppe *ppe) 47 { 48 u32 sram_tb_size, sram_num_entries, dram_num_entries; 49 struct airoha_eth *eth = ppe->eth; 50 int i; 51 52 sram_tb_size = PPE_SRAM_NUM_ENTRIES * sizeof(struct airoha_foe_entry); 53 dram_num_entries = PPE_RAM_NUM_ENTRIES_SHIFT(PPE_DRAM_NUM_ENTRIES); 54 55 for (i = 0; i < PPE_NUM; i++) { 56 int p; 57 58 airoha_fe_wr(eth, REG_PPE_TB_BASE(i), 59 ppe->foe_dma + sram_tb_size); 60 61 airoha_fe_rmw(eth, REG_PPE_BND_AGE0(i), 62 PPE_BIND_AGE0_DELTA_NON_L4 | 63 PPE_BIND_AGE0_DELTA_UDP, 64 FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 1) | 65 FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 12)); 66 airoha_fe_rmw(eth, REG_PPE_BND_AGE1(i), 67 PPE_BIND_AGE1_DELTA_TCP_FIN | 68 PPE_BIND_AGE1_DELTA_TCP, 69 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 70 FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 7)); 71 72 airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i), 73 PPE_SRAM_TABLE_EN_MASK | 74 PPE_SRAM_HASH1_EN_MASK | 75 PPE_DRAM_TABLE_EN_MASK | 76 PPE_SRAM_HASH0_MODE_MASK | 77 PPE_SRAM_HASH1_MODE_MASK | 78 PPE_DRAM_HASH0_MODE_MASK | 79 PPE_DRAM_HASH1_MODE_MASK, 80 FIELD_PREP(PPE_SRAM_TABLE_EN_MASK, 1) | 81 FIELD_PREP(PPE_SRAM_HASH1_EN_MASK, 1) | 82 FIELD_PREP(PPE_SRAM_HASH1_MODE_MASK, 1) | 83 FIELD_PREP(PPE_DRAM_HASH1_MODE_MASK, 3)); 84 85 airoha_fe_rmw(eth, REG_PPE_TB_CFG(i), 86 PPE_TB_CFG_SEARCH_MISS_MASK | 87 PPE_TB_CFG_KEEPALIVE_MASK | 88 PPE_TB_ENTRY_SIZE_MASK, 89 FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) | 90 FIELD_PREP(PPE_TB_ENTRY_SIZE_MASK, 0)); 91 92 airoha_fe_wr(eth, REG_PPE_HASH_SEED(i), PPE_HASH_SEED); 93 94 for (p = 0; p < ARRAY_SIZE(eth->ports); p++) 95 airoha_fe_rmw(eth, REG_PPE_MTU(i, p), 96 FP0_EGRESS_MTU_MASK | 97 FP1_EGRESS_MTU_MASK, 98 FIELD_PREP(FP0_EGRESS_MTU_MASK, 99 AIROHA_MAX_MTU) | 100 FIELD_PREP(FP1_EGRESS_MTU_MASK, 101 AIROHA_MAX_MTU)); 102 } 103 104 if (airoha_ppe2_is_enabled(eth)) { 105 sram_num_entries = 106 PPE_RAM_NUM_ENTRIES_SHIFT(PPE1_SRAM_NUM_DATA_ENTRIES); 107 airoha_fe_rmw(eth, REG_PPE_TB_CFG(0), 108 PPE_SRAM_TB_NUM_ENTRY_MASK | 109 PPE_DRAM_TB_NUM_ENTRY_MASK, 110 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 111 sram_num_entries) | 112 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 113 dram_num_entries)); 114 airoha_fe_rmw(eth, REG_PPE_TB_CFG(1), 115 PPE_SRAM_TB_NUM_ENTRY_MASK | 116 PPE_DRAM_TB_NUM_ENTRY_MASK, 117 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 118 sram_num_entries) | 119 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 120 dram_num_entries)); 121 } else { 122 sram_num_entries = 123 PPE_RAM_NUM_ENTRIES_SHIFT(PPE_SRAM_NUM_DATA_ENTRIES); 124 airoha_fe_rmw(eth, REG_PPE_TB_CFG(0), 125 PPE_SRAM_TB_NUM_ENTRY_MASK | 126 PPE_DRAM_TB_NUM_ENTRY_MASK, 127 FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK, 128 sram_num_entries) | 129 FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 130 dram_num_entries)); 131 } 132 } 133 134 static void airoha_ppe_flow_mangle_eth(const struct flow_action_entry *act, void *eth) 135 { 136 void *dest = eth + act->mangle.offset; 137 const void *src = &act->mangle.val; 138 139 if (act->mangle.offset > 8) 140 return; 141 142 if (act->mangle.mask == 0xffff) { 143 src += 2; 144 dest += 2; 145 } 146 147 memcpy(dest, src, act->mangle.mask ? 2 : 4); 148 } 149 150 static int airoha_ppe_flow_mangle_ports(const struct flow_action_entry *act, 151 struct airoha_flow_data *data) 152 { 153 u32 val = be32_to_cpu((__force __be32)act->mangle.val); 154 155 switch (act->mangle.offset) { 156 case 0: 157 if ((__force __be32)act->mangle.mask == ~cpu_to_be32(0xffff)) 158 data->dst_port = cpu_to_be16(val); 159 else 160 data->src_port = cpu_to_be16(val >> 16); 161 break; 162 case 2: 163 data->dst_port = cpu_to_be16(val); 164 break; 165 default: 166 return -EINVAL; 167 } 168 169 return 0; 170 } 171 172 static int airoha_ppe_flow_mangle_ipv4(const struct flow_action_entry *act, 173 struct airoha_flow_data *data) 174 { 175 __be32 *dest; 176 177 switch (act->mangle.offset) { 178 case offsetof(struct iphdr, saddr): 179 dest = &data->v4.src_addr; 180 break; 181 case offsetof(struct iphdr, daddr): 182 dest = &data->v4.dst_addr; 183 break; 184 default: 185 return -EINVAL; 186 } 187 188 memcpy(dest, &act->mangle.val, sizeof(u32)); 189 190 return 0; 191 } 192 193 static int airoha_get_dsa_port(struct net_device **dev) 194 { 195 #if IS_ENABLED(CONFIG_NET_DSA) 196 struct dsa_port *dp = dsa_port_from_netdev(*dev); 197 198 if (IS_ERR(dp)) 199 return -ENODEV; 200 201 *dev = dsa_port_to_conduit(dp); 202 return dp->index; 203 #else 204 return -ENODEV; 205 #endif 206 } 207 208 static void airoha_ppe_foe_set_bridge_addrs(struct airoha_foe_bridge *br, 209 struct ethhdr *eh) 210 { 211 br->dest_mac_hi = get_unaligned_be32(eh->h_dest); 212 br->dest_mac_lo = get_unaligned_be16(eh->h_dest + 4); 213 br->src_mac_hi = get_unaligned_be16(eh->h_source); 214 br->src_mac_lo = get_unaligned_be32(eh->h_source + 2); 215 } 216 217 static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth, 218 struct airoha_foe_entry *hwe, 219 struct net_device *dev, int type, 220 struct airoha_flow_data *data, 221 int l4proto) 222 { 223 int dsa_port = airoha_get_dsa_port(&dev); 224 struct airoha_foe_mac_info_common *l2; 225 u32 qdata, ports_pad, val; 226 227 memset(hwe, 0, sizeof(*hwe)); 228 229 val = FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, AIROHA_FOE_STATE_BIND) | 230 FIELD_PREP(AIROHA_FOE_IB1_BIND_PACKET_TYPE, type) | 231 FIELD_PREP(AIROHA_FOE_IB1_BIND_UDP, l4proto == IPPROTO_UDP) | 232 FIELD_PREP(AIROHA_FOE_IB1_BIND_VLAN_LAYER, data->vlan.num) | 233 FIELD_PREP(AIROHA_FOE_IB1_BIND_VPM, data->vlan.num) | 234 AIROHA_FOE_IB1_BIND_TTL; 235 hwe->ib1 = val; 236 237 val = FIELD_PREP(AIROHA_FOE_IB2_PORT_AG, 0x1f) | 238 AIROHA_FOE_IB2_PSE_QOS; 239 if (dsa_port >= 0) 240 val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, dsa_port); 241 242 if (dev) { 243 struct airoha_gdm_port *port = netdev_priv(dev); 244 u8 pse_port; 245 246 if (!airoha_is_valid_gdm_port(eth, port)) 247 return -EINVAL; 248 249 if (dsa_port >= 0) 250 pse_port = port->id == 4 ? FE_PSE_PORT_GDM4 : port->id; 251 else 252 pse_port = 2; /* uplink relies on GDM2 loopback */ 253 val |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, pse_port); 254 255 /* For downlink traffic consume SRAM memory for hw forwarding 256 * descriptors queue. 257 */ 258 if (airhoa_is_lan_gdm_port(port)) 259 val |= AIROHA_FOE_IB2_FAST_PATH; 260 } 261 262 if (is_multicast_ether_addr(data->eth.h_dest)) 263 val |= AIROHA_FOE_IB2_MULTICAST; 264 265 ports_pad = 0xa5a5a500 | (l4proto & 0xff); 266 if (type == PPE_PKT_TYPE_IPV4_ROUTE) 267 hwe->ipv4.orig_tuple.ports = ports_pad; 268 if (type == PPE_PKT_TYPE_IPV6_ROUTE_3T) 269 hwe->ipv6.ports = ports_pad; 270 271 qdata = FIELD_PREP(AIROHA_FOE_SHAPER_ID, 0x7f); 272 if (type == PPE_PKT_TYPE_BRIDGE) { 273 airoha_ppe_foe_set_bridge_addrs(&hwe->bridge, &data->eth); 274 hwe->bridge.data = qdata; 275 hwe->bridge.ib2 = val; 276 l2 = &hwe->bridge.l2.common; 277 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 278 hwe->ipv6.data = qdata; 279 hwe->ipv6.ib2 = val; 280 l2 = &hwe->ipv6.l2; 281 } else { 282 hwe->ipv4.data = qdata; 283 hwe->ipv4.ib2 = val; 284 l2 = &hwe->ipv4.l2.common; 285 } 286 287 l2->dest_mac_hi = get_unaligned_be32(data->eth.h_dest); 288 l2->dest_mac_lo = get_unaligned_be16(data->eth.h_dest + 4); 289 if (type <= PPE_PKT_TYPE_IPV4_DSLITE) { 290 l2->src_mac_hi = get_unaligned_be32(data->eth.h_source); 291 hwe->ipv4.l2.src_mac_lo = 292 get_unaligned_be16(data->eth.h_source + 4); 293 } else { 294 l2->src_mac_hi = FIELD_PREP(AIROHA_FOE_MAC_SMAC_ID, 0xf); 295 } 296 297 if (data->vlan.num) { 298 l2->etype = dsa_port >= 0 ? BIT(dsa_port) : 0; 299 l2->vlan1 = data->vlan.hdr[0].id; 300 if (data->vlan.num == 2) 301 l2->vlan2 = data->vlan.hdr[1].id; 302 } else if (dsa_port >= 0) { 303 l2->etype = BIT(15) | BIT(dsa_port); 304 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 305 l2->etype = ETH_P_IPV6; 306 } else { 307 l2->etype = ETH_P_IP; 308 } 309 310 return 0; 311 } 312 313 static int airoha_ppe_foe_entry_set_ipv4_tuple(struct airoha_foe_entry *hwe, 314 struct airoha_flow_data *data, 315 bool egress) 316 { 317 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 318 struct airoha_foe_ipv4_tuple *t; 319 320 switch (type) { 321 case PPE_PKT_TYPE_IPV4_HNAPT: 322 if (egress) { 323 t = &hwe->ipv4.new_tuple; 324 break; 325 } 326 fallthrough; 327 case PPE_PKT_TYPE_IPV4_DSLITE: 328 case PPE_PKT_TYPE_IPV4_ROUTE: 329 t = &hwe->ipv4.orig_tuple; 330 break; 331 default: 332 WARN_ON_ONCE(1); 333 return -EINVAL; 334 } 335 336 t->src_ip = be32_to_cpu(data->v4.src_addr); 337 t->dest_ip = be32_to_cpu(data->v4.dst_addr); 338 339 if (type != PPE_PKT_TYPE_IPV4_ROUTE) { 340 t->src_port = be16_to_cpu(data->src_port); 341 t->dest_port = be16_to_cpu(data->dst_port); 342 } 343 344 return 0; 345 } 346 347 static int airoha_ppe_foe_entry_set_ipv6_tuple(struct airoha_foe_entry *hwe, 348 struct airoha_flow_data *data) 349 350 { 351 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 352 u32 *src, *dest; 353 354 switch (type) { 355 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 356 case PPE_PKT_TYPE_IPV6_6RD: 357 hwe->ipv6.src_port = be16_to_cpu(data->src_port); 358 hwe->ipv6.dest_port = be16_to_cpu(data->dst_port); 359 fallthrough; 360 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 361 src = hwe->ipv6.src_ip; 362 dest = hwe->ipv6.dest_ip; 363 break; 364 default: 365 WARN_ON_ONCE(1); 366 return -EINVAL; 367 } 368 369 ipv6_addr_be32_to_cpu(src, data->v6.src_addr.s6_addr32); 370 ipv6_addr_be32_to_cpu(dest, data->v6.dst_addr.s6_addr32); 371 372 return 0; 373 } 374 375 static u32 airoha_ppe_foe_get_entry_hash(struct airoha_foe_entry *hwe) 376 { 377 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 378 u32 hash, hv1, hv2, hv3; 379 380 switch (type) { 381 case PPE_PKT_TYPE_IPV4_ROUTE: 382 case PPE_PKT_TYPE_IPV4_HNAPT: 383 hv1 = hwe->ipv4.orig_tuple.ports; 384 hv2 = hwe->ipv4.orig_tuple.dest_ip; 385 hv3 = hwe->ipv4.orig_tuple.src_ip; 386 break; 387 case PPE_PKT_TYPE_IPV6_ROUTE_3T: 388 case PPE_PKT_TYPE_IPV6_ROUTE_5T: 389 hv1 = hwe->ipv6.src_ip[3] ^ hwe->ipv6.dest_ip[3]; 390 hv1 ^= hwe->ipv6.ports; 391 392 hv2 = hwe->ipv6.src_ip[2] ^ hwe->ipv6.dest_ip[2]; 393 hv2 ^= hwe->ipv6.dest_ip[0]; 394 395 hv3 = hwe->ipv6.src_ip[1] ^ hwe->ipv6.dest_ip[1]; 396 hv3 ^= hwe->ipv6.src_ip[0]; 397 break; 398 case PPE_PKT_TYPE_BRIDGE: { 399 struct airoha_foe_mac_info *l2 = &hwe->bridge.l2; 400 401 hv1 = l2->common.src_mac_hi & 0xffff; 402 hv1 = hv1 << 16 | l2->src_mac_lo; 403 404 hv2 = l2->common.dest_mac_lo; 405 hv2 = hv2 << 16; 406 hv2 = hv2 | ((l2->common.src_mac_hi & 0xffff0000) >> 16); 407 408 hv3 = l2->common.dest_mac_hi; 409 break; 410 } 411 case PPE_PKT_TYPE_IPV4_DSLITE: 412 case PPE_PKT_TYPE_IPV6_6RD: 413 default: 414 WARN_ON_ONCE(1); 415 return PPE_HASH_MASK; 416 } 417 418 hash = (hv1 & hv2) | ((~hv1) & hv3); 419 hash = (hash >> 24) | ((hash & 0xffffff) << 8); 420 hash ^= hv1 ^ hv2 ^ hv3; 421 hash ^= hash >> 16; 422 hash &= PPE_NUM_ENTRIES - 1; 423 424 return hash; 425 } 426 427 static u32 airoha_ppe_foe_get_flow_stats_index(struct airoha_ppe *ppe, u32 hash) 428 { 429 if (!airoha_ppe2_is_enabled(ppe->eth)) 430 return hash; 431 432 return hash >= PPE_STATS_NUM_ENTRIES ? hash - PPE1_STATS_NUM_ENTRIES 433 : hash; 434 } 435 436 static void airoha_ppe_foe_flow_stat_entry_reset(struct airoha_ppe *ppe, 437 struct airoha_npu *npu, 438 int index) 439 { 440 memset_io(&npu->stats[index], 0, sizeof(*npu->stats)); 441 memset(&ppe->foe_stats[index], 0, sizeof(*ppe->foe_stats)); 442 } 443 444 static void airoha_ppe_foe_flow_stats_reset(struct airoha_ppe *ppe, 445 struct airoha_npu *npu) 446 { 447 int i; 448 449 for (i = 0; i < PPE_STATS_NUM_ENTRIES; i++) 450 airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, i); 451 } 452 453 static void airoha_ppe_foe_flow_stats_update(struct airoha_ppe *ppe, 454 struct airoha_npu *npu, 455 struct airoha_foe_entry *hwe, 456 u32 hash) 457 { 458 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1); 459 u32 index, pse_port, val, *data, *ib2, *meter; 460 u8 nbq; 461 462 index = airoha_ppe_foe_get_flow_stats_index(ppe, hash); 463 if (index >= PPE_STATS_NUM_ENTRIES) 464 return; 465 466 if (type == PPE_PKT_TYPE_BRIDGE) { 467 data = &hwe->bridge.data; 468 ib2 = &hwe->bridge.ib2; 469 meter = &hwe->bridge.l2.meter; 470 } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) { 471 data = &hwe->ipv6.data; 472 ib2 = &hwe->ipv6.ib2; 473 meter = &hwe->ipv6.meter; 474 } else { 475 data = &hwe->ipv4.data; 476 ib2 = &hwe->ipv4.ib2; 477 meter = &hwe->ipv4.l2.meter; 478 } 479 480 airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, index); 481 482 val = FIELD_GET(AIROHA_FOE_CHANNEL | AIROHA_FOE_QID, *data); 483 *data = (*data & ~AIROHA_FOE_ACTDP) | 484 FIELD_PREP(AIROHA_FOE_ACTDP, val); 485 486 val = *ib2 & (AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT | 487 AIROHA_FOE_IB2_PSE_QOS | AIROHA_FOE_IB2_FAST_PATH); 488 *meter |= FIELD_PREP(AIROHA_FOE_TUNNEL_MTU, val); 489 490 pse_port = FIELD_GET(AIROHA_FOE_IB2_PSE_PORT, *ib2); 491 nbq = pse_port == 1 ? 6 : 5; 492 *ib2 &= ~(AIROHA_FOE_IB2_NBQ | AIROHA_FOE_IB2_PSE_PORT | 493 AIROHA_FOE_IB2_PSE_QOS); 494 *ib2 |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, 6) | 495 FIELD_PREP(AIROHA_FOE_IB2_NBQ, nbq); 496 } 497 498 struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe, 499 u32 hash) 500 { 501 if (hash < PPE_SRAM_NUM_ENTRIES) { 502 u32 *hwe = ppe->foe + hash * sizeof(struct airoha_foe_entry); 503 struct airoha_eth *eth = ppe->eth; 504 bool ppe2; 505 u32 val; 506 int i; 507 508 ppe2 = airoha_ppe2_is_enabled(ppe->eth) && 509 hash >= PPE1_SRAM_NUM_ENTRIES; 510 airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2), 511 FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) | 512 PPE_SRAM_CTRL_REQ_MASK); 513 if (read_poll_timeout_atomic(airoha_fe_rr, val, 514 val & PPE_SRAM_CTRL_ACK_MASK, 515 10, 100, false, eth, 516 REG_PPE_RAM_CTRL(ppe2))) 517 return NULL; 518 519 for (i = 0; i < sizeof(struct airoha_foe_entry) / 4; i++) 520 hwe[i] = airoha_fe_rr(eth, 521 REG_PPE_RAM_ENTRY(ppe2, i)); 522 } 523 524 return ppe->foe + hash * sizeof(struct airoha_foe_entry); 525 } 526 527 static bool airoha_ppe_foe_compare_entry(struct airoha_flow_table_entry *e, 528 struct airoha_foe_entry *hwe) 529 { 530 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 531 int len; 532 533 if ((hwe->ib1 ^ e->data.ib1) & AIROHA_FOE_IB1_BIND_UDP) 534 return false; 535 536 if (type > PPE_PKT_TYPE_IPV4_DSLITE) 537 len = offsetof(struct airoha_foe_entry, ipv6.data); 538 else 539 len = offsetof(struct airoha_foe_entry, ipv4.ib2); 540 541 return !memcmp(&e->data.d, &hwe->d, len - sizeof(hwe->ib1)); 542 } 543 544 static int airoha_ppe_foe_commit_entry(struct airoha_ppe *ppe, 545 struct airoha_foe_entry *e, 546 u32 hash) 547 { 548 struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe); 549 u32 ts = airoha_ppe_get_timestamp(ppe); 550 struct airoha_eth *eth = ppe->eth; 551 struct airoha_npu *npu; 552 int err = 0; 553 554 memcpy(&hwe->d, &e->d, sizeof(*hwe) - sizeof(hwe->ib1)); 555 wmb(); 556 557 e->ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP; 558 e->ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_TIMESTAMP, ts); 559 hwe->ib1 = e->ib1; 560 561 rcu_read_lock(); 562 563 npu = rcu_dereference(eth->npu); 564 if (!npu) { 565 err = -ENODEV; 566 goto unlock; 567 } 568 569 airoha_ppe_foe_flow_stats_update(ppe, npu, hwe, hash); 570 571 if (hash < PPE_SRAM_NUM_ENTRIES) { 572 dma_addr_t addr = ppe->foe_dma + hash * sizeof(*hwe); 573 bool ppe2 = airoha_ppe2_is_enabled(eth) && 574 hash >= PPE1_SRAM_NUM_ENTRIES; 575 576 err = npu->ops.ppe_foe_commit_entry(npu, addr, sizeof(*hwe), 577 hash, ppe2); 578 } 579 unlock: 580 rcu_read_unlock(); 581 582 return err; 583 } 584 585 static void airoha_ppe_foe_remove_flow(struct airoha_ppe *ppe, 586 struct airoha_flow_table_entry *e) 587 { 588 lockdep_assert_held(&ppe_lock); 589 590 hlist_del_init(&e->list); 591 if (e->hash != 0xffff) { 592 e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_STATE; 593 e->data.ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE, 594 AIROHA_FOE_STATE_INVALID); 595 airoha_ppe_foe_commit_entry(ppe, &e->data, e->hash); 596 e->hash = 0xffff; 597 } 598 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 599 hlist_del_init(&e->l2_subflow_node); 600 kfree(e); 601 } 602 } 603 604 static void airoha_ppe_foe_remove_l2_flow(struct airoha_ppe *ppe, 605 struct airoha_flow_table_entry *e) 606 { 607 struct hlist_head *head = &e->l2_flows; 608 struct hlist_node *n; 609 610 lockdep_assert_held(&ppe_lock); 611 612 rhashtable_remove_fast(&ppe->l2_flows, &e->l2_node, 613 airoha_l2_flow_table_params); 614 hlist_for_each_entry_safe(e, n, head, l2_subflow_node) 615 airoha_ppe_foe_remove_flow(ppe, e); 616 } 617 618 static void airoha_ppe_foe_flow_remove_entry(struct airoha_ppe *ppe, 619 struct airoha_flow_table_entry *e) 620 { 621 spin_lock_bh(&ppe_lock); 622 623 if (e->type == FLOW_TYPE_L2) 624 airoha_ppe_foe_remove_l2_flow(ppe, e); 625 else 626 airoha_ppe_foe_remove_flow(ppe, e); 627 628 spin_unlock_bh(&ppe_lock); 629 } 630 631 static int 632 airoha_ppe_foe_commit_subflow_entry(struct airoha_ppe *ppe, 633 struct airoha_flow_table_entry *e, 634 u32 hash) 635 { 636 u32 mask = AIROHA_FOE_IB1_BIND_PACKET_TYPE | AIROHA_FOE_IB1_BIND_UDP; 637 struct airoha_foe_entry *hwe_p, hwe; 638 struct airoha_flow_table_entry *f; 639 struct airoha_foe_mac_info *l2; 640 int type; 641 642 hwe_p = airoha_ppe_foe_get_entry(ppe, hash); 643 if (!hwe_p) 644 return -EINVAL; 645 646 f = kzalloc(sizeof(*f), GFP_ATOMIC); 647 if (!f) 648 return -ENOMEM; 649 650 hlist_add_head(&f->l2_subflow_node, &e->l2_flows); 651 f->type = FLOW_TYPE_L2_SUBFLOW; 652 f->hash = hash; 653 654 memcpy(&hwe, hwe_p, sizeof(*hwe_p)); 655 hwe.ib1 = (hwe.ib1 & mask) | (e->data.ib1 & ~mask); 656 l2 = &hwe.bridge.l2; 657 memcpy(l2, &e->data.bridge.l2, sizeof(*l2)); 658 659 type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe.ib1); 660 if (type == PPE_PKT_TYPE_IPV4_HNAPT) 661 memcpy(&hwe.ipv4.new_tuple, &hwe.ipv4.orig_tuple, 662 sizeof(hwe.ipv4.new_tuple)); 663 else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T && 664 l2->common.etype == ETH_P_IP) 665 l2->common.etype = ETH_P_IPV6; 666 667 hwe.bridge.ib2 = e->data.bridge.ib2; 668 hwe.bridge.data = e->data.bridge.data; 669 airoha_ppe_foe_commit_entry(ppe, &hwe, hash); 670 671 return 0; 672 } 673 674 static void airoha_ppe_foe_insert_entry(struct airoha_ppe *ppe, 675 struct sk_buff *skb, 676 u32 hash) 677 { 678 struct airoha_flow_table_entry *e; 679 struct airoha_foe_bridge br = {}; 680 struct airoha_foe_entry *hwe; 681 bool commit_done = false; 682 struct hlist_node *n; 683 u32 index, state; 684 685 spin_lock_bh(&ppe_lock); 686 687 hwe = airoha_ppe_foe_get_entry(ppe, hash); 688 if (!hwe) 689 goto unlock; 690 691 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 692 if (state == AIROHA_FOE_STATE_BIND) 693 goto unlock; 694 695 index = airoha_ppe_foe_get_entry_hash(hwe); 696 hlist_for_each_entry_safe(e, n, &ppe->foe_flow[index], list) { 697 if (e->type == FLOW_TYPE_L2_SUBFLOW) { 698 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1); 699 if (state != AIROHA_FOE_STATE_BIND) { 700 e->hash = 0xffff; 701 airoha_ppe_foe_remove_flow(ppe, e); 702 } 703 continue; 704 } 705 706 if (commit_done || !airoha_ppe_foe_compare_entry(e, hwe)) { 707 e->hash = 0xffff; 708 continue; 709 } 710 711 airoha_ppe_foe_commit_entry(ppe, &e->data, hash); 712 commit_done = true; 713 e->hash = hash; 714 } 715 716 if (commit_done) 717 goto unlock; 718 719 airoha_ppe_foe_set_bridge_addrs(&br, eth_hdr(skb)); 720 e = rhashtable_lookup_fast(&ppe->l2_flows, &br, 721 airoha_l2_flow_table_params); 722 if (e) 723 airoha_ppe_foe_commit_subflow_entry(ppe, e, hash); 724 unlock: 725 spin_unlock_bh(&ppe_lock); 726 } 727 728 static int 729 airoha_ppe_foe_l2_flow_commit_entry(struct airoha_ppe *ppe, 730 struct airoha_flow_table_entry *e) 731 { 732 struct airoha_flow_table_entry *prev; 733 734 e->type = FLOW_TYPE_L2; 735 prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &e->l2_node, 736 airoha_l2_flow_table_params); 737 if (!prev) 738 return 0; 739 740 if (IS_ERR(prev)) 741 return PTR_ERR(prev); 742 743 return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node, 744 &e->l2_node, 745 airoha_l2_flow_table_params); 746 } 747 748 static int airoha_ppe_foe_flow_commit_entry(struct airoha_ppe *ppe, 749 struct airoha_flow_table_entry *e) 750 { 751 int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1); 752 u32 hash; 753 754 if (type == PPE_PKT_TYPE_BRIDGE) 755 return airoha_ppe_foe_l2_flow_commit_entry(ppe, e); 756 757 hash = airoha_ppe_foe_get_entry_hash(&e->data); 758 e->type = FLOW_TYPE_L4; 759 e->hash = 0xffff; 760 761 spin_lock_bh(&ppe_lock); 762 hlist_add_head(&e->list, &ppe->foe_flow[hash]); 763 spin_unlock_bh(&ppe_lock); 764 765 return 0; 766 } 767 768 static int airoha_ppe_get_entry_idle_time(struct airoha_ppe *ppe, u32 ib1) 769 { 770 u32 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1); 771 u32 ts, ts_mask, now = airoha_ppe_get_timestamp(ppe); 772 int idle; 773 774 if (state == AIROHA_FOE_STATE_BIND) { 775 ts = FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, ib1); 776 ts_mask = AIROHA_FOE_IB1_BIND_TIMESTAMP; 777 } else { 778 ts = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, ib1); 779 now = FIELD_GET(AIROHA_FOE_IB1_UNBIND_TIMESTAMP, now); 780 ts_mask = AIROHA_FOE_IB1_UNBIND_TIMESTAMP; 781 } 782 idle = now - ts; 783 784 return idle < 0 ? idle + ts_mask + 1 : idle; 785 } 786 787 static void 788 airoha_ppe_foe_flow_l2_entry_update(struct airoha_ppe *ppe, 789 struct airoha_flow_table_entry *e) 790 { 791 int min_idle = airoha_ppe_get_entry_idle_time(ppe, e->data.ib1); 792 struct airoha_flow_table_entry *iter; 793 struct hlist_node *n; 794 795 lockdep_assert_held(&ppe_lock); 796 797 hlist_for_each_entry_safe(iter, n, &e->l2_flows, l2_subflow_node) { 798 struct airoha_foe_entry *hwe; 799 u32 ib1, state; 800 int idle; 801 802 hwe = airoha_ppe_foe_get_entry(ppe, iter->hash); 803 ib1 = READ_ONCE(hwe->ib1); 804 805 state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, ib1); 806 if (state != AIROHA_FOE_STATE_BIND) { 807 iter->hash = 0xffff; 808 airoha_ppe_foe_remove_flow(ppe, iter); 809 continue; 810 } 811 812 idle = airoha_ppe_get_entry_idle_time(ppe, ib1); 813 if (idle >= min_idle) 814 continue; 815 816 min_idle = idle; 817 e->data.ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP; 818 e->data.ib1 |= ib1 & AIROHA_FOE_IB1_BIND_TIMESTAMP; 819 } 820 } 821 822 static void airoha_ppe_foe_flow_entry_update(struct airoha_ppe *ppe, 823 struct airoha_flow_table_entry *e) 824 { 825 struct airoha_foe_entry *hwe_p, hwe = {}; 826 827 spin_lock_bh(&ppe_lock); 828 829 if (e->type == FLOW_TYPE_L2) { 830 airoha_ppe_foe_flow_l2_entry_update(ppe, e); 831 goto unlock; 832 } 833 834 if (e->hash == 0xffff) 835 goto unlock; 836 837 hwe_p = airoha_ppe_foe_get_entry(ppe, e->hash); 838 if (!hwe_p) 839 goto unlock; 840 841 memcpy(&hwe, hwe_p, sizeof(*hwe_p)); 842 if (!airoha_ppe_foe_compare_entry(e, &hwe)) { 843 e->hash = 0xffff; 844 goto unlock; 845 } 846 847 e->data.ib1 = hwe.ib1; 848 unlock: 849 spin_unlock_bh(&ppe_lock); 850 } 851 852 static int airoha_ppe_entry_idle_time(struct airoha_ppe *ppe, 853 struct airoha_flow_table_entry *e) 854 { 855 airoha_ppe_foe_flow_entry_update(ppe, e); 856 857 return airoha_ppe_get_entry_idle_time(ppe, e->data.ib1); 858 } 859 860 static int airoha_ppe_flow_offload_replace(struct airoha_gdm_port *port, 861 struct flow_cls_offload *f) 862 { 863 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 864 struct airoha_eth *eth = port->qdma->eth; 865 struct airoha_flow_table_entry *e; 866 struct airoha_flow_data data = {}; 867 struct net_device *odev = NULL; 868 struct flow_action_entry *act; 869 struct airoha_foe_entry hwe; 870 int err, i, offload_type; 871 u16 addr_type = 0; 872 u8 l4proto = 0; 873 874 if (rhashtable_lookup(ð->flow_table, &f->cookie, 875 airoha_flow_table_params)) 876 return -EEXIST; 877 878 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) 879 return -EOPNOTSUPP; 880 881 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 882 struct flow_match_control match; 883 884 flow_rule_match_control(rule, &match); 885 addr_type = match.key->addr_type; 886 if (flow_rule_has_control_flags(match.mask->flags, 887 f->common.extack)) 888 return -EOPNOTSUPP; 889 } else { 890 return -EOPNOTSUPP; 891 } 892 893 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 894 struct flow_match_basic match; 895 896 flow_rule_match_basic(rule, &match); 897 l4proto = match.key->ip_proto; 898 } else { 899 return -EOPNOTSUPP; 900 } 901 902 switch (addr_type) { 903 case 0: 904 offload_type = PPE_PKT_TYPE_BRIDGE; 905 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 906 struct flow_match_eth_addrs match; 907 908 flow_rule_match_eth_addrs(rule, &match); 909 memcpy(data.eth.h_dest, match.key->dst, ETH_ALEN); 910 memcpy(data.eth.h_source, match.key->src, ETH_ALEN); 911 } else { 912 return -EOPNOTSUPP; 913 } 914 break; 915 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 916 offload_type = PPE_PKT_TYPE_IPV4_HNAPT; 917 break; 918 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 919 offload_type = PPE_PKT_TYPE_IPV6_ROUTE_5T; 920 break; 921 default: 922 return -EOPNOTSUPP; 923 } 924 925 flow_action_for_each(i, act, &rule->action) { 926 switch (act->id) { 927 case FLOW_ACTION_MANGLE: 928 if (offload_type == PPE_PKT_TYPE_BRIDGE) 929 return -EOPNOTSUPP; 930 931 if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH) 932 airoha_ppe_flow_mangle_eth(act, &data.eth); 933 break; 934 case FLOW_ACTION_REDIRECT: 935 odev = act->dev; 936 break; 937 case FLOW_ACTION_CSUM: 938 break; 939 case FLOW_ACTION_VLAN_PUSH: 940 if (data.vlan.num == 2 || 941 act->vlan.proto != htons(ETH_P_8021Q)) 942 return -EOPNOTSUPP; 943 944 data.vlan.hdr[data.vlan.num].id = act->vlan.vid; 945 data.vlan.hdr[data.vlan.num].proto = act->vlan.proto; 946 data.vlan.num++; 947 break; 948 case FLOW_ACTION_VLAN_POP: 949 break; 950 case FLOW_ACTION_PPPOE_PUSH: 951 break; 952 default: 953 return -EOPNOTSUPP; 954 } 955 } 956 957 if (!is_valid_ether_addr(data.eth.h_source) || 958 !is_valid_ether_addr(data.eth.h_dest)) 959 return -EINVAL; 960 961 err = airoha_ppe_foe_entry_prepare(eth, &hwe, odev, offload_type, 962 &data, l4proto); 963 if (err) 964 return err; 965 966 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 967 struct flow_match_ports ports; 968 969 if (offload_type == PPE_PKT_TYPE_BRIDGE) 970 return -EOPNOTSUPP; 971 972 flow_rule_match_ports(rule, &ports); 973 data.src_port = ports.key->src; 974 data.dst_port = ports.key->dst; 975 } else if (offload_type != PPE_PKT_TYPE_BRIDGE) { 976 return -EOPNOTSUPP; 977 } 978 979 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 980 struct flow_match_ipv4_addrs addrs; 981 982 flow_rule_match_ipv4_addrs(rule, &addrs); 983 data.v4.src_addr = addrs.key->src; 984 data.v4.dst_addr = addrs.key->dst; 985 airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, false); 986 } 987 988 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 989 struct flow_match_ipv6_addrs addrs; 990 991 flow_rule_match_ipv6_addrs(rule, &addrs); 992 993 data.v6.src_addr = addrs.key->src; 994 data.v6.dst_addr = addrs.key->dst; 995 airoha_ppe_foe_entry_set_ipv6_tuple(&hwe, &data); 996 } 997 998 flow_action_for_each(i, act, &rule->action) { 999 if (act->id != FLOW_ACTION_MANGLE) 1000 continue; 1001 1002 if (offload_type == PPE_PKT_TYPE_BRIDGE) 1003 return -EOPNOTSUPP; 1004 1005 switch (act->mangle.htype) { 1006 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 1007 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 1008 err = airoha_ppe_flow_mangle_ports(act, &data); 1009 break; 1010 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 1011 err = airoha_ppe_flow_mangle_ipv4(act, &data); 1012 break; 1013 case FLOW_ACT_MANGLE_HDR_TYPE_ETH: 1014 /* handled earlier */ 1015 break; 1016 default: 1017 return -EOPNOTSUPP; 1018 } 1019 1020 if (err) 1021 return err; 1022 } 1023 1024 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1025 err = airoha_ppe_foe_entry_set_ipv4_tuple(&hwe, &data, true); 1026 if (err) 1027 return err; 1028 } 1029 1030 e = kzalloc(sizeof(*e), GFP_KERNEL); 1031 if (!e) 1032 return -ENOMEM; 1033 1034 e->cookie = f->cookie; 1035 memcpy(&e->data, &hwe, sizeof(e->data)); 1036 1037 err = airoha_ppe_foe_flow_commit_entry(eth->ppe, e); 1038 if (err) 1039 goto free_entry; 1040 1041 err = rhashtable_insert_fast(ð->flow_table, &e->node, 1042 airoha_flow_table_params); 1043 if (err < 0) 1044 goto remove_foe_entry; 1045 1046 return 0; 1047 1048 remove_foe_entry: 1049 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 1050 free_entry: 1051 kfree(e); 1052 1053 return err; 1054 } 1055 1056 static int airoha_ppe_flow_offload_destroy(struct airoha_gdm_port *port, 1057 struct flow_cls_offload *f) 1058 { 1059 struct airoha_eth *eth = port->qdma->eth; 1060 struct airoha_flow_table_entry *e; 1061 1062 e = rhashtable_lookup(ð->flow_table, &f->cookie, 1063 airoha_flow_table_params); 1064 if (!e) 1065 return -ENOENT; 1066 1067 airoha_ppe_foe_flow_remove_entry(eth->ppe, e); 1068 rhashtable_remove_fast(ð->flow_table, &e->node, 1069 airoha_flow_table_params); 1070 kfree(e); 1071 1072 return 0; 1073 } 1074 1075 void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 hash, 1076 struct airoha_foe_stats64 *stats) 1077 { 1078 u32 index = airoha_ppe_foe_get_flow_stats_index(ppe, hash); 1079 struct airoha_eth *eth = ppe->eth; 1080 struct airoha_npu *npu; 1081 1082 if (index >= PPE_STATS_NUM_ENTRIES) 1083 return; 1084 1085 rcu_read_lock(); 1086 1087 npu = rcu_dereference(eth->npu); 1088 if (npu) { 1089 u64 packets = ppe->foe_stats[index].packets; 1090 u64 bytes = ppe->foe_stats[index].bytes; 1091 struct airoha_foe_stats npu_stats; 1092 1093 memcpy_fromio(&npu_stats, &npu->stats[index], 1094 sizeof(*npu->stats)); 1095 stats->packets = packets << 32 | npu_stats.packets; 1096 stats->bytes = bytes << 32 | npu_stats.bytes; 1097 } 1098 1099 rcu_read_unlock(); 1100 } 1101 1102 static int airoha_ppe_flow_offload_stats(struct airoha_gdm_port *port, 1103 struct flow_cls_offload *f) 1104 { 1105 struct airoha_eth *eth = port->qdma->eth; 1106 struct airoha_flow_table_entry *e; 1107 u32 idle; 1108 1109 e = rhashtable_lookup(ð->flow_table, &f->cookie, 1110 airoha_flow_table_params); 1111 if (!e) 1112 return -ENOENT; 1113 1114 idle = airoha_ppe_entry_idle_time(eth->ppe, e); 1115 f->stats.lastused = jiffies - idle * HZ; 1116 1117 if (e->hash != 0xffff) { 1118 struct airoha_foe_stats64 stats = {}; 1119 1120 airoha_ppe_foe_entry_get_stats(eth->ppe, e->hash, &stats); 1121 f->stats.pkts += (stats.packets - e->stats.packets); 1122 f->stats.bytes += (stats.bytes - e->stats.bytes); 1123 e->stats = stats; 1124 } 1125 1126 return 0; 1127 } 1128 1129 static int airoha_ppe_flow_offload_cmd(struct airoha_gdm_port *port, 1130 struct flow_cls_offload *f) 1131 { 1132 switch (f->command) { 1133 case FLOW_CLS_REPLACE: 1134 return airoha_ppe_flow_offload_replace(port, f); 1135 case FLOW_CLS_DESTROY: 1136 return airoha_ppe_flow_offload_destroy(port, f); 1137 case FLOW_CLS_STATS: 1138 return airoha_ppe_flow_offload_stats(port, f); 1139 default: 1140 break; 1141 } 1142 1143 return -EOPNOTSUPP; 1144 } 1145 1146 static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe, 1147 struct airoha_npu *npu) 1148 { 1149 int i, sram_num_entries = PPE_SRAM_NUM_ENTRIES; 1150 struct airoha_foe_entry *hwe = ppe->foe; 1151 1152 if (airoha_ppe2_is_enabled(ppe->eth)) 1153 sram_num_entries = sram_num_entries / 2; 1154 1155 for (i = 0; i < sram_num_entries; i++) 1156 memset(&hwe[i], 0, sizeof(*hwe)); 1157 1158 return npu->ops.ppe_flush_sram_entries(npu, ppe->foe_dma, 1159 PPE_SRAM_NUM_ENTRIES); 1160 } 1161 1162 static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth) 1163 { 1164 struct airoha_npu *npu = airoha_npu_get(eth->dev, 1165 ð->ppe->foe_stats_dma); 1166 1167 if (IS_ERR(npu)) { 1168 request_module("airoha-npu"); 1169 npu = airoha_npu_get(eth->dev, ð->ppe->foe_stats_dma); 1170 } 1171 1172 return npu; 1173 } 1174 1175 static int airoha_ppe_offload_setup(struct airoha_eth *eth) 1176 { 1177 struct airoha_npu *npu = airoha_ppe_npu_get(eth); 1178 int err; 1179 1180 if (IS_ERR(npu)) 1181 return PTR_ERR(npu); 1182 1183 err = npu->ops.ppe_init(npu); 1184 if (err) 1185 goto error_npu_put; 1186 1187 airoha_ppe_hw_init(eth->ppe); 1188 err = airoha_ppe_flush_sram_entries(eth->ppe, npu); 1189 if (err) 1190 goto error_npu_put; 1191 1192 airoha_ppe_foe_flow_stats_reset(eth->ppe, npu); 1193 1194 rcu_assign_pointer(eth->npu, npu); 1195 synchronize_rcu(); 1196 1197 return 0; 1198 1199 error_npu_put: 1200 airoha_npu_put(npu); 1201 1202 return err; 1203 } 1204 1205 int airoha_ppe_setup_tc_block_cb(struct net_device *dev, void *type_data) 1206 { 1207 struct airoha_gdm_port *port = netdev_priv(dev); 1208 struct flow_cls_offload *cls = type_data; 1209 struct airoha_eth *eth = port->qdma->eth; 1210 int err = 0; 1211 1212 mutex_lock(&flow_offload_mutex); 1213 1214 if (!eth->npu) 1215 err = airoha_ppe_offload_setup(eth); 1216 if (!err) 1217 err = airoha_ppe_flow_offload_cmd(port, cls); 1218 1219 mutex_unlock(&flow_offload_mutex); 1220 1221 return err; 1222 } 1223 1224 void airoha_ppe_check_skb(struct airoha_ppe *ppe, struct sk_buff *skb, 1225 u16 hash) 1226 { 1227 u16 now, diff; 1228 1229 if (hash > PPE_HASH_MASK) 1230 return; 1231 1232 now = (u16)jiffies; 1233 diff = now - ppe->foe_check_time[hash]; 1234 if (diff < HZ / 10) 1235 return; 1236 1237 ppe->foe_check_time[hash] = now; 1238 airoha_ppe_foe_insert_entry(ppe, skb, hash); 1239 } 1240 1241 int airoha_ppe_init(struct airoha_eth *eth) 1242 { 1243 struct airoha_ppe *ppe; 1244 int foe_size, err; 1245 1246 ppe = devm_kzalloc(eth->dev, sizeof(*ppe), GFP_KERNEL); 1247 if (!ppe) 1248 return -ENOMEM; 1249 1250 foe_size = PPE_NUM_ENTRIES * sizeof(struct airoha_foe_entry); 1251 ppe->foe = dmam_alloc_coherent(eth->dev, foe_size, &ppe->foe_dma, 1252 GFP_KERNEL); 1253 if (!ppe->foe) 1254 return -ENOMEM; 1255 1256 ppe->eth = eth; 1257 eth->ppe = ppe; 1258 1259 ppe->foe_flow = devm_kzalloc(eth->dev, 1260 PPE_NUM_ENTRIES * sizeof(*ppe->foe_flow), 1261 GFP_KERNEL); 1262 if (!ppe->foe_flow) 1263 return -ENOMEM; 1264 1265 foe_size = PPE_STATS_NUM_ENTRIES * sizeof(*ppe->foe_stats); 1266 if (foe_size) { 1267 ppe->foe_stats = dmam_alloc_coherent(eth->dev, foe_size, 1268 &ppe->foe_stats_dma, 1269 GFP_KERNEL); 1270 if (!ppe->foe_stats) 1271 return -ENOMEM; 1272 } 1273 1274 err = rhashtable_init(ð->flow_table, &airoha_flow_table_params); 1275 if (err) 1276 return err; 1277 1278 err = rhashtable_init(&ppe->l2_flows, &airoha_l2_flow_table_params); 1279 if (err) 1280 goto error_flow_table_destroy; 1281 1282 err = airoha_ppe_debugfs_init(ppe); 1283 if (err) 1284 goto error_l2_flow_table_destroy; 1285 1286 return 0; 1287 1288 error_l2_flow_table_destroy: 1289 rhashtable_destroy(&ppe->l2_flows); 1290 error_flow_table_destroy: 1291 rhashtable_destroy(ð->flow_table); 1292 1293 return err; 1294 } 1295 1296 void airoha_ppe_deinit(struct airoha_eth *eth) 1297 { 1298 struct airoha_npu *npu; 1299 1300 rcu_read_lock(); 1301 npu = rcu_dereference(eth->npu); 1302 if (npu) { 1303 npu->ops.ppe_deinit(npu); 1304 airoha_npu_put(npu); 1305 } 1306 rcu_read_unlock(); 1307 1308 rhashtable_destroy(ð->ppe->l2_flows); 1309 rhashtable_destroy(ð->flow_table); 1310 debugfs_remove(eth->ppe->debugfs_dir); 1311 } 1312