1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2021 Corigine, Inc. */ 3 4 #include <net/tc_act/tc_csum.h> 5 #include <net/tc_act/tc_ct.h> 6 7 #include "conntrack.h" 8 #include "../nfp_port.h" 9 10 const struct rhashtable_params nfp_tc_ct_merge_params = { 11 .head_offset = offsetof(struct nfp_fl_ct_tc_merge, 12 hash_node), 13 .key_len = sizeof(unsigned long) * 2, 14 .key_offset = offsetof(struct nfp_fl_ct_tc_merge, cookie), 15 .automatic_shrinking = true, 16 }; 17 18 const struct rhashtable_params nfp_nft_ct_merge_params = { 19 .head_offset = offsetof(struct nfp_fl_nft_tc_merge, 20 hash_node), 21 .key_len = sizeof(unsigned long) * 3, 22 .key_offset = offsetof(struct nfp_fl_nft_tc_merge, cookie), 23 .automatic_shrinking = true, 24 }; 25 26 static struct flow_action_entry *get_flow_act(struct flow_rule *rule, 27 enum flow_action_id act_id); 28 29 /** 30 * get_hashentry() - Wrapper around hashtable lookup. 31 * @ht: hashtable where entry could be found 32 * @key: key to lookup 33 * @params: hashtable params 34 * @size: size of entry to allocate if not in table 35 * 36 * Returns an entry from a hashtable. If entry does not exist 37 * yet allocate the memory for it and return the new entry. 38 */ 39 static void *get_hashentry(struct rhashtable *ht, void *key, 40 const struct rhashtable_params params, size_t size) 41 { 42 void *result; 43 44 result = rhashtable_lookup_fast(ht, key, params); 45 46 if (result) 47 return result; 48 49 result = kzalloc(size, GFP_KERNEL); 50 if (!result) 51 return ERR_PTR(-ENOMEM); 52 53 return result; 54 } 55 56 bool is_pre_ct_flow(struct flow_cls_offload *flow) 57 { 58 struct flow_rule *rule = flow_cls_offload_flow_rule(flow); 59 struct flow_dissector *dissector = rule->match.dissector; 60 struct flow_action_entry *act; 61 struct flow_match_ct ct; 62 int i; 63 64 if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) { 65 flow_rule_match_ct(rule, &ct); 66 if (ct.key->ct_state) 67 return false; 68 } 69 70 if (flow->common.chain_index) 71 return false; 72 73 flow_action_for_each(i, act, &flow->rule->action) { 74 if (act->id == FLOW_ACTION_CT) { 75 /* The pre_ct rule only have the ct or ct nat action, cannot 76 * contains other ct action e.g ct commit and so on. 77 */ 78 if ((!act->ct.action || act->ct.action == TCA_CT_ACT_NAT)) 79 return true; 80 else 81 return false; 82 } 83 } 84 85 return false; 86 } 87 88 bool is_post_ct_flow(struct flow_cls_offload *flow) 89 { 90 struct flow_rule *rule = flow_cls_offload_flow_rule(flow); 91 struct flow_dissector *dissector = rule->match.dissector; 92 struct flow_action_entry *act; 93 bool exist_ct_clear = false; 94 struct flow_match_ct ct; 95 int i; 96 97 if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) { 98 flow_rule_match_ct(rule, &ct); 99 if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) 100 return true; 101 } else { 102 /* post ct entry cannot contains any ct action except ct_clear. */ 103 flow_action_for_each(i, act, &flow->rule->action) { 104 if (act->id == FLOW_ACTION_CT) { 105 /* ignore ct clear action. */ 106 if (act->ct.action == TCA_CT_ACT_CLEAR) { 107 exist_ct_clear = true; 108 continue; 109 } 110 111 return false; 112 } 113 } 114 /* when do nat with ct, the post ct entry ignore the ct status, 115 * will match the nat field(sip/dip) instead. In this situation, 116 * the flow chain index is not zero and contains ct clear action. 117 */ 118 if (flow->common.chain_index && exist_ct_clear) 119 return true; 120 } 121 122 return false; 123 } 124 125 /** 126 * get_mangled_key() - Mangle the key if mangle act exists 127 * @rule: rule that carries the actions 128 * @buf: pointer to key to be mangled 129 * @offset: used to adjust mangled offset in L2/L3/L4 header 130 * @key_sz: key size 131 * @htype: mangling type 132 * 133 * Returns buf where the mangled key stores. 134 */ 135 static void *get_mangled_key(struct flow_rule *rule, void *buf, 136 u32 offset, size_t key_sz, 137 enum flow_action_mangle_base htype) 138 { 139 struct flow_action_entry *act; 140 u32 *val = (u32 *)buf; 141 u32 off, msk, key; 142 int i; 143 144 flow_action_for_each(i, act, &rule->action) { 145 if (act->id == FLOW_ACTION_MANGLE && 146 act->mangle.htype == htype) { 147 off = act->mangle.offset - offset; 148 msk = act->mangle.mask; 149 key = act->mangle.val; 150 151 /* Mangling is supposed to be u32 aligned */ 152 if (off % 4 || off >= key_sz) 153 continue; 154 155 val[off >> 2] &= msk; 156 val[off >> 2] |= key; 157 } 158 } 159 160 return buf; 161 } 162 163 /* Only tos and ttl are involved in flow_match_ip structure, which 164 * doesn't conform to the layout of ip/ipv6 header definition. So 165 * they need particular process here: fill them into the ip/ipv6 166 * header, so that mangling actions can work directly. 167 */ 168 #define NFP_IPV4_TOS_MASK GENMASK(23, 16) 169 #define NFP_IPV4_TTL_MASK GENMASK(31, 24) 170 #define NFP_IPV6_TCLASS_MASK GENMASK(27, 20) 171 #define NFP_IPV6_HLIMIT_MASK GENMASK(7, 0) 172 static void *get_mangled_tos_ttl(struct flow_rule *rule, void *buf, 173 bool is_v6) 174 { 175 struct flow_match_ip match; 176 /* IPv4's ttl field is in third dword. */ 177 __be32 ip_hdr[3]; 178 u32 tmp, hdr_len; 179 180 flow_rule_match_ip(rule, &match); 181 182 if (is_v6) { 183 tmp = FIELD_PREP(NFP_IPV6_TCLASS_MASK, match.key->tos); 184 ip_hdr[0] = cpu_to_be32(tmp); 185 tmp = FIELD_PREP(NFP_IPV6_HLIMIT_MASK, match.key->ttl); 186 ip_hdr[1] = cpu_to_be32(tmp); 187 hdr_len = 2 * sizeof(__be32); 188 } else { 189 tmp = FIELD_PREP(NFP_IPV4_TOS_MASK, match.key->tos); 190 ip_hdr[0] = cpu_to_be32(tmp); 191 tmp = FIELD_PREP(NFP_IPV4_TTL_MASK, match.key->ttl); 192 ip_hdr[2] = cpu_to_be32(tmp); 193 hdr_len = 3 * sizeof(__be32); 194 } 195 196 get_mangled_key(rule, ip_hdr, 0, hdr_len, 197 is_v6 ? FLOW_ACT_MANGLE_HDR_TYPE_IP6 : 198 FLOW_ACT_MANGLE_HDR_TYPE_IP4); 199 200 match.key = buf; 201 202 if (is_v6) { 203 tmp = be32_to_cpu(ip_hdr[0]); 204 match.key->tos = FIELD_GET(NFP_IPV6_TCLASS_MASK, tmp); 205 tmp = be32_to_cpu(ip_hdr[1]); 206 match.key->ttl = FIELD_GET(NFP_IPV6_HLIMIT_MASK, tmp); 207 } else { 208 tmp = be32_to_cpu(ip_hdr[0]); 209 match.key->tos = FIELD_GET(NFP_IPV4_TOS_MASK, tmp); 210 tmp = be32_to_cpu(ip_hdr[2]); 211 match.key->ttl = FIELD_GET(NFP_IPV4_TTL_MASK, tmp); 212 } 213 214 return buf; 215 } 216 217 /* Note entry1 and entry2 are not swappable. only skip ip and 218 * tport merge check for pre_ct and post_ct when pre_ct do nat. 219 */ 220 static bool nfp_ct_merge_check_cannot_skip(struct nfp_fl_ct_flow_entry *entry1, 221 struct nfp_fl_ct_flow_entry *entry2) 222 { 223 /* only pre_ct have NFP_FL_ACTION_DO_NAT flag. */ 224 if ((entry1->flags & NFP_FL_ACTION_DO_NAT) && 225 entry2->type == CT_TYPE_POST_CT) 226 return false; 227 228 return true; 229 } 230 231 /* Note entry1 and entry2 are not swappable, entry1 should be 232 * the former flow whose mangle action need be taken into account 233 * if existed, and entry2 should be the latter flow whose action 234 * we don't care. 235 */ 236 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, 237 struct nfp_fl_ct_flow_entry *entry2) 238 { 239 unsigned long long ovlp_keys; 240 bool out, is_v6 = false; 241 u8 ip_proto = 0; 242 ovlp_keys = entry1->rule->match.dissector->used_keys & 243 entry2->rule->match.dissector->used_keys; 244 /* Temporary buffer for mangling keys, 64 is enough to cover max 245 * struct size of key in various fields that may be mangled. 246 * Supported fields to mangle: 247 * mac_src/mac_dst(struct flow_match_eth_addrs, 12B) 248 * nw_tos/nw_ttl(struct flow_match_ip, 2B) 249 * nw_src/nw_dst(struct flow_match_ipv4/6_addrs, 32B) 250 * tp_src/tp_dst(struct flow_match_ports, 4B) 251 */ 252 char buf[64]; 253 254 if (entry1->netdev && entry2->netdev && 255 entry1->netdev != entry2->netdev) 256 return -EINVAL; 257 258 /* Check the overlapped fields one by one, the unmasked part 259 * should not conflict with each other. 260 */ 261 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL)) { 262 struct flow_match_control match1, match2; 263 264 flow_rule_match_control(entry1->rule, &match1); 265 flow_rule_match_control(entry2->rule, &match2); 266 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 267 if (out) 268 goto check_failed; 269 } 270 271 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_BASIC)) { 272 struct flow_match_basic match1, match2; 273 274 flow_rule_match_basic(entry1->rule, &match1); 275 flow_rule_match_basic(entry2->rule, &match2); 276 277 /* n_proto field is a must in ct-related flows, 278 * it should be either ipv4 or ipv6. 279 */ 280 is_v6 = match1.key->n_proto == htons(ETH_P_IPV6); 281 /* ip_proto field is a must when port field is cared */ 282 ip_proto = match1.key->ip_proto; 283 284 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 285 if (out) 286 goto check_failed; 287 } 288 289 /* if pre ct entry do nat, the nat ip exists in nft entry, 290 * will be do merge check when do nft and post ct merge, 291 * so skip this ip merge check here. 292 */ 293 if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) && 294 nfp_ct_merge_check_cannot_skip(entry1, entry2)) { 295 struct flow_match_ipv4_addrs match1, match2; 296 297 flow_rule_match_ipv4_addrs(entry1->rule, &match1); 298 flow_rule_match_ipv4_addrs(entry2->rule, &match2); 299 300 memcpy(buf, match1.key, sizeof(*match1.key)); 301 match1.key = get_mangled_key(entry1->rule, buf, 302 offsetof(struct iphdr, saddr), 303 sizeof(*match1.key), 304 FLOW_ACT_MANGLE_HDR_TYPE_IP4); 305 306 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 307 if (out) 308 goto check_failed; 309 } 310 311 /* if pre ct entry do nat, the nat ip exists in nft entry, 312 * will be do merge check when do nft and post ct merge, 313 * so skip this ip merge check here. 314 */ 315 if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) && 316 nfp_ct_merge_check_cannot_skip(entry1, entry2)) { 317 struct flow_match_ipv6_addrs match1, match2; 318 319 flow_rule_match_ipv6_addrs(entry1->rule, &match1); 320 flow_rule_match_ipv6_addrs(entry2->rule, &match2); 321 322 memcpy(buf, match1.key, sizeof(*match1.key)); 323 match1.key = get_mangled_key(entry1->rule, buf, 324 offsetof(struct ipv6hdr, saddr), 325 sizeof(*match1.key), 326 FLOW_ACT_MANGLE_HDR_TYPE_IP6); 327 328 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 329 if (out) 330 goto check_failed; 331 } 332 333 /* if pre ct entry do nat, the nat tport exists in nft entry, 334 * will be do merge check when do nft and post ct merge, 335 * so skip this tport merge check here. 336 */ 337 if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_PORTS)) && 338 nfp_ct_merge_check_cannot_skip(entry1, entry2)) { 339 enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC; 340 struct flow_match_ports match1, match2; 341 342 flow_rule_match_ports(entry1->rule, &match1); 343 flow_rule_match_ports(entry2->rule, &match2); 344 345 if (ip_proto == IPPROTO_UDP) 346 htype = FLOW_ACT_MANGLE_HDR_TYPE_UDP; 347 else if (ip_proto == IPPROTO_TCP) 348 htype = FLOW_ACT_MANGLE_HDR_TYPE_TCP; 349 350 memcpy(buf, match1.key, sizeof(*match1.key)); 351 match1.key = get_mangled_key(entry1->rule, buf, 0, 352 sizeof(*match1.key), htype); 353 354 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 355 if (out) 356 goto check_failed; 357 } 358 359 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 360 struct flow_match_eth_addrs match1, match2; 361 362 flow_rule_match_eth_addrs(entry1->rule, &match1); 363 flow_rule_match_eth_addrs(entry2->rule, &match2); 364 365 memcpy(buf, match1.key, sizeof(*match1.key)); 366 match1.key = get_mangled_key(entry1->rule, buf, 0, 367 sizeof(*match1.key), 368 FLOW_ACT_MANGLE_HDR_TYPE_ETH); 369 370 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 371 if (out) 372 goto check_failed; 373 } 374 375 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN)) { 376 struct flow_match_vlan match1, match2; 377 378 flow_rule_match_vlan(entry1->rule, &match1); 379 flow_rule_match_vlan(entry2->rule, &match2); 380 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 381 if (out) 382 goto check_failed; 383 } 384 385 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_MPLS)) { 386 struct flow_match_mpls match1, match2; 387 388 flow_rule_match_mpls(entry1->rule, &match1); 389 flow_rule_match_mpls(entry2->rule, &match2); 390 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 391 if (out) 392 goto check_failed; 393 } 394 395 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_TCP)) { 396 struct flow_match_tcp match1, match2; 397 398 flow_rule_match_tcp(entry1->rule, &match1); 399 flow_rule_match_tcp(entry2->rule, &match2); 400 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 401 if (out) 402 goto check_failed; 403 } 404 405 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IP)) { 406 struct flow_match_ip match1, match2; 407 408 flow_rule_match_ip(entry1->rule, &match1); 409 flow_rule_match_ip(entry2->rule, &match2); 410 411 match1.key = get_mangled_tos_ttl(entry1->rule, buf, is_v6); 412 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 413 if (out) 414 goto check_failed; 415 } 416 417 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID)) { 418 struct flow_match_enc_keyid match1, match2; 419 420 flow_rule_match_enc_keyid(entry1->rule, &match1); 421 flow_rule_match_enc_keyid(entry2->rule, &match2); 422 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 423 if (out) 424 goto check_failed; 425 } 426 427 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { 428 struct flow_match_ipv4_addrs match1, match2; 429 430 flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1); 431 flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2); 432 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 433 if (out) 434 goto check_failed; 435 } 436 437 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { 438 struct flow_match_ipv6_addrs match1, match2; 439 440 flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1); 441 flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2); 442 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 443 if (out) 444 goto check_failed; 445 } 446 447 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 448 struct flow_match_control match1, match2; 449 450 flow_rule_match_enc_control(entry1->rule, &match1); 451 flow_rule_match_enc_control(entry2->rule, &match2); 452 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 453 if (out) 454 goto check_failed; 455 } 456 457 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP)) { 458 struct flow_match_ip match1, match2; 459 460 flow_rule_match_enc_ip(entry1->rule, &match1); 461 flow_rule_match_enc_ip(entry2->rule, &match2); 462 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 463 if (out) 464 goto check_failed; 465 } 466 467 if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS)) { 468 struct flow_match_enc_opts match1, match2; 469 470 flow_rule_match_enc_opts(entry1->rule, &match1); 471 flow_rule_match_enc_opts(entry2->rule, &match2); 472 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 473 if (out) 474 goto check_failed; 475 } 476 477 return 0; 478 479 check_failed: 480 return -EINVAL; 481 } 482 483 static int nfp_ct_check_vlan_merge(struct flow_action_entry *a_in, 484 struct flow_rule *rule) 485 { 486 struct flow_match_vlan match; 487 488 if (unlikely(flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN))) 489 return -EOPNOTSUPP; 490 491 /* post_ct does not match VLAN KEY, can be merged. */ 492 if (likely(!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN))) 493 return 0; 494 495 switch (a_in->id) { 496 /* pre_ct has pop vlan, post_ct cannot match VLAN KEY, cannot be merged. */ 497 case FLOW_ACTION_VLAN_POP: 498 return -EOPNOTSUPP; 499 500 case FLOW_ACTION_VLAN_PUSH: 501 case FLOW_ACTION_VLAN_MANGLE: 502 flow_rule_match_vlan(rule, &match); 503 /* different vlan id, cannot be merged. */ 504 if ((match.key->vlan_id & match.mask->vlan_id) ^ 505 (a_in->vlan.vid & match.mask->vlan_id)) 506 return -EOPNOTSUPP; 507 508 /* different tpid, cannot be merged. */ 509 if ((match.key->vlan_tpid & match.mask->vlan_tpid) ^ 510 (a_in->vlan.proto & match.mask->vlan_tpid)) 511 return -EOPNOTSUPP; 512 513 /* different priority, cannot be merged. */ 514 if ((match.key->vlan_priority & match.mask->vlan_priority) ^ 515 (a_in->vlan.prio & match.mask->vlan_priority)) 516 return -EOPNOTSUPP; 517 518 break; 519 default: 520 return -EOPNOTSUPP; 521 } 522 523 return 0; 524 } 525 526 /* Extra check for multiple ct-zones merge 527 * currently surpport nft entries merge check in different zones 528 */ 529 static int nfp_ct_merge_extra_check(struct nfp_fl_ct_flow_entry *nft_entry, 530 struct nfp_fl_ct_tc_merge *tc_m_entry) 531 { 532 struct nfp_fl_nft_tc_merge *prev_nft_m_entry; 533 struct nfp_fl_ct_flow_entry *pre_ct_entry; 534 535 pre_ct_entry = tc_m_entry->pre_ct_parent; 536 prev_nft_m_entry = pre_ct_entry->prev_m_entries[pre_ct_entry->num_prev_m_entries - 1]; 537 538 return nfp_ct_merge_check(prev_nft_m_entry->nft_parent, nft_entry); 539 } 540 541 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry, 542 struct nfp_fl_ct_flow_entry *post_ct_entry, 543 struct nfp_fl_ct_flow_entry *nft_entry) 544 { 545 struct flow_action_entry *act; 546 int i, err; 547 548 /* Check for pre_ct->action conflicts */ 549 flow_action_for_each(i, act, &pre_ct_entry->rule->action) { 550 switch (act->id) { 551 case FLOW_ACTION_VLAN_PUSH: 552 case FLOW_ACTION_VLAN_POP: 553 case FLOW_ACTION_VLAN_MANGLE: 554 err = nfp_ct_check_vlan_merge(act, post_ct_entry->rule); 555 if (err) 556 return err; 557 break; 558 case FLOW_ACTION_MPLS_PUSH: 559 case FLOW_ACTION_MPLS_POP: 560 case FLOW_ACTION_MPLS_MANGLE: 561 return -EOPNOTSUPP; 562 default: 563 break; 564 } 565 } 566 567 /* Check for nft->action conflicts */ 568 flow_action_for_each(i, act, &nft_entry->rule->action) { 569 switch (act->id) { 570 case FLOW_ACTION_VLAN_PUSH: 571 case FLOW_ACTION_VLAN_POP: 572 case FLOW_ACTION_VLAN_MANGLE: 573 case FLOW_ACTION_MPLS_PUSH: 574 case FLOW_ACTION_MPLS_POP: 575 case FLOW_ACTION_MPLS_MANGLE: 576 return -EOPNOTSUPP; 577 default: 578 break; 579 } 580 } 581 return 0; 582 } 583 584 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry, 585 struct nfp_fl_ct_flow_entry *nft_entry) 586 { 587 struct flow_dissector *dissector = post_ct_entry->rule->match.dissector; 588 struct flow_action_entry *ct_met; 589 struct flow_match_ct ct; 590 int i; 591 592 ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA); 593 if (ct_met && (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT))) { 594 u32 *act_lbl; 595 596 act_lbl = ct_met->ct_metadata.labels; 597 flow_rule_match_ct(post_ct_entry->rule, &ct); 598 for (i = 0; i < 4; i++) { 599 if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^ 600 (act_lbl[i] & ct.mask->ct_labels[i])) 601 return -EINVAL; 602 } 603 604 if ((ct.key->ct_mark & ct.mask->ct_mark) ^ 605 (ct_met->ct_metadata.mark & ct.mask->ct_mark)) 606 return -EINVAL; 607 608 return 0; 609 } else { 610 /* post_ct with ct clear action will not match the 611 * ct status when nft is nat entry. 612 */ 613 if (nft_entry->flags & NFP_FL_ACTION_DO_MANGLE) 614 return 0; 615 } 616 617 return -EINVAL; 618 } 619 620 static int 621 nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls, uint16_t *map) 622 { 623 int key_size; 624 625 /* This field must always be present */ 626 key_size = sizeof(struct nfp_flower_meta_tci); 627 map[FLOW_PAY_META_TCI] = 0; 628 629 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_EXT_META) { 630 map[FLOW_PAY_EXT_META] = key_size; 631 key_size += sizeof(struct nfp_flower_ext_meta); 632 } 633 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_PORT) { 634 map[FLOW_PAY_INPORT] = key_size; 635 key_size += sizeof(struct nfp_flower_in_port); 636 } 637 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_MAC) { 638 map[FLOW_PAY_MAC_MPLS] = key_size; 639 key_size += sizeof(struct nfp_flower_mac_mpls); 640 } 641 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_TP) { 642 map[FLOW_PAY_L4] = key_size; 643 key_size += sizeof(struct nfp_flower_tp_ports); 644 } 645 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV4) { 646 map[FLOW_PAY_IPV4] = key_size; 647 key_size += sizeof(struct nfp_flower_ipv4); 648 } 649 if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV6) { 650 map[FLOW_PAY_IPV6] = key_size; 651 key_size += sizeof(struct nfp_flower_ipv6); 652 } 653 654 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_QINQ) { 655 map[FLOW_PAY_QINQ] = key_size; 656 key_size += sizeof(struct nfp_flower_vlan); 657 } 658 659 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GRE) { 660 map[FLOW_PAY_GRE] = key_size; 661 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) 662 key_size += sizeof(struct nfp_flower_ipv6_gre_tun); 663 else 664 key_size += sizeof(struct nfp_flower_ipv4_gre_tun); 665 } 666 667 if ((in_key_ls.key_layer & NFP_FLOWER_LAYER_VXLAN) || 668 (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE)) { 669 map[FLOW_PAY_UDP_TUN] = key_size; 670 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) 671 key_size += sizeof(struct nfp_flower_ipv6_udp_tun); 672 else 673 key_size += sizeof(struct nfp_flower_ipv4_udp_tun); 674 } 675 676 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) { 677 map[FLOW_PAY_GENEVE_OPT] = key_size; 678 key_size += sizeof(struct nfp_flower_geneve_options); 679 } 680 681 return key_size; 682 } 683 684 /* get the csum flag according the ip proto and mangle action. */ 685 static void nfp_fl_get_csum_flag(struct flow_action_entry *a_in, u8 ip_proto, u32 *csum) 686 { 687 if (a_in->id != FLOW_ACTION_MANGLE) 688 return; 689 690 switch (a_in->mangle.htype) { 691 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 692 *csum |= TCA_CSUM_UPDATE_FLAG_IPV4HDR; 693 if (ip_proto == IPPROTO_TCP) 694 *csum |= TCA_CSUM_UPDATE_FLAG_TCP; 695 else if (ip_proto == IPPROTO_UDP) 696 *csum |= TCA_CSUM_UPDATE_FLAG_UDP; 697 break; 698 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 699 *csum |= TCA_CSUM_UPDATE_FLAG_TCP; 700 break; 701 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 702 *csum |= TCA_CSUM_UPDATE_FLAG_UDP; 703 break; 704 default: 705 break; 706 } 707 } 708 709 static int nfp_fl_merge_actions_offload(struct flow_rule **rules, 710 struct nfp_flower_priv *priv, 711 struct net_device *netdev, 712 struct nfp_fl_payload *flow_pay, 713 int num_rules) 714 { 715 enum flow_action_hw_stats tmp_stats = FLOW_ACTION_HW_STATS_DONT_CARE; 716 struct flow_action_entry *a_in; 717 int i, j, id, num_actions = 0; 718 struct flow_rule *a_rule; 719 int err = 0, offset = 0; 720 721 for (i = 0; i < num_rules; i++) 722 num_actions += rules[i]->action.num_entries; 723 724 /* Add one action to make sure there is enough room to add an checksum action 725 * when do nat. 726 */ 727 a_rule = flow_rule_alloc(num_actions + (num_rules / 2)); 728 if (!a_rule) 729 return -ENOMEM; 730 731 /* post_ct entry have one action at least. */ 732 if (rules[num_rules - 1]->action.num_entries != 0) 733 tmp_stats = rules[num_rules - 1]->action.entries[0].hw_stats; 734 735 /* Actions need a BASIC dissector. */ 736 a_rule->match = rules[0]->match; 737 738 /* Copy actions */ 739 for (j = 0; j < num_rules; j++) { 740 u32 csum_updated = 0; 741 u8 ip_proto = 0; 742 743 if (flow_rule_match_key(rules[j], FLOW_DISSECTOR_KEY_BASIC)) { 744 struct flow_match_basic match; 745 746 /* ip_proto is the only field that is needed in later compile_action, 747 * needed to set the correct checksum flags. It doesn't really matter 748 * which input rule's ip_proto field we take as the earlier merge checks 749 * would have made sure that they don't conflict. We do not know which 750 * of the subflows would have the ip_proto filled in, so we need to iterate 751 * through the subflows and assign the proper subflow to a_rule 752 */ 753 flow_rule_match_basic(rules[j], &match); 754 if (match.mask->ip_proto) { 755 a_rule->match = rules[j]->match; 756 ip_proto = match.key->ip_proto; 757 } 758 } 759 760 for (i = 0; i < rules[j]->action.num_entries; i++) { 761 a_in = &rules[j]->action.entries[i]; 762 id = a_in->id; 763 764 /* Ignore CT related actions as these would already have 765 * been taken care of by previous checks, and we do not send 766 * any CT actions to the firmware. 767 */ 768 switch (id) { 769 case FLOW_ACTION_CT: 770 case FLOW_ACTION_GOTO: 771 case FLOW_ACTION_CT_METADATA: 772 continue; 773 default: 774 /* nft entry is generated by tc ct, which mangle action do not care 775 * the stats, inherit the post entry stats to meet the 776 * flow_action_hw_stats_check. 777 * nft entry flow rules are at odd array index. 778 */ 779 if (j & 0x01) { 780 if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE) 781 a_in->hw_stats = tmp_stats; 782 nfp_fl_get_csum_flag(a_in, ip_proto, &csum_updated); 783 } 784 memcpy(&a_rule->action.entries[offset++], 785 a_in, sizeof(struct flow_action_entry)); 786 break; 787 } 788 } 789 /* nft entry have mangle action, but do not have checksum action when do NAT, 790 * hardware will automatically fix IPv4 and TCP/UDP checksum. so add an csum action 791 * to meet csum action check. 792 */ 793 if (csum_updated) { 794 struct flow_action_entry *csum_action; 795 796 csum_action = &a_rule->action.entries[offset++]; 797 csum_action->id = FLOW_ACTION_CSUM; 798 csum_action->csum_flags = csum_updated; 799 csum_action->hw_stats = tmp_stats; 800 } 801 } 802 803 /* Some actions would have been ignored, so update the num_entries field */ 804 a_rule->action.num_entries = offset; 805 err = nfp_flower_compile_action(priv->app, a_rule, netdev, flow_pay, NULL); 806 kfree(a_rule); 807 808 return err; 809 } 810 811 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry) 812 { 813 enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE; 814 struct nfp_fl_ct_zone_entry *zt = m_entry->zt; 815 struct flow_rule *rules[NFP_MAX_ENTRY_RULES]; 816 struct nfp_fl_ct_flow_entry *pre_ct_entry; 817 struct nfp_fl_key_ls key_layer, tmp_layer; 818 struct nfp_flower_priv *priv = zt->priv; 819 u16 key_map[_FLOW_PAY_LAYERS_MAX]; 820 struct nfp_fl_payload *flow_pay; 821 u8 *key, *msk, *kdata, *mdata; 822 struct nfp_port *port = NULL; 823 int num_rules, err, i, j = 0; 824 struct net_device *netdev; 825 bool qinq_sup; 826 u32 port_id; 827 u16 offset; 828 829 netdev = m_entry->netdev; 830 qinq_sup = !!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_QINQ); 831 832 pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent; 833 num_rules = pre_ct_entry->num_prev_m_entries * 2 + _CT_TYPE_MAX; 834 835 for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) { 836 rules[j++] = pre_ct_entry->prev_m_entries[i]->tc_m_parent->pre_ct_parent->rule; 837 rules[j++] = pre_ct_entry->prev_m_entries[i]->nft_parent->rule; 838 } 839 840 rules[j++] = m_entry->tc_m_parent->pre_ct_parent->rule; 841 rules[j++] = m_entry->nft_parent->rule; 842 rules[j++] = m_entry->tc_m_parent->post_ct_parent->rule; 843 844 memset(&key_layer, 0, sizeof(struct nfp_fl_key_ls)); 845 memset(&key_map, 0, sizeof(key_map)); 846 847 /* Calculate the resultant key layer and size for offload */ 848 for (i = 0; i < num_rules; i++) { 849 err = nfp_flower_calculate_key_layers(priv->app, 850 m_entry->netdev, 851 &tmp_layer, rules[i], 852 &tun_type, NULL); 853 if (err) 854 return err; 855 856 key_layer.key_layer |= tmp_layer.key_layer; 857 key_layer.key_layer_two |= tmp_layer.key_layer_two; 858 } 859 key_layer.key_size = nfp_fl_calc_key_layers_sz(key_layer, key_map); 860 861 flow_pay = nfp_flower_allocate_new(&key_layer); 862 if (!flow_pay) 863 return -ENOMEM; 864 865 memset(flow_pay->unmasked_data, 0, key_layer.key_size); 866 memset(flow_pay->mask_data, 0, key_layer.key_size); 867 868 kdata = flow_pay->unmasked_data; 869 mdata = flow_pay->mask_data; 870 871 offset = key_map[FLOW_PAY_META_TCI]; 872 key = kdata + offset; 873 msk = mdata + offset; 874 nfp_flower_compile_meta((struct nfp_flower_meta_tci *)key, 875 (struct nfp_flower_meta_tci *)msk, 876 key_layer.key_layer); 877 878 if (NFP_FLOWER_LAYER_EXT_META & key_layer.key_layer) { 879 offset = key_map[FLOW_PAY_EXT_META]; 880 key = kdata + offset; 881 msk = mdata + offset; 882 nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)key, 883 key_layer.key_layer_two); 884 nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)msk, 885 key_layer.key_layer_two); 886 } 887 888 /* Using in_port from the -trk rule. The tc merge checks should already 889 * be checking that the ingress netdevs are the same 890 */ 891 port_id = nfp_flower_get_port_id_from_netdev(priv->app, netdev); 892 offset = key_map[FLOW_PAY_INPORT]; 893 key = kdata + offset; 894 msk = mdata + offset; 895 err = nfp_flower_compile_port((struct nfp_flower_in_port *)key, 896 port_id, false, tun_type, NULL); 897 if (err) 898 goto ct_offload_err; 899 err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk, 900 port_id, true, tun_type, NULL); 901 if (err) 902 goto ct_offload_err; 903 904 /* This following part works on the assumption that previous checks has 905 * already filtered out flows that has different values for the different 906 * layers. Here we iterate through all three rules and merge their respective 907 * masked value(cared bits), basic method is: 908 * final_key = (r1_key & r1_mask) | (r2_key & r2_mask) | (r3_key & r3_mask) 909 * final_mask = r1_mask | r2_mask | r3_mask 910 * If none of the rules contains a match that is also fine, that simply means 911 * that the layer is not present. 912 */ 913 if (!qinq_sup) { 914 for (i = 0; i < num_rules; i++) { 915 offset = key_map[FLOW_PAY_META_TCI]; 916 key = kdata + offset; 917 msk = mdata + offset; 918 nfp_flower_compile_tci((struct nfp_flower_meta_tci *)key, 919 (struct nfp_flower_meta_tci *)msk, 920 rules[i]); 921 } 922 } 923 924 if (NFP_FLOWER_LAYER_MAC & key_layer.key_layer) { 925 offset = key_map[FLOW_PAY_MAC_MPLS]; 926 key = kdata + offset; 927 msk = mdata + offset; 928 for (i = 0; i < num_rules; i++) { 929 nfp_flower_compile_mac((struct nfp_flower_mac_mpls *)key, 930 (struct nfp_flower_mac_mpls *)msk, 931 rules[i]); 932 err = nfp_flower_compile_mpls((struct nfp_flower_mac_mpls *)key, 933 (struct nfp_flower_mac_mpls *)msk, 934 rules[i], NULL); 935 if (err) 936 goto ct_offload_err; 937 } 938 } 939 940 if (NFP_FLOWER_LAYER_IPV4 & key_layer.key_layer) { 941 offset = key_map[FLOW_PAY_IPV4]; 942 key = kdata + offset; 943 msk = mdata + offset; 944 for (i = 0; i < num_rules; i++) { 945 nfp_flower_compile_ipv4((struct nfp_flower_ipv4 *)key, 946 (struct nfp_flower_ipv4 *)msk, 947 rules[i]); 948 } 949 } 950 951 if (NFP_FLOWER_LAYER_IPV6 & key_layer.key_layer) { 952 offset = key_map[FLOW_PAY_IPV6]; 953 key = kdata + offset; 954 msk = mdata + offset; 955 for (i = 0; i < num_rules; i++) { 956 nfp_flower_compile_ipv6((struct nfp_flower_ipv6 *)key, 957 (struct nfp_flower_ipv6 *)msk, 958 rules[i]); 959 } 960 } 961 962 if (NFP_FLOWER_LAYER_TP & key_layer.key_layer) { 963 offset = key_map[FLOW_PAY_L4]; 964 key = kdata + offset; 965 msk = mdata + offset; 966 for (i = 0; i < num_rules; i++) { 967 nfp_flower_compile_tport((struct nfp_flower_tp_ports *)key, 968 (struct nfp_flower_tp_ports *)msk, 969 rules[i]); 970 } 971 } 972 973 if (NFP_FLOWER_LAYER2_QINQ & key_layer.key_layer_two) { 974 offset = key_map[FLOW_PAY_QINQ]; 975 key = kdata + offset; 976 msk = mdata + offset; 977 for (i = 0; i < num_rules; i++) { 978 nfp_flower_compile_vlan((struct nfp_flower_vlan *)key, 979 (struct nfp_flower_vlan *)msk, 980 rules[i]); 981 } 982 } 983 984 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GRE) { 985 offset = key_map[FLOW_PAY_GRE]; 986 key = kdata + offset; 987 msk = mdata + offset; 988 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) { 989 struct nfp_flower_ipv6_gre_tun *gre_match; 990 struct nfp_ipv6_addr_entry *entry; 991 struct in6_addr *dst; 992 993 for (i = 0; i < num_rules; i++) { 994 nfp_flower_compile_ipv6_gre_tun((void *)key, 995 (void *)msk, rules[i]); 996 } 997 gre_match = (struct nfp_flower_ipv6_gre_tun *)key; 998 dst = &gre_match->ipv6.dst; 999 1000 entry = nfp_tunnel_add_ipv6_off(priv->app, dst); 1001 if (!entry) { 1002 err = -ENOMEM; 1003 goto ct_offload_err; 1004 } 1005 1006 flow_pay->nfp_tun_ipv6 = entry; 1007 } else { 1008 __be32 dst; 1009 1010 for (i = 0; i < num_rules; i++) { 1011 nfp_flower_compile_ipv4_gre_tun((void *)key, 1012 (void *)msk, rules[i]); 1013 } 1014 dst = ((struct nfp_flower_ipv4_gre_tun *)key)->ipv4.dst; 1015 1016 /* Store the tunnel destination in the rule data. 1017 * This must be present and be an exact match. 1018 */ 1019 flow_pay->nfp_tun_ipv4_addr = dst; 1020 nfp_tunnel_add_ipv4_off(priv->app, dst); 1021 } 1022 } 1023 1024 if (key_layer.key_layer & NFP_FLOWER_LAYER_VXLAN || 1025 key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE) { 1026 offset = key_map[FLOW_PAY_UDP_TUN]; 1027 key = kdata + offset; 1028 msk = mdata + offset; 1029 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) { 1030 struct nfp_flower_ipv6_udp_tun *udp_match; 1031 struct nfp_ipv6_addr_entry *entry; 1032 struct in6_addr *dst; 1033 1034 for (i = 0; i < num_rules; i++) { 1035 nfp_flower_compile_ipv6_udp_tun((void *)key, 1036 (void *)msk, rules[i]); 1037 } 1038 udp_match = (struct nfp_flower_ipv6_udp_tun *)key; 1039 dst = &udp_match->ipv6.dst; 1040 1041 entry = nfp_tunnel_add_ipv6_off(priv->app, dst); 1042 if (!entry) { 1043 err = -ENOMEM; 1044 goto ct_offload_err; 1045 } 1046 1047 flow_pay->nfp_tun_ipv6 = entry; 1048 } else { 1049 __be32 dst; 1050 1051 for (i = 0; i < num_rules; i++) { 1052 nfp_flower_compile_ipv4_udp_tun((void *)key, 1053 (void *)msk, rules[i]); 1054 } 1055 dst = ((struct nfp_flower_ipv4_udp_tun *)key)->ipv4.dst; 1056 1057 /* Store the tunnel destination in the rule data. 1058 * This must be present and be an exact match. 1059 */ 1060 flow_pay->nfp_tun_ipv4_addr = dst; 1061 nfp_tunnel_add_ipv4_off(priv->app, dst); 1062 } 1063 1064 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) { 1065 offset = key_map[FLOW_PAY_GENEVE_OPT]; 1066 key = kdata + offset; 1067 msk = mdata + offset; 1068 for (i = 0; i < num_rules; i++) 1069 nfp_flower_compile_geneve_opt(key, msk, rules[i]); 1070 } 1071 } 1072 1073 /* Merge actions into flow_pay */ 1074 err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay, num_rules); 1075 if (err) 1076 goto ct_offload_err; 1077 1078 /* Use the pointer address as the cookie, but set the last bit to 1. 1079 * This is to avoid the 'is_merge_flow' check from detecting this as 1080 * an already merged flow. This works since address alignment means 1081 * that the last bit for pointer addresses will be 0. 1082 */ 1083 flow_pay->tc_flower_cookie = ((unsigned long)flow_pay) | 0x1; 1084 err = nfp_compile_flow_metadata(priv->app, flow_pay->tc_flower_cookie, 1085 flow_pay, netdev, NULL); 1086 if (err) 1087 goto ct_offload_err; 1088 1089 if (nfp_netdev_is_nfp_repr(netdev)) 1090 port = nfp_port_from_netdev(netdev); 1091 1092 err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node, 1093 nfp_flower_table_params); 1094 if (err) 1095 goto ct_release_offload_meta_err; 1096 1097 err = nfp_flower_xmit_flow(priv->app, flow_pay, 1098 NFP_FLOWER_CMSG_TYPE_FLOW_ADD); 1099 if (err) 1100 goto ct_remove_rhash_err; 1101 1102 m_entry->tc_flower_cookie = flow_pay->tc_flower_cookie; 1103 m_entry->flow_pay = flow_pay; 1104 1105 if (port) 1106 port->tc_offload_cnt++; 1107 1108 return err; 1109 1110 ct_remove_rhash_err: 1111 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table, 1112 &flow_pay->fl_node, 1113 nfp_flower_table_params)); 1114 ct_release_offload_meta_err: 1115 nfp_modify_flow_metadata(priv->app, flow_pay); 1116 ct_offload_err: 1117 if (flow_pay->nfp_tun_ipv4_addr) 1118 nfp_tunnel_del_ipv4_off(priv->app, flow_pay->nfp_tun_ipv4_addr); 1119 if (flow_pay->nfp_tun_ipv6) 1120 nfp_tunnel_put_ipv6_off(priv->app, flow_pay->nfp_tun_ipv6); 1121 kfree(flow_pay->action_data); 1122 kfree(flow_pay->mask_data); 1123 kfree(flow_pay->unmasked_data); 1124 kfree(flow_pay); 1125 return err; 1126 } 1127 1128 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie, 1129 struct net_device *netdev) 1130 { 1131 struct nfp_flower_priv *priv = app->priv; 1132 struct nfp_fl_payload *flow_pay; 1133 struct nfp_port *port = NULL; 1134 int err = 0; 1135 1136 if (nfp_netdev_is_nfp_repr(netdev)) 1137 port = nfp_port_from_netdev(netdev); 1138 1139 flow_pay = nfp_flower_search_fl_table(app, cookie, netdev); 1140 if (!flow_pay) 1141 return -ENOENT; 1142 1143 err = nfp_modify_flow_metadata(app, flow_pay); 1144 if (err) 1145 goto err_free_merge_flow; 1146 1147 if (flow_pay->nfp_tun_ipv4_addr) 1148 nfp_tunnel_del_ipv4_off(app, flow_pay->nfp_tun_ipv4_addr); 1149 1150 if (flow_pay->nfp_tun_ipv6) 1151 nfp_tunnel_put_ipv6_off(app, flow_pay->nfp_tun_ipv6); 1152 1153 if (!flow_pay->in_hw) { 1154 err = 0; 1155 goto err_free_merge_flow; 1156 } 1157 1158 err = nfp_flower_xmit_flow(app, flow_pay, 1159 NFP_FLOWER_CMSG_TYPE_FLOW_DEL); 1160 1161 err_free_merge_flow: 1162 nfp_flower_del_linked_merge_flows(app, flow_pay); 1163 if (port) 1164 port->tc_offload_cnt--; 1165 kfree(flow_pay->action_data); 1166 kfree(flow_pay->mask_data); 1167 kfree(flow_pay->unmasked_data); 1168 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table, 1169 &flow_pay->fl_node, 1170 nfp_flower_table_params)); 1171 kfree_rcu(flow_pay, rcu); 1172 return err; 1173 } 1174 1175 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt, 1176 struct nfp_fl_ct_flow_entry *nft_entry, 1177 struct nfp_fl_ct_tc_merge *tc_m_entry) 1178 { 1179 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry; 1180 struct nfp_fl_nft_tc_merge *nft_m_entry; 1181 unsigned long new_cookie[3]; 1182 int err; 1183 1184 pre_ct_entry = tc_m_entry->pre_ct_parent; 1185 post_ct_entry = tc_m_entry->post_ct_parent; 1186 1187 err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry); 1188 if (err) 1189 return err; 1190 1191 /* Check that the two tc flows are also compatible with 1192 * the nft entry. No need to check the pre_ct and post_ct 1193 * entries as that was already done during pre_merge. 1194 * The nft entry does not have a chain populated, so 1195 * skip this check. 1196 */ 1197 err = nfp_ct_merge_check(pre_ct_entry, nft_entry); 1198 if (err) 1199 return err; 1200 err = nfp_ct_merge_check(nft_entry, post_ct_entry); 1201 if (err) 1202 return err; 1203 err = nfp_ct_check_meta(post_ct_entry, nft_entry); 1204 if (err) 1205 return err; 1206 1207 if (pre_ct_entry->num_prev_m_entries > 0) { 1208 err = nfp_ct_merge_extra_check(nft_entry, tc_m_entry); 1209 if (err) 1210 return err; 1211 } 1212 1213 /* Combine tc_merge and nft cookies for this cookie. */ 1214 new_cookie[0] = tc_m_entry->cookie[0]; 1215 new_cookie[1] = tc_m_entry->cookie[1]; 1216 new_cookie[2] = nft_entry->cookie; 1217 nft_m_entry = get_hashentry(&zt->nft_merge_tb, 1218 &new_cookie, 1219 nfp_nft_ct_merge_params, 1220 sizeof(*nft_m_entry)); 1221 1222 if (IS_ERR(nft_m_entry)) 1223 return PTR_ERR(nft_m_entry); 1224 1225 /* nft_m_entry already present, not merging again */ 1226 if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie))) 1227 return 0; 1228 1229 memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie)); 1230 nft_m_entry->zt = zt; 1231 nft_m_entry->tc_m_parent = tc_m_entry; 1232 nft_m_entry->nft_parent = nft_entry; 1233 nft_m_entry->tc_flower_cookie = 0; 1234 /* Copy the netdev from the pre_ct entry. When the tc_m_entry was created 1235 * it only combined them if the netdevs were the same, so can use any of them. 1236 */ 1237 nft_m_entry->netdev = pre_ct_entry->netdev; 1238 1239 /* Add this entry to the tc_m_list and nft_flow lists */ 1240 list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children); 1241 list_add(&nft_m_entry->nft_flow_list, &nft_entry->children); 1242 1243 err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node, 1244 nfp_nft_ct_merge_params); 1245 if (err) 1246 goto err_nft_ct_merge_insert; 1247 1248 zt->nft_merge_count++; 1249 1250 if (post_ct_entry->goto_chain_index > 0) 1251 return nfp_fl_create_new_pre_ct(nft_m_entry); 1252 1253 /* Generate offload structure and send to nfp */ 1254 err = nfp_fl_ct_add_offload(nft_m_entry); 1255 if (err) 1256 goto err_nft_ct_offload; 1257 1258 return err; 1259 1260 err_nft_ct_offload: 1261 nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie, 1262 nft_m_entry->netdev); 1263 WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb, 1264 &nft_m_entry->hash_node, 1265 nfp_nft_ct_merge_params)); 1266 zt->nft_merge_count--; 1267 err_nft_ct_merge_insert: 1268 list_del(&nft_m_entry->tc_merge_list); 1269 list_del(&nft_m_entry->nft_flow_list); 1270 kfree(nft_m_entry); 1271 return err; 1272 } 1273 1274 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt, 1275 struct nfp_fl_ct_flow_entry *ct_entry1, 1276 struct nfp_fl_ct_flow_entry *ct_entry2) 1277 { 1278 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry; 1279 struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp; 1280 struct nfp_fl_ct_tc_merge *m_entry; 1281 unsigned long new_cookie[2]; 1282 int err; 1283 1284 if (ct_entry1->type == CT_TYPE_PRE_CT) { 1285 pre_ct_entry = ct_entry1; 1286 post_ct_entry = ct_entry2; 1287 } else { 1288 post_ct_entry = ct_entry1; 1289 pre_ct_entry = ct_entry2; 1290 } 1291 1292 /* Checks that the chain_index of the filter matches the 1293 * chain_index of the GOTO action. 1294 */ 1295 if (post_ct_entry->chain_index != pre_ct_entry->goto_chain_index) 1296 return -EINVAL; 1297 1298 err = nfp_ct_merge_check(pre_ct_entry, post_ct_entry); 1299 if (err) 1300 return err; 1301 1302 new_cookie[0] = pre_ct_entry->cookie; 1303 new_cookie[1] = post_ct_entry->cookie; 1304 m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie, 1305 nfp_tc_ct_merge_params, sizeof(*m_entry)); 1306 if (IS_ERR(m_entry)) 1307 return PTR_ERR(m_entry); 1308 1309 /* m_entry already present, not merging again */ 1310 if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie))) 1311 return 0; 1312 1313 memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie)); 1314 m_entry->zt = zt; 1315 m_entry->post_ct_parent = post_ct_entry; 1316 m_entry->pre_ct_parent = pre_ct_entry; 1317 1318 /* Add this entry to the pre_ct and post_ct lists */ 1319 list_add(&m_entry->post_ct_list, &post_ct_entry->children); 1320 list_add(&m_entry->pre_ct_list, &pre_ct_entry->children); 1321 INIT_LIST_HEAD(&m_entry->children); 1322 1323 err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node, 1324 nfp_tc_ct_merge_params); 1325 if (err) 1326 goto err_ct_tc_merge_insert; 1327 zt->tc_merge_count++; 1328 1329 /* Merge with existing nft flows */ 1330 list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list, 1331 list_node) { 1332 nfp_ct_do_nft_merge(zt, nft_entry, m_entry); 1333 } 1334 1335 return 0; 1336 1337 err_ct_tc_merge_insert: 1338 list_del(&m_entry->post_ct_list); 1339 list_del(&m_entry->pre_ct_list); 1340 kfree(m_entry); 1341 return err; 1342 } 1343 1344 static struct 1345 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv, 1346 u16 zone, bool wildcarded) 1347 { 1348 struct nfp_fl_ct_zone_entry *zt; 1349 int err; 1350 1351 if (wildcarded && priv->ct_zone_wc) 1352 return priv->ct_zone_wc; 1353 1354 if (!wildcarded) { 1355 zt = get_hashentry(&priv->ct_zone_table, &zone, 1356 nfp_zone_table_params, sizeof(*zt)); 1357 1358 /* If priv is set this is an existing entry, just return it */ 1359 if (IS_ERR(zt) || zt->priv) 1360 return zt; 1361 } else { 1362 zt = kzalloc_obj(*zt); 1363 if (!zt) 1364 return ERR_PTR(-ENOMEM); 1365 } 1366 1367 zt->zone = zone; 1368 zt->priv = priv; 1369 zt->nft = NULL; 1370 1371 /* init the various hash tables and lists */ 1372 INIT_LIST_HEAD(&zt->pre_ct_list); 1373 INIT_LIST_HEAD(&zt->post_ct_list); 1374 INIT_LIST_HEAD(&zt->nft_flows_list); 1375 1376 err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params); 1377 if (err) 1378 goto err_tc_merge_tb_init; 1379 1380 err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params); 1381 if (err) 1382 goto err_nft_merge_tb_init; 1383 1384 if (wildcarded) { 1385 priv->ct_zone_wc = zt; 1386 } else { 1387 err = rhashtable_insert_fast(&priv->ct_zone_table, 1388 &zt->hash_node, 1389 nfp_zone_table_params); 1390 if (err) 1391 goto err_zone_insert; 1392 } 1393 1394 return zt; 1395 1396 err_zone_insert: 1397 rhashtable_destroy(&zt->nft_merge_tb); 1398 err_nft_merge_tb_init: 1399 rhashtable_destroy(&zt->tc_merge_tb); 1400 err_tc_merge_tb_init: 1401 kfree(zt); 1402 return ERR_PTR(err); 1403 } 1404 1405 static struct net_device *get_netdev_from_rule(struct flow_rule *rule) 1406 { 1407 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) { 1408 struct flow_match_meta match; 1409 1410 flow_rule_match_meta(rule, &match); 1411 if (match.key->ingress_ifindex & match.mask->ingress_ifindex) 1412 return __dev_get_by_index(&init_net, 1413 match.key->ingress_ifindex); 1414 } 1415 1416 return NULL; 1417 } 1418 1419 static void nfp_nft_ct_translate_mangle_action(struct flow_action_entry *mangle_action) 1420 { 1421 if (mangle_action->id != FLOW_ACTION_MANGLE) 1422 return; 1423 1424 switch (mangle_action->mangle.htype) { 1425 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 1426 case FLOW_ACT_MANGLE_HDR_TYPE_IP6: 1427 mangle_action->mangle.val = (__force u32)cpu_to_be32(mangle_action->mangle.val); 1428 mangle_action->mangle.mask = (__force u32)cpu_to_be32(mangle_action->mangle.mask); 1429 return; 1430 1431 /* Both struct tcphdr and struct udphdr start with 1432 * __be16 source; 1433 * __be16 dest; 1434 * so we can use the same code for both. 1435 */ 1436 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 1437 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 1438 if (mangle_action->mangle.offset == offsetof(struct tcphdr, source)) { 1439 mangle_action->mangle.val = 1440 (__force u32)cpu_to_be32(mangle_action->mangle.val << 16); 1441 /* The mask of mangle action is inverse mask, 1442 * so clear the dest tp port with 0xFFFF to 1443 * instead of rotate-left operation. 1444 */ 1445 mangle_action->mangle.mask = 1446 (__force u32)cpu_to_be32(mangle_action->mangle.mask << 16 | 0xFFFF); 1447 } 1448 if (mangle_action->mangle.offset == offsetof(struct tcphdr, dest)) { 1449 mangle_action->mangle.offset = 0; 1450 mangle_action->mangle.val = 1451 (__force u32)cpu_to_be32(mangle_action->mangle.val); 1452 mangle_action->mangle.mask = 1453 (__force u32)cpu_to_be32(mangle_action->mangle.mask); 1454 } 1455 return; 1456 1457 default: 1458 return; 1459 } 1460 } 1461 1462 static int nfp_nft_ct_set_flow_flag(struct flow_action_entry *act, 1463 struct nfp_fl_ct_flow_entry *entry) 1464 { 1465 switch (act->id) { 1466 case FLOW_ACTION_CT: 1467 if (act->ct.action == TCA_CT_ACT_NAT) 1468 entry->flags |= NFP_FL_ACTION_DO_NAT; 1469 break; 1470 1471 case FLOW_ACTION_MANGLE: 1472 entry->flags |= NFP_FL_ACTION_DO_MANGLE; 1473 break; 1474 1475 default: 1476 break; 1477 } 1478 1479 return 0; 1480 } 1481 1482 static struct 1483 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt, 1484 struct net_device *netdev, 1485 struct flow_cls_offload *flow, 1486 bool is_nft, struct netlink_ext_ack *extack) 1487 { 1488 struct nf_flow_match *nft_match = NULL; 1489 struct nfp_fl_ct_flow_entry *entry; 1490 struct nfp_fl_ct_map_entry *map; 1491 struct flow_action_entry *act; 1492 int err, i; 1493 1494 entry = kzalloc_obj(*entry); 1495 if (!entry) 1496 return ERR_PTR(-ENOMEM); 1497 1498 entry->rule = flow_rule_alloc(flow->rule->action.num_entries); 1499 if (!entry->rule) { 1500 err = -ENOMEM; 1501 goto err_pre_ct_rule; 1502 } 1503 1504 /* nft flows gets destroyed after callback return, so need 1505 * to do a full copy instead of just a reference. 1506 */ 1507 if (is_nft) { 1508 nft_match = kzalloc_obj(*nft_match); 1509 if (!nft_match) { 1510 err = -ENOMEM; 1511 goto err_pre_ct_act; 1512 } 1513 memcpy(&nft_match->dissector, flow->rule->match.dissector, 1514 sizeof(nft_match->dissector)); 1515 memcpy(&nft_match->mask, flow->rule->match.mask, 1516 sizeof(nft_match->mask)); 1517 memcpy(&nft_match->key, flow->rule->match.key, 1518 sizeof(nft_match->key)); 1519 entry->rule->match.dissector = &nft_match->dissector; 1520 entry->rule->match.mask = &nft_match->mask; 1521 entry->rule->match.key = &nft_match->key; 1522 1523 if (!netdev) 1524 netdev = get_netdev_from_rule(entry->rule); 1525 } else { 1526 entry->rule->match.dissector = flow->rule->match.dissector; 1527 entry->rule->match.mask = flow->rule->match.mask; 1528 entry->rule->match.key = flow->rule->match.key; 1529 } 1530 1531 entry->zt = zt; 1532 entry->netdev = netdev; 1533 entry->cookie = flow->cookie > 0 ? flow->cookie : (unsigned long)entry; 1534 entry->chain_index = flow->common.chain_index; 1535 entry->tun_offset = NFP_FL_CT_NO_TUN; 1536 1537 /* Copy over action data. Unfortunately we do not get a handle to the 1538 * original tcf_action data, and the flow objects gets destroyed, so we 1539 * cannot just save a pointer to this either, so need to copy over the 1540 * data unfortunately. 1541 */ 1542 entry->rule->action.num_entries = flow->rule->action.num_entries; 1543 flow_action_for_each(i, act, &flow->rule->action) { 1544 struct flow_action_entry *new_act; 1545 1546 new_act = &entry->rule->action.entries[i]; 1547 memcpy(new_act, act, sizeof(struct flow_action_entry)); 1548 /* nft entry mangle field is host byte order, need translate to 1549 * network byte order. 1550 */ 1551 if (is_nft) 1552 nfp_nft_ct_translate_mangle_action(new_act); 1553 1554 nfp_nft_ct_set_flow_flag(new_act, entry); 1555 /* Entunnel is a special case, need to allocate and copy 1556 * tunnel info. 1557 */ 1558 if (act->id == FLOW_ACTION_TUNNEL_ENCAP) { 1559 struct ip_tunnel_info *tun = act->tunnel; 1560 size_t tun_size = sizeof(*tun) + tun->options_len; 1561 1562 new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC); 1563 if (!new_act->tunnel) { 1564 err = -ENOMEM; 1565 goto err_pre_ct_tun_cp; 1566 } 1567 entry->tun_offset = i; 1568 } 1569 } 1570 1571 INIT_LIST_HEAD(&entry->children); 1572 1573 if (flow->cookie == 0) 1574 return entry; 1575 1576 /* Now add a ct map entry to flower-priv */ 1577 map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie, 1578 nfp_ct_map_params, sizeof(*map)); 1579 if (IS_ERR(map)) { 1580 NL_SET_ERR_MSG_MOD(extack, 1581 "offload error: ct map entry creation failed"); 1582 err = -ENOMEM; 1583 goto err_ct_flow_insert; 1584 } 1585 map->cookie = flow->cookie; 1586 map->ct_entry = entry; 1587 err = rhashtable_insert_fast(&zt->priv->ct_map_table, 1588 &map->hash_node, 1589 nfp_ct_map_params); 1590 if (err) { 1591 NL_SET_ERR_MSG_MOD(extack, 1592 "offload error: ct map entry table add failed"); 1593 goto err_map_insert; 1594 } 1595 1596 return entry; 1597 1598 err_map_insert: 1599 kfree(map); 1600 err_ct_flow_insert: 1601 if (entry->tun_offset != NFP_FL_CT_NO_TUN) 1602 kfree(entry->rule->action.entries[entry->tun_offset].tunnel); 1603 err_pre_ct_tun_cp: 1604 kfree(nft_match); 1605 err_pre_ct_act: 1606 kfree(entry->rule); 1607 err_pre_ct_rule: 1608 kfree(entry); 1609 return ERR_PTR(err); 1610 } 1611 1612 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry) 1613 { 1614 struct nfp_fl_ct_zone_entry *zt; 1615 int err; 1616 1617 zt = m_entry->zt; 1618 1619 /* Flow is in HW, need to delete */ 1620 if (m_entry->tc_flower_cookie) { 1621 err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie, 1622 m_entry->netdev); 1623 if (err) 1624 return; 1625 } 1626 1627 WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb, 1628 &m_entry->hash_node, 1629 nfp_nft_ct_merge_params)); 1630 zt->nft_merge_count--; 1631 list_del(&m_entry->tc_merge_list); 1632 list_del(&m_entry->nft_flow_list); 1633 1634 if (m_entry->next_pre_ct_entry) { 1635 struct nfp_fl_ct_map_entry pre_ct_map_ent; 1636 1637 pre_ct_map_ent.ct_entry = m_entry->next_pre_ct_entry; 1638 pre_ct_map_ent.cookie = 0; 1639 nfp_fl_ct_del_flow(&pre_ct_map_ent); 1640 } 1641 1642 kfree(m_entry); 1643 } 1644 1645 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow) 1646 { 1647 struct nfp_fl_nft_tc_merge *m_entry, *tmp; 1648 1649 /* These post entries are parts of two lists, one is a list of nft_entries 1650 * and the other is of from a list of tc_merge structures. Iterate 1651 * through the relevant list and cleanup the entries. 1652 */ 1653 1654 if (is_nft_flow) { 1655 /* Need to iterate through list of nft_flow entries */ 1656 struct nfp_fl_ct_flow_entry *ct_entry = entry; 1657 1658 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, 1659 nft_flow_list) { 1660 cleanup_nft_merge_entry(m_entry); 1661 } 1662 } else { 1663 /* Need to iterate through list of tc_merged_flow entries */ 1664 struct nfp_fl_ct_tc_merge *ct_entry = entry; 1665 1666 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, 1667 tc_merge_list) { 1668 cleanup_nft_merge_entry(m_entry); 1669 } 1670 } 1671 } 1672 1673 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent) 1674 { 1675 struct nfp_fl_ct_zone_entry *zt; 1676 int err; 1677 1678 zt = m_ent->zt; 1679 err = rhashtable_remove_fast(&zt->tc_merge_tb, 1680 &m_ent->hash_node, 1681 nfp_tc_ct_merge_params); 1682 if (err) 1683 pr_warn("WARNING: could not remove merge_entry from hashtable\n"); 1684 zt->tc_merge_count--; 1685 list_del(&m_ent->post_ct_list); 1686 list_del(&m_ent->pre_ct_list); 1687 1688 if (!list_empty(&m_ent->children)) 1689 nfp_free_nft_merge_children(m_ent, false); 1690 kfree(m_ent); 1691 } 1692 1693 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry) 1694 { 1695 struct nfp_fl_ct_tc_merge *m_ent, *tmp; 1696 1697 switch (entry->type) { 1698 case CT_TYPE_PRE_CT: 1699 list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) { 1700 nfp_del_tc_merge_entry(m_ent); 1701 } 1702 break; 1703 case CT_TYPE_POST_CT: 1704 list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) { 1705 nfp_del_tc_merge_entry(m_ent); 1706 } 1707 break; 1708 default: 1709 break; 1710 } 1711 } 1712 1713 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry) 1714 { 1715 list_del(&entry->list_node); 1716 1717 if (!list_empty(&entry->children)) { 1718 if (entry->type == CT_TYPE_NFT) 1719 nfp_free_nft_merge_children(entry, true); 1720 else 1721 nfp_free_tc_merge_children(entry); 1722 } 1723 1724 if (entry->tun_offset != NFP_FL_CT_NO_TUN) 1725 kfree(entry->rule->action.entries[entry->tun_offset].tunnel); 1726 1727 if (entry->type == CT_TYPE_NFT) { 1728 struct nf_flow_match *nft_match; 1729 1730 nft_match = container_of(entry->rule->match.dissector, 1731 struct nf_flow_match, dissector); 1732 kfree(nft_match); 1733 } 1734 1735 kfree(entry->rule); 1736 kfree(entry); 1737 } 1738 1739 static struct flow_action_entry *get_flow_act_ct(struct flow_rule *rule) 1740 { 1741 struct flow_action_entry *act; 1742 int i; 1743 1744 /* More than one ct action may be present in a flow rule, 1745 * Return the first one that is not a CT clear action 1746 */ 1747 flow_action_for_each(i, act, &rule->action) { 1748 if (act->id == FLOW_ACTION_CT && act->ct.action != TCA_CT_ACT_CLEAR) 1749 return act; 1750 } 1751 1752 return NULL; 1753 } 1754 1755 static struct flow_action_entry *get_flow_act(struct flow_rule *rule, 1756 enum flow_action_id act_id) 1757 { 1758 struct flow_action_entry *act = NULL; 1759 int i; 1760 1761 flow_action_for_each(i, act, &rule->action) { 1762 if (act->id == act_id) 1763 return act; 1764 } 1765 return NULL; 1766 } 1767 1768 static void 1769 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1, 1770 struct nfp_fl_ct_zone_entry *zt_src, 1771 struct nfp_fl_ct_zone_entry *zt_dst) 1772 { 1773 struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp; 1774 struct list_head *ct_list; 1775 1776 if (ct_entry1->type == CT_TYPE_PRE_CT) 1777 ct_list = &zt_src->post_ct_list; 1778 else if (ct_entry1->type == CT_TYPE_POST_CT) 1779 ct_list = &zt_src->pre_ct_list; 1780 else 1781 return; 1782 1783 list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list, 1784 list_node) { 1785 nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1); 1786 } 1787 } 1788 1789 static void 1790 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry, 1791 struct nfp_fl_ct_zone_entry *zt) 1792 { 1793 struct nfp_fl_ct_tc_merge *tc_merge_entry; 1794 struct rhashtable_iter iter; 1795 1796 rhashtable_walk_enter(&zt->tc_merge_tb, &iter); 1797 rhashtable_walk_start(&iter); 1798 while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) { 1799 if (IS_ERR(tc_merge_entry)) 1800 continue; 1801 rhashtable_walk_stop(&iter); 1802 nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry); 1803 rhashtable_walk_start(&iter); 1804 } 1805 rhashtable_walk_stop(&iter); 1806 rhashtable_walk_exit(&iter); 1807 } 1808 1809 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv, 1810 struct net_device *netdev, 1811 struct flow_cls_offload *flow, 1812 struct netlink_ext_ack *extack, 1813 struct nfp_fl_nft_tc_merge *m_entry) 1814 { 1815 struct flow_action_entry *ct_act, *ct_goto; 1816 struct nfp_fl_ct_flow_entry *ct_entry; 1817 struct nfp_fl_ct_zone_entry *zt; 1818 int err; 1819 1820 ct_act = get_flow_act_ct(flow->rule); 1821 if (!ct_act) { 1822 NL_SET_ERR_MSG_MOD(extack, 1823 "unsupported offload: Conntrack action empty in conntrack offload"); 1824 return -EOPNOTSUPP; 1825 } 1826 1827 ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO); 1828 if (!ct_goto) { 1829 NL_SET_ERR_MSG_MOD(extack, 1830 "unsupported offload: Conntrack requires ACTION_GOTO"); 1831 return -EOPNOTSUPP; 1832 } 1833 1834 zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false); 1835 if (IS_ERR(zt)) { 1836 NL_SET_ERR_MSG_MOD(extack, 1837 "offload error: Could not create zone table entry"); 1838 return PTR_ERR(zt); 1839 } 1840 1841 if (!zt->nft) { 1842 zt->nft = ct_act->ct.flow_table; 1843 err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt); 1844 if (err) { 1845 NL_SET_ERR_MSG_MOD(extack, 1846 "offload error: Could not register nft_callback"); 1847 return err; 1848 } 1849 } 1850 1851 /* Add entry to pre_ct_list */ 1852 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack); 1853 if (IS_ERR(ct_entry)) 1854 return PTR_ERR(ct_entry); 1855 ct_entry->type = CT_TYPE_PRE_CT; 1856 ct_entry->chain_index = flow->common.chain_index; 1857 ct_entry->goto_chain_index = ct_goto->chain_index; 1858 1859 if (m_entry) { 1860 struct nfp_fl_ct_flow_entry *pre_ct_entry; 1861 int i; 1862 1863 pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent; 1864 for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) 1865 ct_entry->prev_m_entries[i] = pre_ct_entry->prev_m_entries[i]; 1866 ct_entry->prev_m_entries[i++] = m_entry; 1867 ct_entry->num_prev_m_entries = i; 1868 1869 m_entry->next_pre_ct_entry = ct_entry; 1870 } 1871 1872 list_add(&ct_entry->list_node, &zt->pre_ct_list); 1873 zt->pre_ct_count++; 1874 1875 nfp_ct_merge_tc_entries(ct_entry, zt, zt); 1876 1877 /* Need to check and merge with tables in the wc_zone as well */ 1878 if (priv->ct_zone_wc) 1879 nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt); 1880 1881 return 0; 1882 } 1883 1884 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv, 1885 struct net_device *netdev, 1886 struct flow_cls_offload *flow, 1887 struct netlink_ext_ack *extack) 1888 { 1889 struct flow_rule *rule = flow_cls_offload_flow_rule(flow); 1890 struct nfp_fl_ct_flow_entry *ct_entry; 1891 struct flow_action_entry *ct_goto; 1892 struct nfp_fl_ct_zone_entry *zt; 1893 struct flow_action_entry *act; 1894 bool wildcarded = false; 1895 struct flow_match_ct ct; 1896 int i; 1897 1898 flow_action_for_each(i, act, &rule->action) { 1899 switch (act->id) { 1900 case FLOW_ACTION_REDIRECT: 1901 case FLOW_ACTION_REDIRECT_INGRESS: 1902 case FLOW_ACTION_MIRRED: 1903 case FLOW_ACTION_MIRRED_INGRESS: 1904 if (act->dev->rtnl_link_ops && 1905 !strcmp(act->dev->rtnl_link_ops->kind, "openvswitch")) { 1906 NL_SET_ERR_MSG_MOD(extack, 1907 "unsupported offload: out port is openvswitch internal port"); 1908 return -EOPNOTSUPP; 1909 } 1910 break; 1911 default: 1912 break; 1913 } 1914 } 1915 1916 flow_rule_match_ct(rule, &ct); 1917 if (!ct.mask->ct_zone) { 1918 wildcarded = true; 1919 } else if (ct.mask->ct_zone != U16_MAX) { 1920 NL_SET_ERR_MSG_MOD(extack, 1921 "unsupported offload: partially wildcarded ct_zone is not supported"); 1922 return -EOPNOTSUPP; 1923 } 1924 1925 zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded); 1926 if (IS_ERR(zt)) { 1927 NL_SET_ERR_MSG_MOD(extack, 1928 "offload error: Could not create zone table entry"); 1929 return PTR_ERR(zt); 1930 } 1931 1932 /* Add entry to post_ct_list */ 1933 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack); 1934 if (IS_ERR(ct_entry)) 1935 return PTR_ERR(ct_entry); 1936 1937 ct_entry->type = CT_TYPE_POST_CT; 1938 ct_entry->chain_index = flow->common.chain_index; 1939 ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO); 1940 ct_entry->goto_chain_index = ct_goto ? ct_goto->chain_index : 0; 1941 list_add(&ct_entry->list_node, &zt->post_ct_list); 1942 zt->post_ct_count++; 1943 1944 if (wildcarded) { 1945 /* Iterate through all zone tables if not empty, look for merges with 1946 * pre_ct entries and merge them. 1947 */ 1948 struct rhashtable_iter iter; 1949 struct nfp_fl_ct_zone_entry *zone_table; 1950 1951 rhashtable_walk_enter(&priv->ct_zone_table, &iter); 1952 rhashtable_walk_start(&iter); 1953 while ((zone_table = rhashtable_walk_next(&iter)) != NULL) { 1954 if (IS_ERR(zone_table)) 1955 continue; 1956 rhashtable_walk_stop(&iter); 1957 nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table); 1958 rhashtable_walk_start(&iter); 1959 } 1960 rhashtable_walk_stop(&iter); 1961 rhashtable_walk_exit(&iter); 1962 } else { 1963 nfp_ct_merge_tc_entries(ct_entry, zt, zt); 1964 } 1965 1966 return 0; 1967 } 1968 1969 int nfp_fl_create_new_pre_ct(struct nfp_fl_nft_tc_merge *m_entry) 1970 { 1971 struct nfp_fl_ct_flow_entry *pre_ct_entry, *post_ct_entry; 1972 struct flow_cls_offload new_pre_ct_flow; 1973 int err; 1974 1975 pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent; 1976 if (pre_ct_entry->num_prev_m_entries >= NFP_MAX_RECIRC_CT_ZONES - 1) 1977 return -1; 1978 1979 post_ct_entry = m_entry->tc_m_parent->post_ct_parent; 1980 memset(&new_pre_ct_flow, 0, sizeof(struct flow_cls_offload)); 1981 new_pre_ct_flow.rule = post_ct_entry->rule; 1982 new_pre_ct_flow.common.chain_index = post_ct_entry->chain_index; 1983 1984 err = nfp_fl_ct_handle_pre_ct(pre_ct_entry->zt->priv, 1985 pre_ct_entry->netdev, 1986 &new_pre_ct_flow, NULL, 1987 m_entry); 1988 return err; 1989 } 1990 1991 static void 1992 nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge *nft_merge, 1993 enum ct_entry_type type, u64 *m_pkts, 1994 u64 *m_bytes, u64 *m_used) 1995 { 1996 struct nfp_flower_priv *priv = nft_merge->zt->priv; 1997 struct nfp_fl_payload *nfp_flow; 1998 u32 ctx_id; 1999 2000 nfp_flow = nft_merge->flow_pay; 2001 if (!nfp_flow) 2002 return; 2003 2004 ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); 2005 *m_pkts += priv->stats[ctx_id].pkts; 2006 *m_bytes += priv->stats[ctx_id].bytes; 2007 *m_used = max_t(u64, *m_used, priv->stats[ctx_id].used); 2008 2009 /* If request is for a sub_flow which is part of a tunnel merged 2010 * flow then update stats from tunnel merged flows first. 2011 */ 2012 if (!list_empty(&nfp_flow->linked_flows)) 2013 nfp_flower_update_merge_stats(priv->app, nfp_flow); 2014 2015 if (type != CT_TYPE_NFT) { 2016 /* Update nft cached stats */ 2017 flow_stats_update(&nft_merge->nft_parent->stats, 2018 priv->stats[ctx_id].bytes, 2019 priv->stats[ctx_id].pkts, 2020 0, priv->stats[ctx_id].used, 2021 FLOW_ACTION_HW_STATS_DELAYED); 2022 } else { 2023 /* Update pre_ct cached stats */ 2024 flow_stats_update(&nft_merge->tc_m_parent->pre_ct_parent->stats, 2025 priv->stats[ctx_id].bytes, 2026 priv->stats[ctx_id].pkts, 2027 0, priv->stats[ctx_id].used, 2028 FLOW_ACTION_HW_STATS_DELAYED); 2029 /* Update post_ct cached stats */ 2030 flow_stats_update(&nft_merge->tc_m_parent->post_ct_parent->stats, 2031 priv->stats[ctx_id].bytes, 2032 priv->stats[ctx_id].pkts, 2033 0, priv->stats[ctx_id].used, 2034 FLOW_ACTION_HW_STATS_DELAYED); 2035 } 2036 2037 /* Update previous pre_ct/post_ct/nft flow stats */ 2038 if (nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries > 0) { 2039 struct nfp_fl_nft_tc_merge *tmp_nft_merge; 2040 int i; 2041 2042 for (i = 0; i < nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries; i++) { 2043 tmp_nft_merge = nft_merge->tc_m_parent->pre_ct_parent->prev_m_entries[i]; 2044 flow_stats_update(&tmp_nft_merge->tc_m_parent->pre_ct_parent->stats, 2045 priv->stats[ctx_id].bytes, 2046 priv->stats[ctx_id].pkts, 2047 0, priv->stats[ctx_id].used, 2048 FLOW_ACTION_HW_STATS_DELAYED); 2049 flow_stats_update(&tmp_nft_merge->tc_m_parent->post_ct_parent->stats, 2050 priv->stats[ctx_id].bytes, 2051 priv->stats[ctx_id].pkts, 2052 0, priv->stats[ctx_id].used, 2053 FLOW_ACTION_HW_STATS_DELAYED); 2054 flow_stats_update(&tmp_nft_merge->nft_parent->stats, 2055 priv->stats[ctx_id].bytes, 2056 priv->stats[ctx_id].pkts, 2057 0, priv->stats[ctx_id].used, 2058 FLOW_ACTION_HW_STATS_DELAYED); 2059 } 2060 } 2061 2062 /* Reset stats from the nfp */ 2063 priv->stats[ctx_id].pkts = 0; 2064 priv->stats[ctx_id].bytes = 0; 2065 } 2066 2067 int nfp_fl_ct_stats(struct flow_cls_offload *flow, 2068 struct nfp_fl_ct_map_entry *ct_map_ent) 2069 { 2070 struct nfp_fl_ct_flow_entry *ct_entry = ct_map_ent->ct_entry; 2071 struct nfp_fl_nft_tc_merge *nft_merge, *nft_m_tmp; 2072 struct nfp_fl_ct_tc_merge *tc_merge, *tc_m_tmp; 2073 2074 u64 pkts = 0, bytes = 0, used = 0; 2075 u64 m_pkts, m_bytes, m_used; 2076 2077 spin_lock_bh(&ct_entry->zt->priv->stats_lock); 2078 2079 if (ct_entry->type == CT_TYPE_PRE_CT) { 2080 /* Iterate tc_merge entries associated with this flow */ 2081 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children, 2082 pre_ct_list) { 2083 m_pkts = 0; 2084 m_bytes = 0; 2085 m_used = 0; 2086 /* Iterate nft_merge entries associated with this tc_merge flow */ 2087 list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children, 2088 tc_merge_list) { 2089 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_PRE_CT, 2090 &m_pkts, &m_bytes, &m_used); 2091 } 2092 pkts += m_pkts; 2093 bytes += m_bytes; 2094 used = max_t(u64, used, m_used); 2095 /* Update post_ct partner */ 2096 flow_stats_update(&tc_merge->post_ct_parent->stats, 2097 m_bytes, m_pkts, 0, m_used, 2098 FLOW_ACTION_HW_STATS_DELAYED); 2099 } 2100 } else if (ct_entry->type == CT_TYPE_POST_CT) { 2101 /* Iterate tc_merge entries associated with this flow */ 2102 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children, 2103 post_ct_list) { 2104 m_pkts = 0; 2105 m_bytes = 0; 2106 m_used = 0; 2107 /* Iterate nft_merge entries associated with this tc_merge flow */ 2108 list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children, 2109 tc_merge_list) { 2110 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_POST_CT, 2111 &m_pkts, &m_bytes, &m_used); 2112 } 2113 pkts += m_pkts; 2114 bytes += m_bytes; 2115 used = max_t(u64, used, m_used); 2116 /* Update pre_ct partner */ 2117 flow_stats_update(&tc_merge->pre_ct_parent->stats, 2118 m_bytes, m_pkts, 0, m_used, 2119 FLOW_ACTION_HW_STATS_DELAYED); 2120 } 2121 } else { 2122 /* Iterate nft_merge entries associated with this nft flow */ 2123 list_for_each_entry_safe(nft_merge, nft_m_tmp, &ct_entry->children, 2124 nft_flow_list) { 2125 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_NFT, 2126 &pkts, &bytes, &used); 2127 } 2128 } 2129 2130 /* Add stats from this request to stats potentially cached by 2131 * previous requests. 2132 */ 2133 flow_stats_update(&ct_entry->stats, bytes, pkts, 0, used, 2134 FLOW_ACTION_HW_STATS_DELAYED); 2135 /* Finally update the flow stats from the original stats request */ 2136 flow_stats_update(&flow->stats, ct_entry->stats.bytes, 2137 ct_entry->stats.pkts, 0, 2138 ct_entry->stats.lastused, 2139 FLOW_ACTION_HW_STATS_DELAYED); 2140 /* Stats has been synced to original flow, can now clear 2141 * the cache. 2142 */ 2143 ct_entry->stats.pkts = 0; 2144 ct_entry->stats.bytes = 0; 2145 spin_unlock_bh(&ct_entry->zt->priv->stats_lock); 2146 2147 return 0; 2148 } 2149 2150 static bool 2151 nfp_fl_ct_offload_nft_supported(struct flow_cls_offload *flow) 2152 { 2153 struct flow_rule *flow_rule = flow->rule; 2154 struct flow_action *flow_action = 2155 &flow_rule->action; 2156 struct flow_action_entry *act; 2157 int i; 2158 2159 flow_action_for_each(i, act, flow_action) { 2160 if (act->id == FLOW_ACTION_CT_METADATA) { 2161 enum ip_conntrack_info ctinfo = 2162 act->ct_metadata.cookie & NFCT_INFOMASK; 2163 2164 return ctinfo != IP_CT_NEW; 2165 } 2166 } 2167 2168 return false; 2169 } 2170 2171 static int 2172 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow) 2173 { 2174 struct nfp_fl_ct_map_entry *ct_map_ent; 2175 struct nfp_fl_ct_flow_entry *ct_entry; 2176 struct netlink_ext_ack *extack = NULL; 2177 2178 extack = flow->common.extack; 2179 switch (flow->command) { 2180 case FLOW_CLS_REPLACE: 2181 if (!nfp_fl_ct_offload_nft_supported(flow)) 2182 return -EOPNOTSUPP; 2183 2184 /* Netfilter can request offload multiple times for the same 2185 * flow - protect against adding duplicates. 2186 */ 2187 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, 2188 nfp_ct_map_params); 2189 if (!ct_map_ent) { 2190 ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack); 2191 if (IS_ERR(ct_entry)) 2192 return PTR_ERR(ct_entry); 2193 ct_entry->type = CT_TYPE_NFT; 2194 list_add(&ct_entry->list_node, &zt->nft_flows_list); 2195 zt->nft_flows_count++; 2196 nfp_ct_merge_nft_with_tc(ct_entry, zt); 2197 } 2198 return 0; 2199 case FLOW_CLS_DESTROY: 2200 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, 2201 nfp_ct_map_params); 2202 return nfp_fl_ct_del_flow(ct_map_ent); 2203 case FLOW_CLS_STATS: 2204 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, 2205 nfp_ct_map_params); 2206 if (ct_map_ent) 2207 return nfp_fl_ct_stats(flow, ct_map_ent); 2208 break; 2209 default: 2210 break; 2211 } 2212 return -EINVAL; 2213 } 2214 2215 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv) 2216 { 2217 struct flow_cls_offload *flow = type_data; 2218 struct nfp_fl_ct_zone_entry *zt = cb_priv; 2219 int err = -EOPNOTSUPP; 2220 2221 switch (type) { 2222 case TC_SETUP_CLSFLOWER: 2223 while (!mutex_trylock(&zt->priv->nfp_fl_lock)) { 2224 if (!zt->nft) /* avoid deadlock */ 2225 return err; 2226 msleep(20); 2227 } 2228 err = nfp_fl_ct_offload_nft_flow(zt, flow); 2229 mutex_unlock(&zt->priv->nfp_fl_lock); 2230 break; 2231 default: 2232 return -EOPNOTSUPP; 2233 } 2234 return err; 2235 } 2236 2237 static void 2238 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt) 2239 { 2240 struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp; 2241 struct nfp_fl_ct_map_entry *ct_map_ent; 2242 2243 list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list, 2244 list_node) { 2245 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, 2246 &nft_entry->cookie, 2247 nfp_ct_map_params); 2248 nfp_fl_ct_del_flow(ct_map_ent); 2249 } 2250 } 2251 2252 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent) 2253 { 2254 struct nfp_fl_ct_flow_entry *ct_entry; 2255 struct nfp_fl_ct_zone_entry *zt; 2256 struct rhashtable *m_table; 2257 struct nf_flowtable *nft; 2258 2259 if (!ct_map_ent) 2260 return -ENOENT; 2261 2262 zt = ct_map_ent->ct_entry->zt; 2263 ct_entry = ct_map_ent->ct_entry; 2264 m_table = &zt->priv->ct_map_table; 2265 2266 switch (ct_entry->type) { 2267 case CT_TYPE_PRE_CT: 2268 zt->pre_ct_count--; 2269 if (ct_map_ent->cookie > 0) 2270 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, 2271 nfp_ct_map_params); 2272 nfp_fl_ct_clean_flow_entry(ct_entry); 2273 if (ct_map_ent->cookie > 0) 2274 kfree(ct_map_ent); 2275 2276 if (!zt->pre_ct_count && zt->nft) { 2277 nft = zt->nft; 2278 zt->nft = NULL; /* avoid deadlock */ 2279 nf_flow_table_offload_del_cb(nft, 2280 nfp_fl_ct_handle_nft_flow, 2281 zt); 2282 nfp_fl_ct_clean_nft_entries(zt); 2283 } 2284 break; 2285 case CT_TYPE_POST_CT: 2286 zt->post_ct_count--; 2287 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, 2288 nfp_ct_map_params); 2289 nfp_fl_ct_clean_flow_entry(ct_entry); 2290 kfree(ct_map_ent); 2291 break; 2292 case CT_TYPE_NFT: 2293 zt->nft_flows_count--; 2294 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, 2295 nfp_ct_map_params); 2296 nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry); 2297 kfree(ct_map_ent); 2298 break; 2299 default: 2300 break; 2301 } 2302 2303 return 0; 2304 } 2305