Lines Matching +full:key +full:- +full:release
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright 2020-2022 Xilinx Inc.
48 return ERR_PTR(-EOPNOTSUPP); in efx_tc_flower_lookup_efv()
50 if (dev == efx->net_dev) in efx_tc_flower_lookup_efv()
53 if (dev->netdev_ops != &efx_ef100_rep_netdev_ops) in efx_tc_flower_lookup_efv()
54 return ERR_PTR(-EOPNOTSUPP); in efx_tc_flower_lookup_efv()
59 if (efv->parent != efx) in efx_tc_flower_lookup_efv()
60 return ERR_PTR(-EOPNOTSUPP); in efx_tc_flower_lookup_efv()
64 /* Convert a driver-internal vport ID into an internal device (PF or VF) */
74 efx_mae_mport_mport(efx, efv->mport, &mport); in efx_tc_flower_internal_mport()
78 /* Convert a driver-internal vport ID into an external device (wire or VF) */
88 efx_mae_mport_mport(efx, efv->mport, &mport); in efx_tc_flower_external_mport()
131 return ERR_PTR(-ENOMEM); in efx_tc_flower_get_mac()
132 memcpy(ped->h_addr, h_addr, ETH_ALEN); in efx_tc_flower_get_mac()
133 old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht, in efx_tc_flower_get_mac()
134 &ped->linkage, in efx_tc_flower_get_mac()
141 if (!refcount_inc_not_zero(&old->ref)) in efx_tc_flower_get_mac()
142 return ERR_PTR(-EAGAIN); in efx_tc_flower_get_mac()
154 refcount_set(&ped->ref, 1); in efx_tc_flower_get_mac()
157 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, in efx_tc_flower_get_mac()
166 if (!refcount_dec_and_test(&ped->ref)) in efx_tc_flower_put_mac()
168 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, in efx_tc_flower_put_mac()
182 efx_mae_free_action_set(efx, act->fw_id); in efx_tc_free_action_set()
186 list_del(&act->list); in efx_tc_free_action_set()
188 if (act->count) { in efx_tc_free_action_set()
189 spin_lock_bh(&act->count->cnt->lock); in efx_tc_free_action_set()
190 if (!list_empty(&act->count_user)) in efx_tc_free_action_set()
191 list_del(&act->count_user); in efx_tc_free_action_set()
192 spin_unlock_bh(&act->count->cnt->lock); in efx_tc_free_action_set()
193 efx_tc_flower_put_counter_index(efx, act->count); in efx_tc_free_action_set()
195 if (act->encap_md) { in efx_tc_free_action_set()
196 list_del(&act->encap_user); in efx_tc_free_action_set()
197 efx_tc_flower_release_encap_md(efx, act->encap_md); in efx_tc_free_action_set()
199 if (act->src_mac) in efx_tc_free_action_set()
200 efx_tc_flower_put_mac(efx, act->src_mac); in efx_tc_free_action_set()
201 if (act->dst_mac) in efx_tc_free_action_set()
202 efx_tc_flower_put_mac(efx, act->dst_mac); in efx_tc_free_action_set()
219 list_for_each_entry_safe(act, next, &acts->list, list) in efx_tc_free_action_set_list()
230 match->value._field = fm.key->_tcfield; \
231 match->mask._field = fm.mask->_tcfield; \
243 struct flow_dissector *dissector = rule->match.dissector; in efx_tc_flower_parse_match()
246 /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set in efx_tc_flower_parse_match()
247 * even on IPv4 filters; so rather than relying on dissector->used_keys in efx_tc_flower_parse_match()
248 * we check the addr_type in the CONTROL key. If we don't find it (or in efx_tc_flower_parse_match()
256 if (IS_ALL_ONES(fm.mask->addr_type)) in efx_tc_flower_parse_match()
257 switch (fm.key->addr_type) { in efx_tc_flower_parse_match()
268 if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) { in efx_tc_flower_parse_match()
269 match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT; in efx_tc_flower_parse_match()
270 match->mask.ip_frag = true; in efx_tc_flower_parse_match()
272 if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) { in efx_tc_flower_parse_match()
273 match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG; in efx_tc_flower_parse_match()
274 match->mask.ip_firstfrag = true; in efx_tc_flower_parse_match()
278 fm.mask->flags, extack)) in efx_tc_flower_parse_match()
279 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
281 if (dissector->used_keys & in efx_tc_flower_parse_match()
300 dissector->used_keys); in efx_tc_flower_parse_match()
301 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
306 if (!IS_ALL_ONES(match->mask.eth_proto) || in efx_tc_flower_parse_match()
307 !(match->value.eth_proto == htons(ETH_P_IP) || in efx_tc_flower_parse_match()
308 match->value.eth_proto == htons(ETH_P_IPV6))) in efx_tc_flower_parse_match()
309 if (dissector->used_keys & in efx_tc_flower_parse_match()
317 dissector->used_keys); in efx_tc_flower_parse_match()
318 return -EINVAL; in efx_tc_flower_parse_match()
325 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { in efx_tc_flower_parse_match()
326 match->value.vlan_proto[0] = fm.key->vlan_tpid; in efx_tc_flower_parse_match()
327 match->mask.vlan_proto[0] = fm.mask->vlan_tpid; in efx_tc_flower_parse_match()
328 match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 | in efx_tc_flower_parse_match()
329 fm.key->vlan_id); in efx_tc_flower_parse_match()
330 match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 | in efx_tc_flower_parse_match()
331 fm.mask->vlan_id); in efx_tc_flower_parse_match()
339 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { in efx_tc_flower_parse_match()
340 match->value.vlan_proto[1] = fm.key->vlan_tpid; in efx_tc_flower_parse_match()
341 match->mask.vlan_proto[1] = fm.mask->vlan_tpid; in efx_tc_flower_parse_match()
342 match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 | in efx_tc_flower_parse_match()
343 fm.key->vlan_id); in efx_tc_flower_parse_match()
344 match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 | in efx_tc_flower_parse_match()
345 fm.mask->vlan_id); in efx_tc_flower_parse_match()
353 ether_addr_copy(match->value.eth_saddr, fm.key->src); in efx_tc_flower_parse_match()
354 ether_addr_copy(match->value.eth_daddr, fm.key->dst); in efx_tc_flower_parse_match()
355 ether_addr_copy(match->mask.eth_saddr, fm.mask->src); in efx_tc_flower_parse_match()
356 ether_addr_copy(match->mask.eth_daddr, fm.mask->dst); in efx_tc_flower_parse_match()
361 if ((match->value.ip_proto != IPPROTO_UDP && in efx_tc_flower_parse_match()
362 match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto)) in efx_tc_flower_parse_match()
363 if (dissector->used_keys & in efx_tc_flower_parse_match()
368 dissector->used_keys); in efx_tc_flower_parse_match()
369 return -EINVAL; in efx_tc_flower_parse_match()
390 if (flow_rule_has_enc_control_flags(fm.mask->flags, extack)) in efx_tc_flower_parse_match()
391 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
392 if (!IS_ALL_ONES(fm.mask->addr_type)) { in efx_tc_flower_parse_match()
393 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)", in efx_tc_flower_parse_match()
394 fm.mask->addr_type, in efx_tc_flower_parse_match()
395 fm.key->addr_type); in efx_tc_flower_parse_match()
396 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
398 switch (fm.key->addr_type) { in efx_tc_flower_parse_match()
416 fm.key->addr_type); in efx_tc_flower_parse_match()
417 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
424 } else if (dissector->used_keys & in efx_tc_flower_parse_match()
432 dissector->used_keys); in efx_tc_flower_parse_match()
433 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
439 match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); in efx_tc_flower_parse_match()
440 match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); in efx_tc_flower_parse_match()
441 match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); in efx_tc_flower_parse_match()
442 match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); in efx_tc_flower_parse_match()
443 if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED | in efx_tc_flower_parse_match()
447 fm.mask->ct_state); in efx_tc_flower_parse_match()
448 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
450 match->value.ct_mark = fm.key->ct_mark; in efx_tc_flower_parse_match()
451 match->mask.ct_mark = fm.mask->ct_mark; in efx_tc_flower_parse_match()
452 match->value.ct_zone = fm.key->ct_zone; in efx_tc_flower_parse_match()
453 match->mask.ct_zone = fm.mask->ct_zone; in efx_tc_flower_parse_match()
455 if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) { in efx_tc_flower_parse_match()
457 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
469 if (!refcount_dec_and_test(&encap->ref)) in efx_tc_flower_release_encap_match()
472 if (encap->type == EFX_TC_EM_DIRECT) { in efx_tc_flower_release_encap_match()
478 netif_err(efx, drv, efx->net_dev, in efx_tc_flower_release_encap_match()
479 "Failed to release encap match %#x, rc %d\n", in efx_tc_flower_release_encap_match()
480 encap->fw_id, rc); in efx_tc_flower_release_encap_match()
482 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, in efx_tc_flower_release_encap_match()
484 if (encap->pseudo) in efx_tc_flower_release_encap_match()
485 efx_tc_flower_release_encap_match(efx, encap->pseudo); in efx_tc_flower_release_encap_match()
501 /* We require that the socket-defining fields (IP addrs and UDP dest in efx_tc_flower_record_encap_match()
502 * port) are present and exact-match. Other fields may only be used in efx_tc_flower_record_encap_match()
503 * if the field-set (and any masks) are the same for all encap in efx_tc_flower_record_encap_match()
507 if (match->mask.enc_dst_ip | match->mask.enc_src_ip) { in efx_tc_flower_record_encap_match()
508 if (!IS_ALL_ONES(match->mask.enc_dst_ip)) { in efx_tc_flower_record_encap_match()
511 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
513 if (!IS_ALL_ONES(match->mask.enc_src_ip)) { in efx_tc_flower_record_encap_match()
516 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
519 if (!ipv6_addr_any(&match->mask.enc_dst_ip6) || in efx_tc_flower_record_encap_match()
520 !ipv6_addr_any(&match->mask.enc_src_ip6)) { in efx_tc_flower_record_encap_match()
523 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
527 if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) { in efx_tc_flower_record_encap_match()
530 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
532 if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) { in efx_tc_flower_record_encap_match()
535 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
539 if (!IS_ALL_ONES(match->mask.enc_dport)) { in efx_tc_flower_record_encap_match()
541 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
543 if (match->mask.enc_sport || match->mask.enc_ip_tos) { in efx_tc_flower_record_encap_match()
548 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
556 match->mask.enc_ip_tos, in efx_tc_flower_record_encap_match()
557 match->mask.enc_sport, in efx_tc_flower_record_encap_match()
563 if (match->mask.enc_ip_ttl) { in efx_tc_flower_record_encap_match()
565 rc = -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
569 rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos, in efx_tc_flower_record_encap_match()
570 match->mask.enc_sport, extack); in efx_tc_flower_record_encap_match()
576 rc = -ENOMEM; in efx_tc_flower_record_encap_match()
579 encap->src_ip = match->value.enc_src_ip; in efx_tc_flower_record_encap_match()
580 encap->dst_ip = match->value.enc_dst_ip; in efx_tc_flower_record_encap_match()
582 encap->src_ip6 = match->value.enc_src_ip6; in efx_tc_flower_record_encap_match()
583 encap->dst_ip6 = match->value.enc_dst_ip6; in efx_tc_flower_record_encap_match()
585 encap->udp_dport = match->value.enc_dport; in efx_tc_flower_record_encap_match()
586 encap->tun_type = type; in efx_tc_flower_record_encap_match()
587 encap->ip_tos = match->value.enc_ip_tos; in efx_tc_flower_record_encap_match()
588 encap->ip_tos_mask = match->mask.enc_ip_tos; in efx_tc_flower_record_encap_match()
589 encap->child_ip_tos_mask = child_ip_tos_mask; in efx_tc_flower_record_encap_match()
590 encap->udp_sport = match->value.enc_sport; in efx_tc_flower_record_encap_match()
591 encap->udp_sport_mask = match->mask.enc_sport; in efx_tc_flower_record_encap_match()
592 encap->child_udp_sport_mask = child_udp_sport_mask; in efx_tc_flower_record_encap_match()
593 encap->type = em_type; in efx_tc_flower_record_encap_match()
594 encap->pseudo = pseudo; in efx_tc_flower_record_encap_match()
595 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht, in efx_tc_flower_record_encap_match()
596 &encap->linkage, in efx_tc_flower_record_encap_match()
606 switch (old->type) { in efx_tc_flower_record_encap_match()
614 return -EEXIST; in efx_tc_flower_record_encap_match()
616 /* old EM is protecting a ToS- or src port-qualified in efx_tc_flower_record_encap_match()
624 return -EEXIST; in efx_tc_flower_record_encap_match()
626 if (child_ip_tos_mask != old->child_ip_tos_mask) { in efx_tc_flower_record_encap_match()
630 old->child_ip_tos_mask); in efx_tc_flower_record_encap_match()
631 return -EEXIST; in efx_tc_flower_record_encap_match()
633 if (child_udp_sport_mask != old->child_udp_sport_mask) { in efx_tc_flower_record_encap_match()
637 old->child_udp_sport_mask); in efx_tc_flower_record_encap_match()
638 return -EEXIST; in efx_tc_flower_record_encap_match()
644 * direct-EM or pseudo). in efx_tc_flower_record_encap_match()
649 return -EEXIST; in efx_tc_flower_record_encap_match()
650 default: /* Unrecognised pseudo-type. Just say no */ in efx_tc_flower_record_encap_match()
654 old->type); in efx_tc_flower_record_encap_match()
655 return -EEXIST; in efx_tc_flower_record_encap_match()
658 if (old->tun_type != type) { in efx_tc_flower_record_encap_match()
661 old->tun_type, type); in efx_tc_flower_record_encap_match()
662 return -EEXIST; in efx_tc_flower_record_encap_match()
664 if (!refcount_inc_not_zero(&old->ref)) in efx_tc_flower_record_encap_match()
665 return -EAGAIN; in efx_tc_flower_record_encap_match()
676 refcount_set(&encap->ref, 1); in efx_tc_flower_record_encap_match()
678 match->encap = encap; in efx_tc_flower_record_encap_match()
681 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, in efx_tc_flower_record_encap_match()
699 return ERR_PTR(-ENOMEM); in efx_tc_get_recirc_id()
700 rid->chain_index = chain_index; in efx_tc_get_recirc_id()
701 /* We don't take a reference here, because it's implied - if there's in efx_tc_get_recirc_id()
705 rid->net_dev = net_dev; in efx_tc_get_recirc_id()
706 old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht, in efx_tc_get_recirc_id()
707 &rid->linkage, in efx_tc_get_recirc_id()
714 if (!refcount_inc_not_zero(&old->ref)) in efx_tc_get_recirc_id()
715 return ERR_PTR(-EAGAIN); in efx_tc_get_recirc_id()
719 rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER); in efx_tc_get_recirc_id()
721 rhashtable_remove_fast(&efx->tc->recirc_ht, in efx_tc_get_recirc_id()
722 &rid->linkage, in efx_tc_get_recirc_id()
727 rid->fw_id = rc; in efx_tc_get_recirc_id()
728 refcount_set(&rid->ref, 1); in efx_tc_get_recirc_id()
735 if (!refcount_dec_and_test(&rid->ref)) in efx_tc_put_recirc_id()
737 rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage, in efx_tc_put_recirc_id()
739 ida_free(&efx->tc->recirc_ida, rid->fw_id); in efx_tc_put_recirc_id()
745 efx_mae_delete_rule(efx, rule->fw_id); in efx_tc_delete_rule()
747 /* Release entries in subsidiary tables */ in efx_tc_delete_rule()
748 efx_tc_free_action_set_list(efx, &rule->acts, true); in efx_tc_delete_rule()
749 if (rule->match.rid) in efx_tc_delete_rule()
750 efx_tc_put_recirc_id(efx, rule->match.rid); in efx_tc_delete_rule()
751 if (rule->match.encap) in efx_tc_delete_rule()
752 efx_tc_flower_release_encap_match(efx, rule->match.encap); in efx_tc_delete_rule()
753 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_tc_delete_rule()
771 /* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
788 if (act->decap) in efx_tc_flower_action_order_ok()
793 if (act->dst_mac || act->src_mac) in efx_tc_flower_action_order_ok()
797 if (act->do_ttl_dec) in efx_tc_flower_action_order_ok()
801 if (act->vlan_pop >= 2) in efx_tc_flower_action_order_ok()
807 if (act->vlan_push) in efx_tc_flower_action_order_ok()
811 if (act->vlan_push >= 2) in efx_tc_flower_action_order_ok()
815 if (act->count) in efx_tc_flower_action_order_ok()
820 if (act->encap_md) in efx_tc_flower_action_order_ok()
824 return !act->deliver; in efx_tc_flower_action_order_ok()
826 if (act->encap_md) in efx_tc_flower_action_order_ok()
828 return !act->do_ttl_dec; in efx_tc_flower_action_order_ok()
840 * consequently we support conntrack through the notion of a "left-hand side
842 * "goto chain N", and corresponds to one or more "right-hand side rules" in
859 flow_action_for_each(i, fa, &fr->action) { in efx_tc_rule_is_lhs_rule()
860 switch (fa->id) { in efx_tc_rule_is_lhs_rule()
864 /* If rule is -trk, or doesn't mention trk at all, then in efx_tc_rule_is_lhs_rule()
870 if (!match->mask.ct_state_trk || !match->value.ct_state_trk) in efx_tc_rule_is_lhs_rule()
881 * implied match on enc_ip_proto UDP). Translate these into non-enc_ keys,
888 * Make sure that the non-enc_ keys were not already being matched on, as that
890 * with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible;
898 if (match->mask._key) { \ in efx_tc_flower_translate_flhs_match()
899 rc = -EOPNOTSUPP; \ in efx_tc_flower_translate_flhs_match()
901 match->mask._key = match->mask._ekey; \ in efx_tc_flower_translate_flhs_match()
902 match->mask._ekey = 0; \ in efx_tc_flower_translate_flhs_match()
903 match->value._key = match->value._ekey; \ in efx_tc_flower_translate_flhs_match()
904 match->value._ekey = 0; \ in efx_tc_flower_translate_flhs_match()
909 if (match->mask.ip_proto) in efx_tc_flower_translate_flhs_match()
910 return -EOPNOTSUPP; in efx_tc_flower_translate_flhs_match()
911 match->mask.ip_proto = ~0; in efx_tc_flower_translate_flhs_match()
912 match->value.ip_proto = IPPROTO_UDP; in efx_tc_flower_translate_flhs_match()
916 if (!ipv6_addr_any(&match->mask.src_ip6)) in efx_tc_flower_translate_flhs_match()
917 return -EOPNOTSUPP; in efx_tc_flower_translate_flhs_match()
918 match->mask.src_ip6 = match->mask.enc_src_ip6; in efx_tc_flower_translate_flhs_match()
919 memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr)); in efx_tc_flower_translate_flhs_match()
920 if (!ipv6_addr_any(&match->mask.dst_ip6)) in efx_tc_flower_translate_flhs_match()
921 return -EOPNOTSUPP; in efx_tc_flower_translate_flhs_match()
922 match->mask.dst_ip6 = match->mask.enc_dst_ip6; in efx_tc_flower_translate_flhs_match()
923 memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr)); in efx_tc_flower_translate_flhs_match()
941 * beyond the outer headers. Thus, such rules must use the OR-AR-CT-AR lookup
947 /* matches on inner-header keys can't be done in OR */ in efx_tc_flower_flhs_needs_ar()
948 return match->mask.eth_proto || in efx_tc_flower_flhs_needs_ar()
949 match->mask.vlan_tci[0] || match->mask.vlan_tci[1] || in efx_tc_flower_flhs_needs_ar()
950 match->mask.vlan_proto[0] || match->mask.vlan_proto[1] || in efx_tc_flower_flhs_needs_ar()
951 memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) || in efx_tc_flower_flhs_needs_ar()
952 memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) || in efx_tc_flower_flhs_needs_ar()
953 match->mask.ip_proto || in efx_tc_flower_flhs_needs_ar()
954 match->mask.ip_tos || match->mask.ip_ttl || in efx_tc_flower_flhs_needs_ar()
955 match->mask.src_ip || match->mask.dst_ip || in efx_tc_flower_flhs_needs_ar()
957 !ipv6_addr_any(&match->mask.src_ip6) || in efx_tc_flower_flhs_needs_ar()
958 !ipv6_addr_any(&match->mask.dst_ip6) || in efx_tc_flower_flhs_needs_ar()
960 match->mask.ip_frag || match->mask.ip_firstfrag || in efx_tc_flower_flhs_needs_ar()
961 match->mask.l4_sport || match->mask.l4_dport || in efx_tc_flower_flhs_needs_ar()
962 match->mask.tcp_flags || in efx_tc_flower_flhs_needs_ar()
964 match->mask.enc_keyid; in efx_tc_flower_flhs_needs_ar()
974 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_handle_lhs_actions()
975 struct efx_tc_lhs_action *act = &rule->lhs_act; in efx_tc_flower_handle_lhs_actions()
981 ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR; in efx_tc_flower_handle_lhs_actions()
983 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_handle_lhs_actions()
988 /* more actions after a non-pipe action */ in efx_tc_flower_handle_lhs_actions()
989 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); in efx_tc_flower_handle_lhs_actions()
990 return -EINVAL; in efx_tc_flower_handle_lhs_actions()
992 switch (fa->id) { in efx_tc_flower_handle_lhs_actions()
994 if (!fa->chain_index) { in efx_tc_flower_handle_lhs_actions()
996 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
998 rid = efx_tc_get_recirc_id(efx, fa->chain_index, in efx_tc_flower_handle_lhs_actions()
1004 act->rid = rid; in efx_tc_flower_handle_lhs_actions()
1005 if (fa->hw_stats) { in efx_tc_flower_handle_lhs_actions()
1008 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { in efx_tc_flower_handle_lhs_actions()
1011 fa->hw_stats); in efx_tc_flower_handle_lhs_actions()
1012 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1014 cnt = efx_tc_flower_get_counter_index(efx, tc->cookie, in efx_tc_flower_handle_lhs_actions()
1020 WARN_ON(act->count); /* can't happen */ in efx_tc_flower_handle_lhs_actions()
1021 act->count = cnt; in efx_tc_flower_handle_lhs_actions()
1026 if (act->zone) { in efx_tc_flower_handle_lhs_actions()
1028 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1030 if (fa->ct.action & (TCA_CT_ACT_COMMIT | in efx_tc_flower_handle_lhs_actions()
1033 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1035 if (fa->ct.action & TCA_CT_ACT_CLEAR) { in efx_tc_flower_handle_lhs_actions()
1037 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1039 if (fa->ct.action & (TCA_CT_ACT_NAT | in efx_tc_flower_handle_lhs_actions()
1042 NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet"); in efx_tc_flower_handle_lhs_actions()
1043 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1045 if (fa->ct.action) { in efx_tc_flower_handle_lhs_actions()
1047 fa->ct.action); in efx_tc_flower_handle_lhs_actions()
1048 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1050 ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone, in efx_tc_flower_handle_lhs_actions()
1051 fa->ct.flow_table); in efx_tc_flower_handle_lhs_actions()
1056 act->zone = ct_zone; in efx_tc_flower_handle_lhs_actions()
1060 fa->id); in efx_tc_flower_handle_lhs_actions()
1061 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1067 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
1075 if (act->rid) in efx_tc_flower_release_lhs_actions()
1076 efx_tc_put_recirc_id(efx, act->rid); in efx_tc_flower_release_lhs_actions()
1077 if (act->zone) in efx_tc_flower_release_lhs_actions()
1078 efx_tc_ct_unregister_zone(efx, act->zone); in efx_tc_flower_release_lhs_actions()
1079 if (act->count) in efx_tc_flower_release_lhs_actions()
1080 efx_tc_flower_put_counter_index(efx, act->count); in efx_tc_flower_release_lhs_actions()
1084 * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields
1093 * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not
1099 u8 dst_mac_32:1; /* eth->h_dest[0:3] */
1100 u8 dst_mac_16:1; /* eth->h_dest[4:5] */
1101 u8 src_mac_16:1; /* eth->h_source[0:1] */
1102 u8 src_mac_32:1; /* eth->h_source[2:5] */
1107 /** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung
1125 if (mung->dst_mac_32 && mung->dst_mac_16) { in efx_tc_complete_mac_mangle()
1126 ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack); in efx_tc_complete_mac_mangle()
1131 if (act->dst_mac) in efx_tc_complete_mac_mangle()
1132 efx_tc_flower_put_mac(efx, act->dst_mac); in efx_tc_complete_mac_mangle()
1134 act->dst_mac = ped; in efx_tc_complete_mac_mangle()
1137 mung->dst_mac_32 = 0; in efx_tc_complete_mac_mangle()
1138 mung->dst_mac_16 = 0; in efx_tc_complete_mac_mangle()
1140 if (mung->src_mac_16 && mung->src_mac_32) { in efx_tc_complete_mac_mangle()
1141 ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack); in efx_tc_complete_mac_mangle()
1146 if (act->src_mac) in efx_tc_complete_mac_mangle()
1147 efx_tc_flower_put_mac(efx, act->src_mac); in efx_tc_complete_mac_mangle()
1149 act->src_mac = ped; in efx_tc_complete_mac_mangle()
1152 mung->src_mac_32 = 0; in efx_tc_complete_mac_mangle()
1153 mung->src_mac_16 = 0; in efx_tc_complete_mac_mangle()
1162 switch (fa->mangle.htype) { in efx_tc_pedit_add()
1164 switch (fa->mangle.offset) { in efx_tc_pedit_add()
1167 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) in efx_tc_pedit_add()
1173 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX) in efx_tc_pedit_add()
1180 return -EOPNOTSUPP; in efx_tc_pedit_add()
1182 act->do_ttl_dec = 1; in efx_tc_pedit_add()
1189 switch (fa->mangle.offset) { in efx_tc_pedit_add()
1192 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) in efx_tc_pedit_add()
1198 if ((fa->mangle.val >> 24) != U8_MAX) in efx_tc_pedit_add()
1205 return -EOPNOTSUPP; in efx_tc_pedit_add()
1207 act->do_ttl_dec = 1; in efx_tc_pedit_add()
1219 fa->mangle.htype, fa->mangle.offset, in efx_tc_pedit_add()
1220 fa->mangle.val, fa->mangle.mask); in efx_tc_pedit_add()
1221 return -EOPNOTSUPP; in efx_tc_pedit_add()
1225 * efx_tc_mangle() - handle a single 32-bit (or less) pedit
1249 switch (fa->mangle.htype) { in efx_tc_mangle()
1256 return -EOPNOTSUPP; in efx_tc_mangle()
1258 switch (fa->mangle.offset) { in efx_tc_mangle()
1260 if (fa->mangle.mask) { in efx_tc_mangle()
1263 fa->mangle.mask); in efx_tc_mangle()
1264 return -EOPNOTSUPP; in efx_tc_mangle()
1266 /* Ethernet address is little-endian */ in efx_tc_mangle()
1267 mac32 = cpu_to_le32(fa->mangle.val); in efx_tc_mangle()
1268 memcpy(mung->dst_mac, &mac32, sizeof(mac32)); in efx_tc_mangle()
1269 mung->dst_mac_32 = 1; in efx_tc_mangle()
1272 if (fa->mangle.mask == 0xffff) { in efx_tc_mangle()
1273 mac16 = cpu_to_le16(fa->mangle.val >> 16); in efx_tc_mangle()
1274 memcpy(mung->src_mac, &mac16, sizeof(mac16)); in efx_tc_mangle()
1275 mung->src_mac_16 = 1; in efx_tc_mangle()
1276 } else if (fa->mangle.mask == 0xffff0000) { in efx_tc_mangle()
1277 mac16 = cpu_to_le16((u16)fa->mangle.val); in efx_tc_mangle()
1278 memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16)); in efx_tc_mangle()
1279 mung->dst_mac_16 = 1; in efx_tc_mangle()
1283 fa->mangle.mask); in efx_tc_mangle()
1284 return -EOPNOTSUPP; in efx_tc_mangle()
1288 if (fa->mangle.mask) { in efx_tc_mangle()
1291 fa->mangle.mask); in efx_tc_mangle()
1292 return -EOPNOTSUPP; in efx_tc_mangle()
1294 mac32 = cpu_to_le32(fa->mangle.val); in efx_tc_mangle()
1295 memcpy(mung->src_mac + 2, &mac32, sizeof(mac32)); in efx_tc_mangle()
1296 mung->src_mac_32 = 1; in efx_tc_mangle()
1300 fa->mangle.offset, fa->mangle.val, fa->mangle.mask); in efx_tc_mangle()
1301 return -EOPNOTSUPP; in efx_tc_mangle()
1305 switch (fa->mangle.offset) { in efx_tc_mangle()
1313 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) { in efx_tc_mangle()
1316 fa->mangle.mask); in efx_tc_mangle()
1317 return -EOPNOTSUPP; in efx_tc_mangle()
1323 if (match->mask.ip_ttl != U8_MAX) { in efx_tc_mangle()
1326 match->mask.ip_ttl); in efx_tc_mangle()
1327 return -EOPNOTSUPP; in efx_tc_mangle()
1333 if (match->value.ip_ttl == 0) { in efx_tc_mangle()
1336 return -EOPNOTSUPP; in efx_tc_mangle()
1344 return -EOPNOTSUPP; in efx_tc_mangle()
1348 tr_ttl = match->value.ip_ttl - 1; in efx_tc_mangle()
1349 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) { in efx_tc_mangle()
1350 act->do_ttl_dec = 1; in efx_tc_mangle()
1358 fa->mangle.offset); in efx_tc_mangle()
1359 return -EOPNOTSUPP; in efx_tc_mangle()
1363 switch (fa->mangle.offset) { in efx_tc_mangle()
1371 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) { in efx_tc_mangle()
1374 fa->mangle.mask); in efx_tc_mangle()
1376 return -EOPNOTSUPP; in efx_tc_mangle()
1382 if (match->mask.ip_ttl != U8_MAX) { in efx_tc_mangle()
1385 match->mask.ip_ttl); in efx_tc_mangle()
1386 return -EOPNOTSUPP; in efx_tc_mangle()
1392 if (match->value.ip_ttl == 0) { in efx_tc_mangle()
1395 return -EOPNOTSUPP; in efx_tc_mangle()
1403 return -EOPNOTSUPP; in efx_tc_mangle()
1407 tr_ttl = match->value.ip_ttl - 1; in efx_tc_mangle()
1408 if ((fa->mangle.val >> 24) == tr_ttl) { in efx_tc_mangle()
1409 act->do_ttl_dec = 1; in efx_tc_mangle()
1417 return -EOPNOTSUPP; in efx_tc_mangle()
1421 fa->mangle.htype); in efx_tc_mangle()
1422 return -EOPNOTSUPP; in efx_tc_mangle()
1428 * efx_tc_incomplete_mangle() - check for leftover partial pedits
1435 * and reject them with -%EOPNOTSUPP.
1441 if (mung->dst_mac_32 || mung->dst_mac_16) { in efx_tc_incomplete_mangle()
1443 return -EOPNOTSUPP; in efx_tc_incomplete_mangle()
1445 if (mung->src_mac_16 || mung->src_mac_32) { in efx_tc_incomplete_mangle()
1447 return -EOPNOTSUPP; in efx_tc_incomplete_mangle()
1458 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace_foreign_lhs_ar()
1466 return -EOPNOTSUPP; in efx_tc_flower_replace_foreign_lhs_ar()
1484 match->mask.recirc_id = 0xff; in efx_tc_flower_replace_foreign_lhs_ar()
1485 if (match->mask.ct_state_trk && match->value.ct_state_trk) { in efx_tc_flower_replace_foreign_lhs_ar()
1487 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign_lhs_ar()
1490 /* LHS rules are always -trk, so we don't need to match on that */ in efx_tc_flower_replace_foreign_lhs_ar()
1491 match->mask.ct_state_trk = 0; in efx_tc_flower_replace_foreign_lhs_ar()
1492 match->value.ct_state_trk = 0; in efx_tc_flower_replace_foreign_lhs_ar()
1502 match->mask.tcp_syn_fin_rst = true; in efx_tc_flower_replace_foreign_lhs_ar()
1504 rc = efx_mae_match_check_caps(efx, &match->mask, extack); in efx_tc_flower_replace_foreign_lhs_ar()
1510 rc = -ENOMEM; in efx_tc_flower_replace_foreign_lhs_ar()
1513 rule->cookie = tc->cookie; in efx_tc_flower_replace_foreign_lhs_ar()
1514 rule->is_ar = true; in efx_tc_flower_replace_foreign_lhs_ar()
1515 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, in efx_tc_flower_replace_foreign_lhs_ar()
1516 &rule->linkage, in efx_tc_flower_replace_foreign_lhs_ar()
1519 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign_lhs_ar()
1520 "Already offloaded rule (cookie %lx)\n", tc->cookie); in efx_tc_flower_replace_foreign_lhs_ar()
1521 rc = -EEXIST; in efx_tc_flower_replace_foreign_lhs_ar()
1523 goto release; in efx_tc_flower_replace_foreign_lhs_ar()
1529 goto release; in efx_tc_flower_replace_foreign_lhs_ar()
1531 rule->match = *match; in efx_tc_flower_replace_foreign_lhs_ar()
1532 rule->lhs_act.tun_type = type; in efx_tc_flower_replace_foreign_lhs_ar()
1537 goto release; in efx_tc_flower_replace_foreign_lhs_ar()
1539 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign_lhs_ar()
1541 tc->cookie); in efx_tc_flower_replace_foreign_lhs_ar()
1544 release: in efx_tc_flower_replace_foreign_lhs_ar()
1545 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); in efx_tc_flower_replace_foreign_lhs_ar()
1547 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, in efx_tc_flower_replace_foreign_lhs_ar()
1551 if (match->encap) in efx_tc_flower_replace_foreign_lhs_ar()
1552 efx_tc_flower_release_encap_match(efx, match->encap); in efx_tc_flower_replace_foreign_lhs_ar()
1562 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace_foreign_lhs()
1567 if (tc->common.chain_index) { in efx_tc_flower_replace_foreign_lhs()
1569 return -EOPNOTSUPP; in efx_tc_flower_replace_foreign_lhs()
1572 if (!efx_tc_match_is_encap(&match->mask)) { in efx_tc_flower_replace_foreign_lhs()
1574 netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n"); in efx_tc_flower_replace_foreign_lhs()
1575 return -EOPNOTSUPP; in efx_tc_flower_replace_foreign_lhs()
1585 return -EOPNOTSUPP; in efx_tc_flower_replace_foreign_lhs()
1602 if (match->mask.ct_state_trk && match->value.ct_state_trk) { in efx_tc_flower_replace_foreign_lhs()
1604 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign_lhs()
1607 /* LHS rules are always -trk, so we don't need to match on that */ in efx_tc_flower_replace_foreign_lhs()
1608 match->mask.ct_state_trk = 0; in efx_tc_flower_replace_foreign_lhs()
1609 match->value.ct_state_trk = 0; in efx_tc_flower_replace_foreign_lhs()
1617 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack); in efx_tc_flower_replace_foreign_lhs()
1623 rc = -ENOMEM; in efx_tc_flower_replace_foreign_lhs()
1626 rule->cookie = tc->cookie; in efx_tc_flower_replace_foreign_lhs()
1627 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, in efx_tc_flower_replace_foreign_lhs()
1628 &rule->linkage, in efx_tc_flower_replace_foreign_lhs()
1631 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign_lhs()
1632 "Already offloaded rule (cookie %lx)\n", tc->cookie); in efx_tc_flower_replace_foreign_lhs()
1633 rc = -EEXIST; in efx_tc_flower_replace_foreign_lhs()
1635 goto release; in efx_tc_flower_replace_foreign_lhs()
1641 goto release; in efx_tc_flower_replace_foreign_lhs()
1643 rule->match = *match; in efx_tc_flower_replace_foreign_lhs()
1644 rule->lhs_act.tun_type = type; in efx_tc_flower_replace_foreign_lhs()
1649 goto release; in efx_tc_flower_replace_foreign_lhs()
1651 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign_lhs()
1653 tc->cookie); in efx_tc_flower_replace_foreign_lhs()
1656 release: in efx_tc_flower_replace_foreign_lhs()
1657 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); in efx_tc_flower_replace_foreign_lhs()
1659 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, in efx_tc_flower_replace_foreign_lhs()
1663 if (match->encap) in efx_tc_flower_replace_foreign_lhs()
1664 efx_tc_flower_release_encap_match(efx, match->encap); in efx_tc_flower_replace_foreign_lhs()
1673 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace_foreign()
1691 * the filter with source m-port == wire. in efx_tc_flower_replace_foreign()
1695 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter"); in efx_tc_flower_replace_foreign()
1705 if (tc->common.chain_index) { in efx_tc_flower_replace_foreign()
1708 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev); in efx_tc_flower_replace_foreign()
1712 tc->common.chain_index); in efx_tc_flower_replace_foreign()
1716 match.value.recirc_id = rid->fw_id; in efx_tc_flower_replace_foreign()
1727 * match +trk-est (CT_HIT=0) despite being on an established connection. in efx_tc_flower_replace_foreign()
1728 * So make -est imply -tcp_syn_fin_rst match to ensure these packets in efx_tc_flower_replace_foreign()
1734 NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload"); in efx_tc_flower_replace_foreign()
1735 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1736 goto release; in efx_tc_flower_replace_foreign()
1741 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_replace_foreign()
1742 switch (fa->id) { in efx_tc_flower_replace_foreign()
1745 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); in efx_tc_flower_replace_foreign()
1755 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1757 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1758 goto release; in efx_tc_flower_replace_foreign()
1763 goto release; in efx_tc_flower_replace_foreign()
1772 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1773 goto release; in efx_tc_flower_replace_foreign()
1781 goto release; in efx_tc_flower_replace_foreign()
1788 goto release; in efx_tc_flower_replace_foreign()
1789 } else if (!tc->common.chain_index) { in efx_tc_flower_replace_foreign()
1791 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1793 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1794 goto release; in efx_tc_flower_replace_foreign()
1799 rc = -ENOMEM; in efx_tc_flower_replace_foreign()
1800 goto release; in efx_tc_flower_replace_foreign()
1802 INIT_LIST_HEAD(&rule->acts.list); in efx_tc_flower_replace_foreign()
1803 rule->cookie = tc->cookie; in efx_tc_flower_replace_foreign()
1804 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace_foreign()
1805 &rule->linkage, in efx_tc_flower_replace_foreign()
1809 goto release; in efx_tc_flower_replace_foreign()
1811 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1812 "Ignoring already-offloaded rule (cookie %lx)\n", in efx_tc_flower_replace_foreign()
1813 tc->cookie); in efx_tc_flower_replace_foreign()
1814 rc = -EEXIST; in efx_tc_flower_replace_foreign()
1815 goto release; in efx_tc_flower_replace_foreign()
1820 rc = -ENOMEM; in efx_tc_flower_replace_foreign()
1821 goto release; in efx_tc_flower_replace_foreign()
1828 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_replace_foreign()
1831 switch (fa->id) { in efx_tc_flower_replace_foreign()
1838 if (fa->hw_stats) { in efx_tc_flower_replace_foreign()
1841 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { in efx_tc_flower_replace_foreign()
1844 fa->hw_stats); in efx_tc_flower_replace_foreign()
1845 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1846 goto release; in efx_tc_flower_replace_foreign()
1850 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1851 goto release; in efx_tc_flower_replace_foreign()
1855 tc->cookie, in efx_tc_flower_replace_foreign()
1860 goto release; in efx_tc_flower_replace_foreign()
1862 act->count = ctr; in efx_tc_flower_replace_foreign()
1863 INIT_LIST_HEAD(&act->count_user); in efx_tc_flower_replace_foreign()
1868 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1871 goto release; in efx_tc_flower_replace_foreign()
1873 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); in efx_tc_flower_replace_foreign()
1888 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); in efx_tc_flower_replace_foreign()
1889 goto release; in efx_tc_flower_replace_foreign()
1891 act->dest_mport = rc; in efx_tc_flower_replace_foreign()
1892 act->deliver = 1; in efx_tc_flower_replace_foreign()
1897 goto release; in efx_tc_flower_replace_foreign()
1899 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace_foreign()
1901 if (fa->id == FLOW_ACTION_REDIRECT) in efx_tc_flower_replace_foreign()
1906 rc = -ENOMEM; in efx_tc_flower_replace_foreign()
1907 goto release; in efx_tc_flower_replace_foreign()
1913 rc = -EINVAL; in efx_tc_flower_replace_foreign()
1915 goto release; in efx_tc_flower_replace_foreign()
1917 act->decap = 1; in efx_tc_flower_replace_foreign()
1926 fa->id); in efx_tc_flower_replace_foreign()
1927 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1928 goto release; in efx_tc_flower_replace_foreign()
1937 efx_mae_mport_uplink(efx, &act->dest_mport); in efx_tc_flower_replace_foreign()
1938 act->deliver = 1; in efx_tc_flower_replace_foreign()
1943 goto release; in efx_tc_flower_replace_foreign()
1945 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace_foreign()
1946 act = NULL; /* Prevent double-free in error path */ in efx_tc_flower_replace_foreign()
1949 rule->match = match; in efx_tc_flower_replace_foreign()
1951 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1953 tc->cookie); in efx_tc_flower_replace_foreign()
1955 rc = efx_mae_alloc_action_set_list(efx, &rule->acts); in efx_tc_flower_replace_foreign()
1958 goto release; in efx_tc_flower_replace_foreign()
1960 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, in efx_tc_flower_replace_foreign()
1961 rule->acts.fw_id, &rule->fw_id); in efx_tc_flower_replace_foreign()
1969 efx_mae_free_action_set_list(efx, &rule->acts); in efx_tc_flower_replace_foreign()
1970 release: in efx_tc_flower_replace_foreign()
1980 rhashtable_remove_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace_foreign()
1981 &rule->linkage, in efx_tc_flower_replace_foreign()
1983 efx_tc_free_action_set_list(efx, &rule->acts, false); in efx_tc_flower_replace_foreign()
1998 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace_lhs()
2002 if (tc->common.chain_index) { in efx_tc_flower_replace_lhs()
2004 return -EOPNOTSUPP; in efx_tc_flower_replace_lhs()
2007 if (match->mask.ct_state_trk && match->value.ct_state_trk) { in efx_tc_flower_replace_lhs()
2009 return -EOPNOTSUPP; in efx_tc_flower_replace_lhs()
2011 /* LHS rules are always -trk, so we don't need to match on that */ in efx_tc_flower_replace_lhs()
2012 match->mask.ct_state_trk = 0; in efx_tc_flower_replace_lhs()
2013 match->value.ct_state_trk = 0; in efx_tc_flower_replace_lhs()
2015 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack); in efx_tc_flower_replace_lhs()
2021 return -ENOMEM; in efx_tc_flower_replace_lhs()
2022 rule->cookie = tc->cookie; in efx_tc_flower_replace_lhs()
2023 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, in efx_tc_flower_replace_lhs()
2024 &rule->linkage, in efx_tc_flower_replace_lhs()
2028 goto release; in efx_tc_flower_replace_lhs()
2030 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_lhs()
2031 "Already offloaded rule (cookie %lx)\n", tc->cookie); in efx_tc_flower_replace_lhs()
2032 rc = -EEXIST; in efx_tc_flower_replace_lhs()
2034 goto release; in efx_tc_flower_replace_lhs()
2041 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule); in efx_tc_flower_replace_lhs()
2043 goto release; in efx_tc_flower_replace_lhs()
2045 rule->match = *match; in efx_tc_flower_replace_lhs()
2050 goto release; in efx_tc_flower_replace_lhs()
2052 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_lhs()
2054 tc->cookie); in efx_tc_flower_replace_lhs()
2057 release: in efx_tc_flower_replace_lhs()
2058 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); in efx_tc_flower_replace_lhs()
2060 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, in efx_tc_flower_replace_lhs()
2072 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace()
2084 if (!tc_can_offload_extack(efx->net_dev, extack)) in efx_tc_flower_replace()
2085 return -EOPNOTSUPP; in efx_tc_flower_replace()
2086 if (WARN_ON(!efx->tc)) in efx_tc_flower_replace()
2087 return -ENETDOWN; in efx_tc_flower_replace()
2088 if (WARN_ON(!efx->tc->up)) in efx_tc_flower_replace()
2089 return -ENETDOWN; in efx_tc_flower_replace()
2100 netdev_name(net_dev), efv ? "non-" : "", in efx_tc_flower_replace()
2101 from_efv ? "non-" : ""); in efx_tc_flower_replace()
2102 return -EINVAL; in efx_tc_flower_replace()
2109 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port"); in efx_tc_flower_replace()
2119 return -EOPNOTSUPP; in efx_tc_flower_replace()
2130 if (tc->common.chain_index) { in efx_tc_flower_replace()
2139 * PF, but that would require a bunch of additional IDAs - in efx_tc_flower_replace()
2140 * one for each representor - and that's not likely to be in efx_tc_flower_replace()
2143 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, in efx_tc_flower_replace()
2144 efx->net_dev); in efx_tc_flower_replace()
2148 tc->common.chain_index); in efx_tc_flower_replace()
2152 match.value.recirc_id = rid->fw_id; in efx_tc_flower_replace()
2163 * match +trk-est (CT_HIT=0) despite being on an established connection. in efx_tc_flower_replace()
2164 * So make -est imply -tcp_syn_fin_rst match to ensure these packets in efx_tc_flower_replace()
2170 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2171 goto release; in efx_tc_flower_replace()
2178 goto release; in efx_tc_flower_replace()
2182 rc = -ENOMEM; in efx_tc_flower_replace()
2183 goto release; in efx_tc_flower_replace()
2185 INIT_LIST_HEAD(&rule->acts.list); in efx_tc_flower_replace()
2186 rule->cookie = tc->cookie; in efx_tc_flower_replace()
2187 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace()
2188 &rule->linkage, in efx_tc_flower_replace()
2192 goto release; in efx_tc_flower_replace()
2194 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace()
2195 "Already offloaded rule (cookie %lx)\n", tc->cookie); in efx_tc_flower_replace()
2197 rc = -EEXIST; in efx_tc_flower_replace()
2198 goto release; in efx_tc_flower_replace()
2204 rc = -ENOMEM; in efx_tc_flower_replace()
2205 goto release; in efx_tc_flower_replace()
2217 * which describes the cumulative effect of all the packet-mutating in efx_tc_flower_replace()
2219 * action, once the action-set has been inserted into hardware, we in efx_tc_flower_replace()
2220 * append @act to the action-set list (@rule->acts); if this is a pipe in efx_tc_flower_replace()
2224 * This ensures that every allocated action-set is either attached to in efx_tc_flower_replace()
2225 * @rule->acts or pointed to by @act (and never both), and that only in efx_tc_flower_replace()
2226 * those action-sets in @rule->acts exist in hardware. Consequently, in efx_tc_flower_replace()
2228 * for @rule->acts we remove each action-set from hardware before in efx_tc_flower_replace()
2229 * freeing it (efx_tc_free_action_set_list()), even if the action-set in efx_tc_flower_replace()
2232 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_replace()
2237 /* more actions after a non-pipe action */ in efx_tc_flower_replace()
2238 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); in efx_tc_flower_replace()
2239 rc = -EINVAL; in efx_tc_flower_replace()
2240 goto release; in efx_tc_flower_replace()
2243 if ((fa->id == FLOW_ACTION_REDIRECT || in efx_tc_flower_replace()
2244 fa->id == FLOW_ACTION_MIRRED || in efx_tc_flower_replace()
2245 fa->id == FLOW_ACTION_DROP) && fa->hw_stats) { in efx_tc_flower_replace()
2249 * mirred and gact (ok, shot, trap, goto-chain), which in efx_tc_flower_replace()
2262 NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)"); in efx_tc_flower_replace()
2263 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2264 goto release; in efx_tc_flower_replace()
2267 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { in efx_tc_flower_replace()
2269 fa->hw_stats); in efx_tc_flower_replace()
2270 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2271 goto release; in efx_tc_flower_replace()
2274 ctr = efx_tc_flower_get_counter_index(efx, tc->cookie, in efx_tc_flower_replace()
2279 goto release; in efx_tc_flower_replace()
2281 act->count = ctr; in efx_tc_flower_replace()
2282 INIT_LIST_HEAD(&act->count_user); in efx_tc_flower_replace()
2285 switch (fa->id) { in efx_tc_flower_replace()
2290 goto release; in efx_tc_flower_replace()
2292 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
2304 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2306 goto release; in efx_tc_flower_replace()
2309 efx, encap_info, fa->dev, extack); in efx_tc_flower_replace()
2313 rc = -EIO; /* arbitrary */ in efx_tc_flower_replace()
2314 goto release; in efx_tc_flower_replace()
2316 act->encap_md = encap; in efx_tc_flower_replace()
2317 list_add_tail(&act->encap_user, &encap->users); in efx_tc_flower_replace()
2318 act->dest_mport = encap->dest_mport; in efx_tc_flower_replace()
2319 act->deliver = 1; in efx_tc_flower_replace()
2320 if (act->count && !WARN_ON(!act->count->cnt)) { in efx_tc_flower_replace()
2326 spin_lock_bh(&act->count->cnt->lock); in efx_tc_flower_replace()
2327 list_add_tail(&act->count_user, in efx_tc_flower_replace()
2328 &act->count->cnt->users); in efx_tc_flower_replace()
2329 spin_unlock_bh(&act->count->cnt->lock); in efx_tc_flower_replace()
2334 goto release; in efx_tc_flower_replace()
2336 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
2337 act->user = &rule->acts; in efx_tc_flower_replace()
2339 if (fa->id == FLOW_ACTION_REDIRECT) in efx_tc_flower_replace()
2345 rc = -ENOMEM; in efx_tc_flower_replace()
2346 goto release; in efx_tc_flower_replace()
2354 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2356 goto release; in efx_tc_flower_replace()
2359 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); in efx_tc_flower_replace()
2363 goto release; in efx_tc_flower_replace()
2367 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); in efx_tc_flower_replace()
2368 goto release; in efx_tc_flower_replace()
2370 act->dest_mport = rc; in efx_tc_flower_replace()
2371 act->deliver = 1; in efx_tc_flower_replace()
2375 goto release; in efx_tc_flower_replace()
2377 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
2379 if (fa->id == FLOW_ACTION_REDIRECT) in efx_tc_flower_replace()
2385 rc = -ENOMEM; in efx_tc_flower_replace()
2386 goto release; in efx_tc_flower_replace()
2391 if (act->vlan_push) { in efx_tc_flower_replace()
2392 act->vlan_push--; in efx_tc_flower_replace()
2394 act->vlan_pop++; in efx_tc_flower_replace()
2398 rc = -EINVAL; in efx_tc_flower_replace()
2399 goto release; in efx_tc_flower_replace()
2404 rc = -EINVAL; in efx_tc_flower_replace()
2407 goto release; in efx_tc_flower_replace()
2409 tci = fa->vlan.vid & VLAN_VID_MASK; in efx_tc_flower_replace()
2410 tci |= fa->vlan.prio << VLAN_PRIO_SHIFT; in efx_tc_flower_replace()
2411 act->vlan_tci[act->vlan_push] = cpu_to_be16(tci); in efx_tc_flower_replace()
2412 act->vlan_proto[act->vlan_push] = fa->vlan.proto; in efx_tc_flower_replace()
2413 act->vlan_push++; in efx_tc_flower_replace()
2418 goto release; in efx_tc_flower_replace()
2423 goto release; in efx_tc_flower_replace()
2432 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set"); in efx_tc_flower_replace()
2433 rc = -EINVAL; in efx_tc_flower_replace()
2434 goto release; in efx_tc_flower_replace()
2436 if (!fa->tunnel) { in efx_tc_flower_replace()
2437 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key"); in efx_tc_flower_replace()
2438 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2439 goto release; in efx_tc_flower_replace()
2441 encap_info = fa->tunnel; in efx_tc_flower_replace()
2449 * (and if we did there'd be no tunnel-device to give in efx_tc_flower_replace()
2454 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2455 goto release; in efx_tc_flower_replace()
2457 if (fa->ct.action != TCA_CT_ACT_NAT) { in efx_tc_flower_replace()
2458 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2459 …NL_SET_ERR_MSG_FMT_MOD(extack, "Can only offload CT 'nat' action in RHS rules, not %d", fa->ct.act… in efx_tc_flower_replace()
2460 goto release; in efx_tc_flower_replace()
2462 act->do_nat = 1; in efx_tc_flower_replace()
2466 fa->id); in efx_tc_flower_replace()
2467 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2468 goto release; in efx_tc_flower_replace()
2474 goto release; in efx_tc_flower_replace()
2481 efx_mae_mport_uplink(efx, &act->dest_mport); in efx_tc_flower_replace()
2487 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, in efx_tc_flower_replace()
2488 &act->dest_mport); in efx_tc_flower_replace()
2489 act->deliver = 1; in efx_tc_flower_replace()
2493 goto release; in efx_tc_flower_replace()
2495 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
2496 act = NULL; /* Prevent double-free in error path */ in efx_tc_flower_replace()
2499 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace()
2501 tc->cookie); in efx_tc_flower_replace()
2503 rule->match = match; in efx_tc_flower_replace()
2505 rc = efx_mae_alloc_action_set_list(efx, &rule->acts); in efx_tc_flower_replace()
2508 goto release; in efx_tc_flower_replace()
2512 rule->fallback = &efx->tc->facts.pf; in efx_tc_flower_replace()
2515 rule->fallback = &efx->tc->facts.reps; in efx_tc_flower_replace()
2517 netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n"); in efx_tc_flower_replace()
2518 acts_id = rule->fallback->fw_id; in efx_tc_flower_replace()
2520 netif_dbg(efx, drv, efx->net_dev, "ready for hw\n"); in efx_tc_flower_replace()
2521 acts_id = rule->acts.fw_id; in efx_tc_flower_replace()
2523 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, in efx_tc_flower_replace()
2524 acts_id, &rule->fw_id); in efx_tc_flower_replace()
2532 efx_mae_free_action_set_list(efx, &rule->acts); in efx_tc_flower_replace()
2533 release: in efx_tc_flower_replace()
2543 rhashtable_remove_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace()
2544 &rule->linkage, in efx_tc_flower_replace()
2546 efx_tc_free_action_set_list(efx, &rule->acts, false); in efx_tc_flower_replace()
2556 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_destroy()
2560 lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie, in efx_tc_flower_destroy()
2566 efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act); in efx_tc_flower_destroy()
2567 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage, in efx_tc_flower_destroy()
2569 if (lhs_rule->match.encap) in efx_tc_flower_destroy()
2570 efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap); in efx_tc_flower_destroy()
2571 netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n", in efx_tc_flower_destroy()
2572 lhs_rule->cookie); in efx_tc_flower_destroy()
2577 rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie, in efx_tc_flower_destroy()
2586 netif_warn(efx, drv, efx->net_dev, in efx_tc_flower_destroy()
2587 "Filter %lx not found to remove\n", tc->cookie); in efx_tc_flower_destroy()
2589 return -ENOENT; in efx_tc_flower_destroy()
2595 rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage, in efx_tc_flower_destroy()
2597 netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie); in efx_tc_flower_destroy()
2605 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_stats()
2610 ctr = efx_tc_flower_find_counter_index(efx, tc->cookie); in efx_tc_flower_stats()
2615 netif_warn(efx, drv, efx->net_dev, in efx_tc_flower_stats()
2617 tc->cookie); in efx_tc_flower_stats()
2619 return -ENOENT; in efx_tc_flower_stats()
2621 if (WARN_ON(!ctr->cnt)) /* can't happen */ in efx_tc_flower_stats()
2622 return -EIO; in efx_tc_flower_stats()
2623 cnt = ctr->cnt; in efx_tc_flower_stats()
2625 spin_lock_bh(&cnt->lock); in efx_tc_flower_stats()
2627 packets = cnt->packets; in efx_tc_flower_stats()
2628 bytes = cnt->bytes; in efx_tc_flower_stats()
2629 flow_stats_update(&tc->stats, bytes - cnt->old_bytes, in efx_tc_flower_stats()
2630 packets - cnt->old_packets, 0, cnt->touched, in efx_tc_flower_stats()
2632 cnt->old_packets = packets; in efx_tc_flower_stats()
2633 cnt->old_bytes = bytes; in efx_tc_flower_stats()
2634 spin_unlock_bh(&cnt->lock); in efx_tc_flower_stats()
2643 if (!efx->tc) in efx_tc_flower()
2644 return -EOPNOTSUPP; in efx_tc_flower()
2646 mutex_lock(&efx->tc->mutex); in efx_tc_flower()
2647 switch (tc->command) { in efx_tc_flower()
2658 rc = -EOPNOTSUPP; in efx_tc_flower()
2661 mutex_unlock(&efx->tc->mutex); in efx_tc_flower()
2668 struct efx_tc_action_set_list *acts = &rule->acts; in efx_tc_configure_default_rule()
2669 struct efx_tc_match *match = &rule->match; in efx_tc_configure_default_rule()
2673 match->value.ingress_port = ing_port; in efx_tc_configure_default_rule()
2674 match->mask.ingress_port = ~0; in efx_tc_configure_default_rule()
2677 return -ENOMEM; in efx_tc_configure_default_rule()
2678 act->deliver = 1; in efx_tc_configure_default_rule()
2679 act->dest_mport = eg_port; in efx_tc_configure_default_rule()
2683 EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); in efx_tc_configure_default_rule()
2684 list_add_tail(&act->list, &acts->list); in efx_tc_configure_default_rule()
2689 acts->fw_id, &rule->fw_id); in efx_tc_configure_default_rule()
2696 list_del(&act->list); in efx_tc_configure_default_rule()
2697 efx_mae_free_action_set(efx, act->fw_id); in efx_tc_configure_default_rule()
2705 struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf; in efx_tc_configure_default_rule_pf()
2715 struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire; in efx_tc_configure_default_rule_wire()
2725 struct efx_tc_flow_rule *rule = &efv->dflt; in efx_tc_configure_default_rule_rep()
2726 struct efx_nic *efx = efv->parent; in efx_tc_configure_default_rule_rep()
2729 efx_mae_mport_mport(efx, efv->mport, &ing_port); in efx_tc_configure_default_rule_rep()
2730 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); in efx_tc_configure_default_rule_rep()
2737 if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL) in efx_tc_deconfigure_default_rule()
2739 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_tc_deconfigure_default_rule()
2750 return -ENOMEM; in efx_tc_configure_fallback_acts()
2751 act->deliver = 1; in efx_tc_configure_fallback_acts()
2752 act->dest_mport = eg_port; in efx_tc_configure_fallback_acts()
2756 EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); in efx_tc_configure_fallback_acts()
2757 list_add_tail(&act->list, &acts->list); in efx_tc_configure_fallback_acts()
2763 list_del(&act->list); in efx_tc_configure_fallback_acts()
2764 efx_mae_free_action_set(efx, act->fw_id); in efx_tc_configure_fallback_acts()
2772 struct efx_tc_action_set_list *acts = &efx->tc->facts.pf; in efx_tc_configure_fallback_acts_pf()
2781 struct efx_tc_action_set_list *acts = &efx->tc->facts.reps; in efx_tc_configure_fallback_acts_reps()
2784 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); in efx_tc_configure_fallback_acts_reps()
2799 rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label); in efx_tc_configure_rep_mport()
2802 pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n", in efx_tc_configure_rep_mport()
2803 efx->tc->reps_mport_id, rep_mport_label); in efx_tc_configure_rep_mport()
2805 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, in efx_tc_configure_rep_mport()
2806 &efx->tc->reps_mport_vport_id); in efx_tc_configure_rep_mport()
2812 efx_mae_free_mport(efx, efx->tc->reps_mport_id); in efx_tc_deconfigure_rep_mport()
2813 efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL; in efx_tc_deconfigure_rep_mport()
2821 if (efx->type->is_vf) in efx_tc_insert_rep_filters()
2823 if (!efx->tc) in efx_tc_insert_rep_filters()
2827 efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id); in efx_tc_insert_rep_filters()
2831 efx->tc->reps_filter_uc = rc; in efx_tc_insert_rep_filters()
2834 efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id); in efx_tc_insert_rep_filters()
2838 efx->tc->reps_filter_mc = rc; in efx_tc_insert_rep_filters()
2844 if (efx->type->is_vf) in efx_tc_remove_rep_filters()
2846 if (!efx->tc) in efx_tc_remove_rep_filters()
2848 if (efx->tc->reps_filter_mc >= 0) in efx_tc_remove_rep_filters()
2849 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc); in efx_tc_remove_rep_filters()
2850 efx->tc->reps_filter_mc = -1; in efx_tc_remove_rep_filters()
2851 if (efx->tc->reps_filter_uc >= 0) in efx_tc_remove_rep_filters()
2852 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc); in efx_tc_remove_rep_filters()
2853 efx->tc->reps_filter_uc = -1; in efx_tc_remove_rep_filters()
2860 rc = efx_mae_get_caps(efx, efx->tc->caps); in efx_init_tc()
2863 if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS) in efx_init_tc()
2868 netif_warn(efx, probe, efx->net_dev, in efx_init_tc()
2870 efx->tc->caps->match_field_count); in efx_init_tc()
2871 if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) { in efx_init_tc()
2872 netif_err(efx, probe, efx->net_dev, in efx_init_tc()
2874 efx->tc->caps->action_prios, EFX_TC_PRIO__NUM); in efx_init_tc()
2875 return -EIO; in efx_init_tc()
2898 efx->tc->up = true; in efx_init_tc()
2908 if (!efx->tc) in efx_fini_tc()
2910 if (efx->tc->up) in efx_fini_tc()
2913 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf); in efx_fini_tc()
2914 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire); in efx_fini_tc()
2915 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf); in efx_fini_tc()
2916 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps); in efx_fini_tc()
2917 efx->tc->up = false; in efx_fini_tc()
2929 WARN_ON(refcount_read(&encap->ref)); in efx_tc_encap_match_free()
2938 WARN_ON(refcount_read(&rid->ref)); in efx_tc_recirc_free()
2939 ida_free(&efx->tc->recirc_ida, rid->fw_id); in efx_tc_recirc_free()
2948 netif_err(efx, drv, efx->net_dev, in efx_tc_lhs_free()
2950 rule->cookie); in efx_tc_lhs_free()
2952 if (rule->lhs_act.zone) in efx_tc_lhs_free()
2953 efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone); in efx_tc_lhs_free()
2954 if (rule->lhs_act.count) in efx_tc_lhs_free()
2955 efx_tc_flower_put_counter_index(efx, rule->lhs_act.count); in efx_tc_lhs_free()
2965 WARN_ON(refcount_read(&ped->ref)); in efx_tc_mac_free()
2974 netif_err(efx, drv, efx->net_dev, in efx_tc_flow_free()
2976 rule->cookie); in efx_tc_flow_free()
2988 if (efx->type->is_vf) in efx_init_struct_tc()
2991 efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL); in efx_init_struct_tc()
2992 if (!efx->tc) in efx_init_struct_tc()
2993 return -ENOMEM; in efx_init_struct_tc()
2994 efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL); in efx_init_struct_tc()
2995 if (!efx->tc->caps) { in efx_init_struct_tc()
2996 rc = -ENOMEM; in efx_init_struct_tc()
2999 INIT_LIST_HEAD(&efx->tc->block_list); in efx_init_struct_tc()
3001 mutex_init(&efx->tc->mutex); in efx_init_struct_tc()
3002 init_waitqueue_head(&efx->tc->flush_wq); in efx_init_struct_tc()
3009 rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params); in efx_init_struct_tc()
3012 rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params); in efx_init_struct_tc()
3015 rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params); in efx_init_struct_tc()
3018 rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params); in efx_init_struct_tc()
3024 rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params); in efx_init_struct_tc()
3027 ida_init(&efx->tc->recirc_ida); in efx_init_struct_tc()
3028 efx->tc->reps_filter_uc = -1; in efx_init_struct_tc()
3029 efx->tc->reps_filter_mc = -1; in efx_init_struct_tc()
3030 INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list); in efx_init_struct_tc()
3031 efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_init_struct_tc()
3032 INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list); in efx_init_struct_tc()
3033 efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_init_struct_tc()
3034 INIT_LIST_HEAD(&efx->tc->facts.pf.list); in efx_init_struct_tc()
3035 efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; in efx_init_struct_tc()
3036 INIT_LIST_HEAD(&efx->tc->facts.reps.list); in efx_init_struct_tc()
3037 efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; in efx_init_struct_tc()
3038 efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type; in efx_init_struct_tc()
3043 rhashtable_destroy(&efx->tc->lhs_rule_ht); in efx_init_struct_tc()
3045 rhashtable_destroy(&efx->tc->match_action_ht); in efx_init_struct_tc()
3047 rhashtable_destroy(&efx->tc->encap_match_ht); in efx_init_struct_tc()
3049 rhashtable_destroy(&efx->tc->mac_ht); in efx_init_struct_tc()
3055 mutex_destroy(&efx->tc->mutex); in efx_init_struct_tc()
3056 kfree(efx->tc->caps); in efx_init_struct_tc()
3058 kfree(efx->tc); in efx_init_struct_tc()
3059 efx->tc = NULL; in efx_init_struct_tc()
3065 if (!efx->tc) in efx_fini_struct_tc()
3068 mutex_lock(&efx->tc->mutex); in efx_fini_struct_tc()
3069 EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id != in efx_fini_struct_tc()
3071 EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id != in efx_fini_struct_tc()
3073 EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id != in efx_fini_struct_tc()
3075 EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id != in efx_fini_struct_tc()
3077 rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx); in efx_fini_struct_tc()
3078 rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free, in efx_fini_struct_tc()
3080 rhashtable_free_and_destroy(&efx->tc->encap_match_ht, in efx_fini_struct_tc()
3083 rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx); in efx_fini_struct_tc()
3084 WARN_ON(!ida_is_empty(&efx->tc->recirc_ida)); in efx_fini_struct_tc()
3085 ida_destroy(&efx->tc->recirc_ida); in efx_fini_struct_tc()
3086 rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL); in efx_fini_struct_tc()
3089 mutex_unlock(&efx->tc->mutex); in efx_fini_struct_tc()
3090 mutex_destroy(&efx->tc->mutex); in efx_fini_struct_tc()
3091 kfree(efx->tc->caps); in efx_fini_struct_tc()
3092 kfree(efx->tc); in efx_fini_struct_tc()
3093 efx->tc = NULL; in efx_fini_struct_tc()