1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2019 Solarflare Communications Inc. 5 * Copyright 2020-2022 Xilinx Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 as published 9 * by the Free Software Foundation, incorporated herein by reference. 10 */ 11 12 #include <net/pkt_cls.h> 13 #include <net/vxlan.h> 14 #include <net/geneve.h> 15 #include <net/tc_act/tc_ct.h> 16 #include "tc.h" 17 #include "tc_bindings.h" 18 #include "tc_encap_actions.h" 19 #include "tc_conntrack.h" 20 #include "mae.h" 21 #include "ef100_rep.h" 22 #include "efx.h" 23 24 enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev) 25 { 26 if (netif_is_vxlan(net_dev)) 27 return EFX_ENCAP_TYPE_VXLAN; 28 if (netif_is_geneve(net_dev)) 29 return EFX_ENCAP_TYPE_GENEVE; 30 31 return EFX_ENCAP_TYPE_NONE; 32 } 33 34 #define EFX_TC_HDR_TYPE_TTL_MASK ((u32)0xff) 35 /* Hoplimit is stored in the most significant byte in the pedit ipv6 header action */ 36 #define EFX_TC_HDR_TYPE_HLIMIT_MASK ~((u32)0xff000000) 37 #define EFX_EFV_PF NULL 38 /* Look up the representor information (efv) for a device. 39 * May return NULL for the PF (us), or an error pointer for a device that 40 * isn't supported as a TC offload endpoint 41 */ 42 struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx, 43 struct net_device *dev) 44 { 45 struct efx_rep *efv; 46 47 if (!dev) 48 return ERR_PTR(-EOPNOTSUPP); 49 /* Is it us (the PF)? */ 50 if (dev == efx->net_dev) 51 return EFX_EFV_PF; 52 /* Is it an efx vfrep at all? */ 53 if (dev->netdev_ops != &efx_ef100_rep_netdev_ops) 54 return ERR_PTR(-EOPNOTSUPP); 55 /* Is it ours? We don't support TC rules that include another 56 * EF100's netdevices (not even on another port of the same NIC). 57 */ 58 efv = netdev_priv(dev); 59 if (efv->parent != efx) 60 return ERR_PTR(-EOPNOTSUPP); 61 return efv; 62 } 63 64 /* Convert a driver-internal vport ID into an internal device (PF or VF) */ 65 static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv) 66 { 67 u32 mport; 68 69 if (IS_ERR(efv)) 70 return PTR_ERR(efv); 71 if (!efv) /* device is PF (us) */ 72 efx_mae_mport_uplink(efx, &mport); 73 else /* device is repr */ 74 efx_mae_mport_mport(efx, efv->mport, &mport); 75 return mport; 76 } 77 78 /* Convert a driver-internal vport ID into an external device (wire or VF) */ 79 s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv) 80 { 81 u32 mport; 82 83 if (IS_ERR(efv)) 84 return PTR_ERR(efv); 85 if (!efv) /* device is PF (us) */ 86 efx_mae_mport_wire(efx, &mport); 87 else /* device is repr */ 88 efx_mae_mport_mport(efx, efv->mport, &mport); 89 return mport; 90 } 91 92 static const struct rhashtable_params efx_tc_mac_ht_params = { 93 .key_len = offsetofend(struct efx_tc_mac_pedit_action, h_addr), 94 .key_offset = 0, 95 .head_offset = offsetof(struct efx_tc_mac_pedit_action, linkage), 96 }; 97 98 static const struct rhashtable_params efx_tc_encap_match_ht_params = { 99 .key_len = offsetof(struct efx_tc_encap_match, linkage), 100 .key_offset = 0, 101 .head_offset = offsetof(struct efx_tc_encap_match, linkage), 102 }; 103 104 static const struct rhashtable_params efx_tc_match_action_ht_params = { 105 .key_len = sizeof(unsigned long), 106 .key_offset = offsetof(struct efx_tc_flow_rule, cookie), 107 .head_offset = offsetof(struct efx_tc_flow_rule, linkage), 108 }; 109 110 static const struct rhashtable_params efx_tc_lhs_rule_ht_params = { 111 .key_len = sizeof(unsigned long), 112 .key_offset = offsetof(struct efx_tc_lhs_rule, cookie), 113 .head_offset = offsetof(struct efx_tc_lhs_rule, linkage), 114 }; 115 116 static const struct rhashtable_params efx_tc_recirc_ht_params = { 117 .key_len = offsetof(struct efx_tc_recirc_id, linkage), 118 .key_offset = 0, 119 .head_offset = offsetof(struct efx_tc_recirc_id, linkage), 120 }; 121 122 static struct efx_tc_mac_pedit_action *efx_tc_flower_get_mac(struct efx_nic *efx, 123 unsigned char h_addr[ETH_ALEN], 124 struct netlink_ext_ack *extack) 125 { 126 struct efx_tc_mac_pedit_action *ped, *old; 127 int rc; 128 129 ped = kzalloc_obj(*ped, GFP_USER); 130 if (!ped) 131 return ERR_PTR(-ENOMEM); 132 memcpy(ped->h_addr, h_addr, ETH_ALEN); 133 old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht, 134 &ped->linkage, 135 efx_tc_mac_ht_params); 136 if (old) { 137 /* don't need our new entry */ 138 kfree(ped); 139 if (IS_ERR(old)) /* oh dear, it's actually an error */ 140 return ERR_CAST(old); 141 if (!refcount_inc_not_zero(&old->ref)) 142 return ERR_PTR(-EAGAIN); 143 /* existing entry found, ref taken */ 144 return old; 145 } 146 147 rc = efx_mae_allocate_pedit_mac(efx, ped); 148 if (rc < 0) { 149 NL_SET_ERR_MSG_MOD(extack, "Failed to store pedit MAC address in hw"); 150 goto out_remove; 151 } 152 153 /* ref and return */ 154 refcount_set(&ped->ref, 1); 155 return ped; 156 out_remove: 157 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, 158 efx_tc_mac_ht_params); 159 kfree(ped); 160 return ERR_PTR(rc); 161 } 162 163 static void efx_tc_flower_put_mac(struct efx_nic *efx, 164 struct efx_tc_mac_pedit_action *ped) 165 { 166 if (!refcount_dec_and_test(&ped->ref)) 167 return; /* still in use */ 168 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, 169 efx_tc_mac_ht_params); 170 efx_mae_free_pedit_mac(efx, ped); 171 kfree(ped); 172 } 173 174 static void efx_tc_free_action_set(struct efx_nic *efx, 175 struct efx_tc_action_set *act, bool in_hw) 176 { 177 /* Failure paths calling this on the 'cursor' action set in_hw=false, 178 * because if the alloc had succeeded we'd've put it in acts.list and 179 * not still have it in act. 180 */ 181 if (in_hw) { 182 efx_mae_free_action_set(efx, act->fw_id); 183 /* in_hw is true iff we are on an acts.list; make sure to 184 * remove ourselves from that list before we are freed. 185 */ 186 list_del(&act->list); 187 } 188 if (act->count) { 189 spin_lock_bh(&act->count->cnt->lock); 190 if (!list_empty(&act->count_user)) 191 list_del(&act->count_user); 192 spin_unlock_bh(&act->count->cnt->lock); 193 efx_tc_flower_put_counter_index(efx, act->count); 194 } 195 if (act->encap_md) { 196 list_del(&act->encap_user); 197 efx_tc_flower_release_encap_md(efx, act->encap_md); 198 } 199 if (act->src_mac) 200 efx_tc_flower_put_mac(efx, act->src_mac); 201 if (act->dst_mac) 202 efx_tc_flower_put_mac(efx, act->dst_mac); 203 kfree(act); 204 } 205 206 static void efx_tc_free_action_set_list(struct efx_nic *efx, 207 struct efx_tc_action_set_list *acts, 208 bool in_hw) 209 { 210 struct efx_tc_action_set *act, *next; 211 212 /* Failure paths set in_hw=false, because usually the acts didn't get 213 * to efx_mae_alloc_action_set_list(); if they did, the failure tree 214 * has a separate efx_mae_free_action_set_list() before calling us. 215 */ 216 if (in_hw) 217 efx_mae_free_action_set_list(efx, acts); 218 /* Any act that's on the list will be in_hw even if the list isn't */ 219 list_for_each_entry_safe(act, next, &acts->list, list) 220 efx_tc_free_action_set(efx, act, true); 221 /* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */ 222 } 223 224 /* Boilerplate for the simple 'copy a field' cases */ 225 #define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \ 226 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) { \ 227 struct flow_match_##_type fm; \ 228 \ 229 flow_rule_match_##_tcget(rule, &fm); \ 230 match->value._field = fm.key->_tcfield; \ 231 match->mask._field = fm.mask->_tcfield; \ 232 } 233 #define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field) \ 234 _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field) 235 #define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \ 236 _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field) 237 238 static int efx_tc_flower_parse_match(struct efx_nic *efx, 239 struct flow_rule *rule, 240 struct efx_tc_match *match, 241 struct netlink_ext_ack *extack) 242 { 243 struct flow_dissector *dissector = rule->match.dissector; 244 unsigned char ipv = 0; 245 246 /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set 247 * even on IPv4 filters; so rather than relying on dissector->used_keys 248 * we check the addr_type in the CONTROL key. If we don't find it (or 249 * it's masked, which should never happen), we treat both IPV4_ADDRS 250 * and IPV6_ADDRS as absent. 251 */ 252 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 253 struct flow_match_control fm; 254 255 flow_rule_match_control(rule, &fm); 256 if (IS_ALL_ONES(fm.mask->addr_type)) 257 switch (fm.key->addr_type) { 258 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 259 ipv = 4; 260 break; 261 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 262 ipv = 6; 263 break; 264 default: 265 break; 266 } 267 268 if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) { 269 match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT; 270 match->mask.ip_frag = true; 271 } 272 if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) { 273 match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG; 274 match->mask.ip_firstfrag = true; 275 } 276 if (!flow_rule_is_supp_control_flags(FLOW_DIS_IS_FRAGMENT | 277 FLOW_DIS_FIRST_FRAG, 278 fm.mask->flags, extack)) 279 return -EOPNOTSUPP; 280 } 281 if (dissector->used_keys & 282 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 283 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | 284 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 285 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | 286 BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) | 287 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 288 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 289 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 290 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | 291 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | 292 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | 293 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | 294 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | 295 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | 296 BIT_ULL(FLOW_DISSECTOR_KEY_CT) | 297 BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | 298 BIT_ULL(FLOW_DISSECTOR_KEY_IP))) { 299 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx", 300 dissector->used_keys); 301 return -EOPNOTSUPP; 302 } 303 304 MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto); 305 /* Make sure we're IP if any L3/L4 keys used. */ 306 if (!IS_ALL_ONES(match->mask.eth_proto) || 307 !(match->value.eth_proto == htons(ETH_P_IP) || 308 match->value.eth_proto == htons(ETH_P_IPV6))) 309 if (dissector->used_keys & 310 (BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 311 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 312 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 313 BIT_ULL(FLOW_DISSECTOR_KEY_IP) | 314 BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) { 315 NL_SET_ERR_MSG_FMT_MOD(extack, 316 "L3/L4 flower keys %#llx require protocol ipv[46]", 317 dissector->used_keys); 318 return -EINVAL; 319 } 320 321 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 322 struct flow_match_vlan fm; 323 324 flow_rule_match_vlan(rule, &fm); 325 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { 326 match->value.vlan_proto[0] = fm.key->vlan_tpid; 327 match->mask.vlan_proto[0] = fm.mask->vlan_tpid; 328 match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 | 329 fm.key->vlan_id); 330 match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 | 331 fm.mask->vlan_id); 332 } 333 } 334 335 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) { 336 struct flow_match_vlan fm; 337 338 flow_rule_match_cvlan(rule, &fm); 339 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { 340 match->value.vlan_proto[1] = fm.key->vlan_tpid; 341 match->mask.vlan_proto[1] = fm.mask->vlan_tpid; 342 match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 | 343 fm.key->vlan_id); 344 match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 | 345 fm.mask->vlan_id); 346 } 347 } 348 349 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 350 struct flow_match_eth_addrs fm; 351 352 flow_rule_match_eth_addrs(rule, &fm); 353 ether_addr_copy(match->value.eth_saddr, fm.key->src); 354 ether_addr_copy(match->value.eth_daddr, fm.key->dst); 355 ether_addr_copy(match->mask.eth_saddr, fm.mask->src); 356 ether_addr_copy(match->mask.eth_daddr, fm.mask->dst); 357 } 358 359 MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto); 360 /* Make sure we're TCP/UDP if any L4 keys used. */ 361 if ((match->value.ip_proto != IPPROTO_UDP && 362 match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto)) 363 if (dissector->used_keys & 364 (BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 365 BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) { 366 NL_SET_ERR_MSG_FMT_MOD(extack, 367 "L4 flower keys %#llx require ipproto udp or tcp", 368 dissector->used_keys); 369 return -EINVAL; 370 } 371 MAP_KEY_AND_MASK(IP, ip, tos, ip_tos); 372 MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl); 373 if (ipv == 4) { 374 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip); 375 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip); 376 } 377 #ifdef CONFIG_IPV6 378 else if (ipv == 6) { 379 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6); 380 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6); 381 } 382 #endif 383 MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport); 384 MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport); 385 MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags); 386 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 387 struct flow_match_control fm; 388 389 flow_rule_match_enc_control(rule, &fm); 390 if (flow_rule_has_enc_control_flags(fm.mask->flags, extack)) 391 return -EOPNOTSUPP; 392 if (!IS_ALL_ONES(fm.mask->addr_type)) { 393 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)", 394 fm.mask->addr_type, 395 fm.key->addr_type); 396 return -EOPNOTSUPP; 397 } 398 switch (fm.key->addr_type) { 399 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 400 MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs, 401 src, enc_src_ip); 402 MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs, 403 dst, enc_dst_ip); 404 break; 405 #ifdef CONFIG_IPV6 406 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 407 MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs, 408 src, enc_src_ip6); 409 MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs, 410 dst, enc_dst_ip6); 411 break; 412 #endif 413 default: 414 NL_SET_ERR_MSG_FMT_MOD(extack, 415 "Unsupported enc addr_type %u (supported are IPv4, IPv6)", 416 fm.key->addr_type); 417 return -EOPNOTSUPP; 418 } 419 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos); 420 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl); 421 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport); 422 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport); 423 MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid); 424 } else if (dissector->used_keys & 425 (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | 426 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | 427 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | 428 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | 429 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) { 430 NL_SET_ERR_MSG_FMT_MOD(extack, 431 "Flower enc keys require enc_control (keys: %#llx)", 432 dissector->used_keys); 433 return -EOPNOTSUPP; 434 } 435 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) { 436 struct flow_match_ct fm; 437 438 flow_rule_match_ct(rule, &fm); 439 match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); 440 match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); 441 match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); 442 match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); 443 if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 444 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)) { 445 NL_SET_ERR_MSG_FMT_MOD(extack, 446 "Unsupported ct_state match %#x", 447 fm.mask->ct_state); 448 return -EOPNOTSUPP; 449 } 450 match->value.ct_mark = fm.key->ct_mark; 451 match->mask.ct_mark = fm.mask->ct_mark; 452 match->value.ct_zone = fm.key->ct_zone; 453 match->mask.ct_zone = fm.mask->ct_zone; 454 455 if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) { 456 NL_SET_ERR_MSG_MOD(extack, "Matching on ct_label not supported"); 457 return -EOPNOTSUPP; 458 } 459 } 460 461 return 0; 462 } 463 464 static void efx_tc_flower_release_encap_match(struct efx_nic *efx, 465 struct efx_tc_encap_match *encap) 466 { 467 int rc; 468 469 if (!refcount_dec_and_test(&encap->ref)) 470 return; /* still in use */ 471 472 if (encap->type == EFX_TC_EM_DIRECT) { 473 rc = efx_mae_unregister_encap_match(efx, encap); 474 if (rc) 475 /* Display message but carry on and remove entry from our 476 * SW tables, because there's not much we can do about it. 477 */ 478 netif_err(efx, drv, efx->net_dev, 479 "Failed to release encap match %#x, rc %d\n", 480 encap->fw_id, rc); 481 } 482 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, 483 efx_tc_encap_match_ht_params); 484 if (encap->pseudo) 485 efx_tc_flower_release_encap_match(efx, encap->pseudo); 486 kfree(encap); 487 } 488 489 static int efx_tc_flower_record_encap_match(struct efx_nic *efx, 490 struct efx_tc_match *match, 491 enum efx_encap_type type, 492 enum efx_tc_em_pseudo_type em_type, 493 u8 child_ip_tos_mask, 494 __be16 child_udp_sport_mask, 495 struct netlink_ext_ack *extack) 496 { 497 struct efx_tc_encap_match *encap, *old, *pseudo = NULL; 498 bool ipv6 = false; 499 int rc; 500 501 /* We require that the socket-defining fields (IP addrs and UDP dest 502 * port) are present and exact-match. Other fields may only be used 503 * if the field-set (and any masks) are the same for all encap 504 * matches on the same <sip,dip,dport> tuple; this is enforced by 505 * pseudo encap matches. 506 */ 507 if (match->mask.enc_dst_ip | match->mask.enc_src_ip) { 508 if (!IS_ALL_ONES(match->mask.enc_dst_ip)) { 509 NL_SET_ERR_MSG_MOD(extack, 510 "Egress encap match is not exact on dst IP address"); 511 return -EOPNOTSUPP; 512 } 513 if (!IS_ALL_ONES(match->mask.enc_src_ip)) { 514 NL_SET_ERR_MSG_MOD(extack, 515 "Egress encap match is not exact on src IP address"); 516 return -EOPNOTSUPP; 517 } 518 #ifdef CONFIG_IPV6 519 if (!ipv6_addr_any(&match->mask.enc_dst_ip6) || 520 !ipv6_addr_any(&match->mask.enc_src_ip6)) { 521 NL_SET_ERR_MSG_MOD(extack, 522 "Egress encap match on both IPv4 and IPv6, don't understand"); 523 return -EOPNOTSUPP; 524 } 525 } else { 526 ipv6 = true; 527 if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) { 528 NL_SET_ERR_MSG_MOD(extack, 529 "Egress encap match is not exact on dst IP address"); 530 return -EOPNOTSUPP; 531 } 532 if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) { 533 NL_SET_ERR_MSG_MOD(extack, 534 "Egress encap match is not exact on src IP address"); 535 return -EOPNOTSUPP; 536 } 537 #endif 538 } 539 if (!IS_ALL_ONES(match->mask.enc_dport)) { 540 NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port"); 541 return -EOPNOTSUPP; 542 } 543 if (match->mask.enc_sport || match->mask.enc_ip_tos) { 544 struct efx_tc_match pmatch = *match; 545 546 if (em_type == EFX_TC_EM_PSEUDO_MASK) { /* can't happen */ 547 NL_SET_ERR_MSG_MOD(extack, "Bad recursion in egress encap match handler"); 548 return -EOPNOTSUPP; 549 } 550 pmatch.value.enc_ip_tos = 0; 551 pmatch.mask.enc_ip_tos = 0; 552 pmatch.value.enc_sport = 0; 553 pmatch.mask.enc_sport = 0; 554 rc = efx_tc_flower_record_encap_match(efx, &pmatch, type, 555 EFX_TC_EM_PSEUDO_MASK, 556 match->mask.enc_ip_tos, 557 match->mask.enc_sport, 558 extack); 559 if (rc) 560 return rc; 561 pseudo = pmatch.encap; 562 } 563 if (match->mask.enc_ip_ttl) { 564 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported"); 565 rc = -EOPNOTSUPP; 566 goto fail_pseudo; 567 } 568 569 rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos, 570 match->mask.enc_sport, extack); 571 if (rc) 572 goto fail_pseudo; 573 574 encap = kzalloc_obj(*encap, GFP_USER); 575 if (!encap) { 576 rc = -ENOMEM; 577 goto fail_pseudo; 578 } 579 encap->src_ip = match->value.enc_src_ip; 580 encap->dst_ip = match->value.enc_dst_ip; 581 #ifdef CONFIG_IPV6 582 encap->src_ip6 = match->value.enc_src_ip6; 583 encap->dst_ip6 = match->value.enc_dst_ip6; 584 #endif 585 encap->udp_dport = match->value.enc_dport; 586 encap->tun_type = type; 587 encap->ip_tos = match->value.enc_ip_tos; 588 encap->ip_tos_mask = match->mask.enc_ip_tos; 589 encap->child_ip_tos_mask = child_ip_tos_mask; 590 encap->udp_sport = match->value.enc_sport; 591 encap->udp_sport_mask = match->mask.enc_sport; 592 encap->child_udp_sport_mask = child_udp_sport_mask; 593 encap->type = em_type; 594 encap->pseudo = pseudo; 595 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht, 596 &encap->linkage, 597 efx_tc_encap_match_ht_params); 598 if (old) { 599 /* don't need our new entry */ 600 kfree(encap); 601 if (pseudo) /* don't need our new pseudo either */ 602 efx_tc_flower_release_encap_match(efx, pseudo); 603 if (IS_ERR(old)) /* oh dear, it's actually an error */ 604 return PTR_ERR(old); 605 /* check old and new em_types are compatible */ 606 switch (old->type) { 607 case EFX_TC_EM_DIRECT: 608 /* old EM is in hardware, so mustn't overlap with a 609 * pseudo, but may be shared with another direct EM 610 */ 611 if (em_type == EFX_TC_EM_DIRECT) 612 break; 613 NL_SET_ERR_MSG_MOD(extack, "Pseudo encap match conflicts with existing direct entry"); 614 return -EEXIST; 615 case EFX_TC_EM_PSEUDO_MASK: 616 /* old EM is protecting a ToS- or src port-qualified 617 * filter, so may only be shared with another pseudo 618 * for the same ToS and src port masks. 619 */ 620 if (em_type != EFX_TC_EM_PSEUDO_MASK) { 621 NL_SET_ERR_MSG_FMT_MOD(extack, 622 "%s encap match conflicts with existing pseudo(MASK) entry", 623 em_type ? "Pseudo" : "Direct"); 624 return -EEXIST; 625 } 626 if (child_ip_tos_mask != old->child_ip_tos_mask) { 627 NL_SET_ERR_MSG_FMT_MOD(extack, 628 "Pseudo encap match for TOS mask %#04x conflicts with existing mask %#04x", 629 child_ip_tos_mask, 630 old->child_ip_tos_mask); 631 return -EEXIST; 632 } 633 if (child_udp_sport_mask != old->child_udp_sport_mask) { 634 NL_SET_ERR_MSG_FMT_MOD(extack, 635 "Pseudo encap match for UDP src port mask %#x conflicts with existing mask %#x", 636 child_udp_sport_mask, 637 old->child_udp_sport_mask); 638 return -EEXIST; 639 } 640 break; 641 case EFX_TC_EM_PSEUDO_OR: 642 /* old EM corresponds to an OR that has to be unique 643 * (it must not overlap with any other OR, whether 644 * direct-EM or pseudo). 645 */ 646 NL_SET_ERR_MSG_FMT_MOD(extack, 647 "%s encap match conflicts with existing pseudo(OR) entry", 648 em_type ? "Pseudo" : "Direct"); 649 return -EEXIST; 650 default: /* Unrecognised pseudo-type. Just say no */ 651 NL_SET_ERR_MSG_FMT_MOD(extack, 652 "%s encap match conflicts with existing pseudo(%d) entry", 653 em_type ? "Pseudo" : "Direct", 654 old->type); 655 return -EEXIST; 656 } 657 /* check old and new tun_types are compatible */ 658 if (old->tun_type != type) { 659 NL_SET_ERR_MSG_FMT_MOD(extack, 660 "Egress encap match with conflicting tun_type %u != %u", 661 old->tun_type, type); 662 return -EEXIST; 663 } 664 if (!refcount_inc_not_zero(&old->ref)) 665 return -EAGAIN; 666 /* existing entry found */ 667 encap = old; 668 } else { 669 if (em_type == EFX_TC_EM_DIRECT) { 670 rc = efx_mae_register_encap_match(efx, encap); 671 if (rc) { 672 NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW"); 673 goto fail; 674 } 675 } 676 refcount_set(&encap->ref, 1); 677 } 678 match->encap = encap; 679 return 0; 680 fail: 681 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, 682 efx_tc_encap_match_ht_params); 683 kfree(encap); 684 fail_pseudo: 685 if (pseudo) 686 efx_tc_flower_release_encap_match(efx, pseudo); 687 return rc; 688 } 689 690 static struct efx_tc_recirc_id *efx_tc_get_recirc_id(struct efx_nic *efx, 691 u32 chain_index, 692 struct net_device *net_dev) 693 { 694 struct efx_tc_recirc_id *rid, *old; 695 int rc; 696 697 rid = kzalloc_obj(*rid, GFP_USER); 698 if (!rid) 699 return ERR_PTR(-ENOMEM); 700 rid->chain_index = chain_index; 701 /* We don't take a reference here, because it's implied - if there's 702 * a rule on the net_dev that's been offloaded to us, then the net_dev 703 * can't go away until the rule has been deoffloaded. 704 */ 705 rid->net_dev = net_dev; 706 old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht, 707 &rid->linkage, 708 efx_tc_recirc_ht_params); 709 if (old) { 710 /* don't need our new entry */ 711 kfree(rid); 712 if (IS_ERR(old)) /* oh dear, it's actually an error */ 713 return ERR_CAST(old); 714 if (!refcount_inc_not_zero(&old->ref)) 715 return ERR_PTR(-EAGAIN); 716 /* existing entry found */ 717 rid = old; 718 } else { 719 rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER); 720 if (rc < 0) { 721 rhashtable_remove_fast(&efx->tc->recirc_ht, 722 &rid->linkage, 723 efx_tc_recirc_ht_params); 724 kfree(rid); 725 return ERR_PTR(rc); 726 } 727 rid->fw_id = rc; 728 refcount_set(&rid->ref, 1); 729 } 730 return rid; 731 } 732 733 static void efx_tc_put_recirc_id(struct efx_nic *efx, struct efx_tc_recirc_id *rid) 734 { 735 if (!refcount_dec_and_test(&rid->ref)) 736 return; /* still in use */ 737 rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage, 738 efx_tc_recirc_ht_params); 739 ida_free(&efx->tc->recirc_ida, rid->fw_id); 740 kfree(rid); 741 } 742 743 static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule) 744 { 745 efx_mae_delete_rule(efx, rule->fw_id); 746 747 /* Release entries in subsidiary tables */ 748 efx_tc_free_action_set_list(efx, &rule->acts, true); 749 if (rule->match.rid) 750 efx_tc_put_recirc_id(efx, rule->match.rid); 751 if (rule->match.encap) 752 efx_tc_flower_release_encap_match(efx, rule->match.encap); 753 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; 754 } 755 756 static const char *efx_tc_encap_type_name(enum efx_encap_type typ) 757 { 758 switch (typ) { 759 case EFX_ENCAP_TYPE_NONE: 760 return "none"; 761 case EFX_ENCAP_TYPE_VXLAN: 762 return "vxlan"; 763 case EFX_ENCAP_TYPE_GENEVE: 764 return "geneve"; 765 default: 766 pr_warn_once("Unknown efx_encap_type %d encountered\n", typ); 767 return "unknown"; 768 } 769 } 770 771 /* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */ 772 enum efx_tc_action_order { 773 EFX_TC_AO_DECAP, 774 EFX_TC_AO_DEC_TTL, 775 EFX_TC_AO_PEDIT_MAC_ADDRS, 776 EFX_TC_AO_VLAN_POP, 777 EFX_TC_AO_VLAN_PUSH, 778 EFX_TC_AO_COUNT, 779 EFX_TC_AO_ENCAP, 780 EFX_TC_AO_DELIVER 781 }; 782 /* Determine whether we can add @new action without violating order */ 783 static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act, 784 enum efx_tc_action_order new) 785 { 786 switch (new) { 787 case EFX_TC_AO_DECAP: 788 if (act->decap) 789 return false; 790 /* PEDIT_MAC_ADDRS must not happen before DECAP, though it 791 * can wait until much later 792 */ 793 if (act->dst_mac || act->src_mac) 794 return false; 795 796 /* Decrementing ttl must not happen before DECAP */ 797 if (act->do_ttl_dec) 798 return false; 799 fallthrough; 800 case EFX_TC_AO_VLAN_POP: 801 if (act->vlan_pop >= 2) 802 return false; 803 /* If we've already pushed a VLAN, we can't then pop it; 804 * the hardware would instead try to pop an existing VLAN 805 * before pushing the new one. 806 */ 807 if (act->vlan_push) 808 return false; 809 fallthrough; 810 case EFX_TC_AO_VLAN_PUSH: 811 if (act->vlan_push >= 2) 812 return false; 813 fallthrough; 814 case EFX_TC_AO_COUNT: 815 if (act->count) 816 return false; 817 fallthrough; 818 case EFX_TC_AO_PEDIT_MAC_ADDRS: 819 case EFX_TC_AO_ENCAP: 820 if (act->encap_md) 821 return false; 822 fallthrough; 823 case EFX_TC_AO_DELIVER: 824 return !act->deliver; 825 case EFX_TC_AO_DEC_TTL: 826 if (act->encap_md) 827 return false; 828 return !act->do_ttl_dec; 829 default: 830 /* Bad caller. Whatever they wanted to do, say they can't. */ 831 WARN_ON_ONCE(1); 832 return false; 833 } 834 } 835 836 /** 837 * DOC: TC conntrack sequences 838 * 839 * The MAE hardware can handle at most two rounds of action rule matching, 840 * consequently we support conntrack through the notion of a "left-hand side 841 * rule". This is a rule which typically contains only the actions "ct" and 842 * "goto chain N", and corresponds to one or more "right-hand side rules" in 843 * chain N, which typically match on +trk+est, and may perform ct(nat) actions. 844 * RHS rules go in the Action Rule table as normal but with a nonzero recirc_id 845 * (the hardware equivalent of chain_index), while LHS rules may go in either 846 * the Action Rule or the Outer Rule table, the latter being preferred for 847 * performance reasons, and set both DO_CT and a recirc_id in their response. 848 * 849 * Besides the RHS rules, there are often also similar rules matching on 850 * +trk+new which perform the ct(commit) action. These are not offloaded. 851 */ 852 853 static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr, 854 struct efx_tc_match *match) 855 { 856 const struct flow_action_entry *fa; 857 int i; 858 859 flow_action_for_each(i, fa, &fr->action) { 860 switch (fa->id) { 861 case FLOW_ACTION_GOTO: 862 return true; 863 case FLOW_ACTION_CT: 864 /* If rule is -trk, or doesn't mention trk at all, then 865 * a CT action implies a conntrack lookup (hence it's an 866 * LHS rule). If rule is +trk, then a CT action could 867 * just be ct(nat) or even ct(commit) (though the latter 868 * can't be offloaded). 869 */ 870 if (!match->mask.ct_state_trk || !match->value.ct_state_trk) 871 return true; 872 break; 873 default: 874 break; 875 } 876 } 877 return false; 878 } 879 880 /* A foreign LHS rule has matches on enc_ keys at the TC layer (including an 881 * implied match on enc_ip_proto UDP). Translate these into non-enc_ keys, 882 * so that we can use the same MAE machinery as local LHS rules (and so that 883 * the lhs_rules entries have uniform semantics). It may seem odd to do it 884 * this way round, given that the corresponding fields in the MAE MCDIs are 885 * all ENC_, but (a) we don't have enc_L2 or enc_ip_proto in struct 886 * efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner 887 * fields so it's just matching on *the* header rather than the outer header. 888 * Make sure that the non-enc_ keys were not already being matched on, as that 889 * would imply a rule that needed a triple lookup. (Hardware can do that, 890 * with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible; 891 * see efx_tc_flower_flhs_needs_ar().) 892 */ 893 static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match) 894 { 895 int rc = 0; 896 897 #define COPY_MASK_AND_VALUE(_key, _ekey) ({ \ 898 if (match->mask._key) { \ 899 rc = -EOPNOTSUPP; \ 900 } else { \ 901 match->mask._key = match->mask._ekey; \ 902 match->mask._ekey = 0; \ 903 match->value._key = match->value._ekey; \ 904 match->value._ekey = 0; \ 905 } \ 906 rc; \ 907 }) 908 #define COPY_FROM_ENC(_key) COPY_MASK_AND_VALUE(_key, enc_##_key) 909 if (match->mask.ip_proto) 910 return -EOPNOTSUPP; 911 match->mask.ip_proto = ~0; 912 match->value.ip_proto = IPPROTO_UDP; 913 if (COPY_FROM_ENC(src_ip) || COPY_FROM_ENC(dst_ip)) 914 return rc; 915 #ifdef CONFIG_IPV6 916 if (!ipv6_addr_any(&match->mask.src_ip6)) 917 return -EOPNOTSUPP; 918 match->mask.src_ip6 = match->mask.enc_src_ip6; 919 memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr)); 920 if (!ipv6_addr_any(&match->mask.dst_ip6)) 921 return -EOPNOTSUPP; 922 match->mask.dst_ip6 = match->mask.enc_dst_ip6; 923 memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr)); 924 #endif 925 if (COPY_FROM_ENC(ip_tos) || COPY_FROM_ENC(ip_ttl)) 926 return rc; 927 /* should really copy enc_ip_frag but we don't have that in 928 * parse_match yet 929 */ 930 if (COPY_MASK_AND_VALUE(l4_sport, enc_sport) || 931 COPY_MASK_AND_VALUE(l4_dport, enc_dport)) 932 return rc; 933 return 0; 934 #undef COPY_FROM_ENC 935 #undef COPY_MASK_AND_VALUE 936 } 937 938 /* If a foreign LHS rule wants to match on keys that are only available after 939 * encap header identification and parsing, then it can't be done in the Outer 940 * Rule lookup, because that lookup determines the encap type used to parse 941 * beyond the outer headers. Thus, such rules must use the OR-AR-CT-AR lookup 942 * sequence, with an EM (struct efx_tc_encap_match) in the OR step. 943 * Return true iff the passed match requires this. 944 */ 945 static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match) 946 { 947 /* matches on inner-header keys can't be done in OR */ 948 return match->mask.eth_proto || 949 match->mask.vlan_tci[0] || match->mask.vlan_tci[1] || 950 match->mask.vlan_proto[0] || match->mask.vlan_proto[1] || 951 memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) || 952 memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) || 953 match->mask.ip_proto || 954 match->mask.ip_tos || match->mask.ip_ttl || 955 match->mask.src_ip || match->mask.dst_ip || 956 #ifdef CONFIG_IPV6 957 !ipv6_addr_any(&match->mask.src_ip6) || 958 !ipv6_addr_any(&match->mask.dst_ip6) || 959 #endif 960 match->mask.ip_frag || match->mask.ip_firstfrag || 961 match->mask.l4_sport || match->mask.l4_dport || 962 match->mask.tcp_flags || 963 /* nor can VNI */ 964 match->mask.enc_keyid; 965 } 966 967 static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx, 968 struct flow_cls_offload *tc, 969 struct flow_rule *fr, 970 struct net_device *net_dev, 971 struct efx_tc_lhs_rule *rule) 972 973 { 974 struct netlink_ext_ack *extack = tc->common.extack; 975 struct efx_tc_lhs_action *act = &rule->lhs_act; 976 const struct flow_action_entry *fa; 977 enum efx_tc_counter_type ctype; 978 bool pipe = true; 979 int i; 980 981 ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR; 982 983 flow_action_for_each(i, fa, &fr->action) { 984 struct efx_tc_ct_zone *ct_zone; 985 struct efx_tc_recirc_id *rid; 986 987 if (!pipe) { 988 /* more actions after a non-pipe action */ 989 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); 990 return -EINVAL; 991 } 992 switch (fa->id) { 993 case FLOW_ACTION_GOTO: 994 if (!fa->chain_index) { 995 NL_SET_ERR_MSG_MOD(extack, "Can't goto chain 0, no looping in hw"); 996 return -EOPNOTSUPP; 997 } 998 rid = efx_tc_get_recirc_id(efx, fa->chain_index, 999 net_dev); 1000 if (IS_ERR(rid)) { 1001 NL_SET_ERR_MSG_MOD(extack, "Failed to allocate a hardware recirculation ID for this chain_index"); 1002 return PTR_ERR(rid); 1003 } 1004 act->rid = rid; 1005 if (fa->hw_stats) { 1006 struct efx_tc_counter_index *cnt; 1007 1008 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { 1009 NL_SET_ERR_MSG_FMT_MOD(extack, 1010 "hw_stats_type %u not supported (only 'delayed')", 1011 fa->hw_stats); 1012 return -EOPNOTSUPP; 1013 } 1014 cnt = efx_tc_flower_get_counter_index(efx, tc->cookie, 1015 ctype); 1016 if (IS_ERR(cnt)) { 1017 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter"); 1018 return PTR_ERR(cnt); 1019 } 1020 WARN_ON(act->count); /* can't happen */ 1021 act->count = cnt; 1022 } 1023 pipe = false; 1024 break; 1025 case FLOW_ACTION_CT: 1026 if (act->zone) { 1027 NL_SET_ERR_MSG_MOD(extack, "Can't offload multiple ct actions"); 1028 return -EOPNOTSUPP; 1029 } 1030 if (fa->ct.action & (TCA_CT_ACT_COMMIT | 1031 TCA_CT_ACT_FORCE)) { 1032 NL_SET_ERR_MSG_MOD(extack, "Can't offload ct commit/force"); 1033 return -EOPNOTSUPP; 1034 } 1035 if (fa->ct.action & TCA_CT_ACT_CLEAR) { 1036 NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule"); 1037 return -EOPNOTSUPP; 1038 } 1039 if (fa->ct.action & (TCA_CT_ACT_NAT | 1040 TCA_CT_ACT_NAT_SRC | 1041 TCA_CT_ACT_NAT_DST)) { 1042 NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet"); 1043 return -EOPNOTSUPP; 1044 } 1045 if (fa->ct.action) { 1046 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule", 1047 fa->ct.action); 1048 return -EOPNOTSUPP; 1049 } 1050 ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone, 1051 fa->ct.flow_table); 1052 if (IS_ERR(ct_zone)) { 1053 NL_SET_ERR_MSG_MOD(extack, "Failed to register for CT updates"); 1054 return PTR_ERR(ct_zone); 1055 } 1056 act->zone = ct_zone; 1057 break; 1058 default: 1059 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule", 1060 fa->id); 1061 return -EOPNOTSUPP; 1062 } 1063 } 1064 1065 if (pipe) { 1066 NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule"); 1067 return -EOPNOTSUPP; 1068 } 1069 return 0; 1070 } 1071 1072 static void efx_tc_flower_release_lhs_actions(struct efx_nic *efx, 1073 struct efx_tc_lhs_action *act) 1074 { 1075 if (act->rid) 1076 efx_tc_put_recirc_id(efx, act->rid); 1077 if (act->zone) 1078 efx_tc_ct_unregister_zone(efx, act->zone); 1079 if (act->count) 1080 efx_tc_flower_put_counter_index(efx, act->count); 1081 } 1082 1083 /** 1084 * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields 1085 * 1086 * @dst_mac_32: dst_mac[0:3] has been populated 1087 * @dst_mac_16: dst_mac[4:5] has been populated 1088 * @src_mac_16: src_mac[0:1] has been populated 1089 * @src_mac_32: src_mac[2:5] has been populated 1090 * @dst_mac: h_dest field of ethhdr 1091 * @src_mac: h_source field of ethhdr 1092 * 1093 * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not 1094 * necessarily equate to whole fields of the packet header, this 1095 * structure is used to hold the cumulative effect of the partial 1096 * field pedits that have been processed so far. 1097 */ 1098 struct efx_tc_mangler_state { 1099 u8 dst_mac_32:1; /* eth->h_dest[0:3] */ 1100 u8 dst_mac_16:1; /* eth->h_dest[4:5] */ 1101 u8 src_mac_16:1; /* eth->h_source[0:1] */ 1102 u8 src_mac_32:1; /* eth->h_source[2:5] */ 1103 unsigned char dst_mac[ETH_ALEN]; 1104 unsigned char src_mac[ETH_ALEN]; 1105 }; 1106 1107 /** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung 1108 * @efx: NIC we're installing a flow rule on 1109 * @act: action set (cursor) to update 1110 * @mung: accumulated partial mangles 1111 * @extack: netlink extended ack for reporting errors 1112 * 1113 * Check @mung to find any combinations of partial mangles that can be 1114 * combined into a complete packet field edit, add that edit to @act, 1115 * and consume the partial mangles from @mung. 1116 */ 1117 1118 static int efx_tc_complete_mac_mangle(struct efx_nic *efx, 1119 struct efx_tc_action_set *act, 1120 struct efx_tc_mangler_state *mung, 1121 struct netlink_ext_ack *extack) 1122 { 1123 struct efx_tc_mac_pedit_action *ped; 1124 1125 if (mung->dst_mac_32 && mung->dst_mac_16) { 1126 ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack); 1127 if (IS_ERR(ped)) 1128 return PTR_ERR(ped); 1129 1130 /* Check that we have not already populated dst_mac */ 1131 if (act->dst_mac) 1132 efx_tc_flower_put_mac(efx, act->dst_mac); 1133 1134 act->dst_mac = ped; 1135 1136 /* consume the incomplete state */ 1137 mung->dst_mac_32 = 0; 1138 mung->dst_mac_16 = 0; 1139 } 1140 if (mung->src_mac_16 && mung->src_mac_32) { 1141 ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack); 1142 if (IS_ERR(ped)) 1143 return PTR_ERR(ped); 1144 1145 /* Check that we have not already populated src_mac */ 1146 if (act->src_mac) 1147 efx_tc_flower_put_mac(efx, act->src_mac); 1148 1149 act->src_mac = ped; 1150 1151 /* consume the incomplete state */ 1152 mung->src_mac_32 = 0; 1153 mung->src_mac_16 = 0; 1154 } 1155 return 0; 1156 } 1157 1158 static int efx_tc_pedit_add(struct efx_nic *efx, struct efx_tc_action_set *act, 1159 const struct flow_action_entry *fa, 1160 struct netlink_ext_ack *extack) 1161 { 1162 switch (fa->mangle.htype) { 1163 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 1164 switch (fa->mangle.offset) { 1165 case offsetof(struct iphdr, ttl): 1166 /* check that pedit applies to ttl only */ 1167 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) 1168 break; 1169 1170 /* Adding 0xff is equivalent to decrementing the ttl. 1171 * Other added values are not supported. 1172 */ 1173 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX) 1174 break; 1175 1176 /* check that we do not decrement ttl twice */ 1177 if (!efx_tc_flower_action_order_ok(act, 1178 EFX_TC_AO_DEC_TTL)) { 1179 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported"); 1180 return -EOPNOTSUPP; 1181 } 1182 act->do_ttl_dec = 1; 1183 return 0; 1184 default: 1185 break; 1186 } 1187 break; 1188 case FLOW_ACT_MANGLE_HDR_TYPE_IP6: 1189 switch (fa->mangle.offset) { 1190 case round_down(offsetof(struct ipv6hdr, hop_limit), 4): 1191 /* check that pedit applies to hoplimit only */ 1192 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) 1193 break; 1194 1195 /* Adding 0xff is equivalent to decrementing the hoplimit. 1196 * Other added values are not supported. 1197 */ 1198 if ((fa->mangle.val >> 24) != U8_MAX) 1199 break; 1200 1201 /* check that we do not decrement hoplimit twice */ 1202 if (!efx_tc_flower_action_order_ok(act, 1203 EFX_TC_AO_DEC_TTL)) { 1204 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported"); 1205 return -EOPNOTSUPP; 1206 } 1207 act->do_ttl_dec = 1; 1208 return 0; 1209 default: 1210 break; 1211 } 1212 break; 1213 default: 1214 break; 1215 } 1216 1217 NL_SET_ERR_MSG_FMT_MOD(extack, 1218 "ttl add action type %x %x %x/%x is not supported", 1219 fa->mangle.htype, fa->mangle.offset, 1220 fa->mangle.val, fa->mangle.mask); 1221 return -EOPNOTSUPP; 1222 } 1223 1224 /** 1225 * efx_tc_mangle() - handle a single 32-bit (or less) pedit 1226 * @efx: NIC we're installing a flow rule on 1227 * @act: action set (cursor) to update 1228 * @fa: FLOW_ACTION_MANGLE action metadata 1229 * @mung: accumulator for partial mangles 1230 * @extack: netlink extended ack for reporting errors 1231 * @match: original match used along with the mangle action 1232 * 1233 * Identify the fields written by a FLOW_ACTION_MANGLE, and record 1234 * the partial mangle state in @mung. If this mangle completes an 1235 * earlier partial mangle, consume and apply to @act by calling 1236 * efx_tc_complete_mac_mangle(). 1237 */ 1238 1239 static int efx_tc_mangle(struct efx_nic *efx, struct efx_tc_action_set *act, 1240 const struct flow_action_entry *fa, 1241 struct efx_tc_mangler_state *mung, 1242 struct netlink_ext_ack *extack, 1243 struct efx_tc_match *match) 1244 { 1245 __le32 mac32; 1246 __le16 mac16; 1247 u8 tr_ttl; 1248 1249 switch (fa->mangle.htype) { 1250 case FLOW_ACT_MANGLE_HDR_TYPE_ETH: 1251 BUILD_BUG_ON(offsetof(struct ethhdr, h_dest) != 0); 1252 BUILD_BUG_ON(offsetof(struct ethhdr, h_source) != 6); 1253 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_PEDIT_MAC_ADDRS)) { 1254 NL_SET_ERR_MSG_MOD(extack, 1255 "Pedit mangle mac action violates action order"); 1256 return -EOPNOTSUPP; 1257 } 1258 switch (fa->mangle.offset) { 1259 case 0: 1260 if (fa->mangle.mask) { 1261 NL_SET_ERR_MSG_FMT_MOD(extack, 1262 "mask (%#x) of eth.dst32 mangle is not supported", 1263 fa->mangle.mask); 1264 return -EOPNOTSUPP; 1265 } 1266 /* Ethernet address is little-endian */ 1267 mac32 = cpu_to_le32(fa->mangle.val); 1268 memcpy(mung->dst_mac, &mac32, sizeof(mac32)); 1269 mung->dst_mac_32 = 1; 1270 return efx_tc_complete_mac_mangle(efx, act, mung, extack); 1271 case 4: 1272 if (fa->mangle.mask == 0xffff) { 1273 mac16 = cpu_to_le16(fa->mangle.val >> 16); 1274 memcpy(mung->src_mac, &mac16, sizeof(mac16)); 1275 mung->src_mac_16 = 1; 1276 } else if (fa->mangle.mask == 0xffff0000) { 1277 mac16 = cpu_to_le16((u16)fa->mangle.val); 1278 memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16)); 1279 mung->dst_mac_16 = 1; 1280 } else { 1281 NL_SET_ERR_MSG_FMT_MOD(extack, 1282 "mask (%#x) of eth+4 mangle is not high or low 16b", 1283 fa->mangle.mask); 1284 return -EOPNOTSUPP; 1285 } 1286 return efx_tc_complete_mac_mangle(efx, act, mung, extack); 1287 case 8: 1288 if (fa->mangle.mask) { 1289 NL_SET_ERR_MSG_FMT_MOD(extack, 1290 "mask (%#x) of eth.src32 mangle is not supported", 1291 fa->mangle.mask); 1292 return -EOPNOTSUPP; 1293 } 1294 mac32 = cpu_to_le32(fa->mangle.val); 1295 memcpy(mung->src_mac + 2, &mac32, sizeof(mac32)); 1296 mung->src_mac_32 = 1; 1297 return efx_tc_complete_mac_mangle(efx, act, mung, extack); 1298 default: 1299 NL_SET_ERR_MSG_FMT_MOD(extack, "mangle eth+%u %x/%x is not supported", 1300 fa->mangle.offset, fa->mangle.val, fa->mangle.mask); 1301 return -EOPNOTSUPP; 1302 } 1303 break; 1304 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 1305 switch (fa->mangle.offset) { 1306 case offsetof(struct iphdr, ttl): 1307 /* we currently only support pedit IP4 when it applies 1308 * to TTL and then only when it can be achieved with a 1309 * decrement ttl action 1310 */ 1311 1312 /* check that pedit applies to ttl only */ 1313 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) { 1314 NL_SET_ERR_MSG_FMT_MOD(extack, 1315 "mask (%#x) out of range, only support mangle action on ipv4.ttl", 1316 fa->mangle.mask); 1317 return -EOPNOTSUPP; 1318 } 1319 1320 /* we can only convert to a dec ttl when we have an 1321 * exact match on the ttl field 1322 */ 1323 if (match->mask.ip_ttl != U8_MAX) { 1324 NL_SET_ERR_MSG_FMT_MOD(extack, 1325 "only support mangle ttl when we have an exact match, current mask (%#x)", 1326 match->mask.ip_ttl); 1327 return -EOPNOTSUPP; 1328 } 1329 1330 /* check that we don't try to decrement 0, which equates 1331 * to setting the ttl to 0xff 1332 */ 1333 if (match->value.ip_ttl == 0) { 1334 NL_SET_ERR_MSG_MOD(extack, 1335 "decrement ttl past 0 is not supported"); 1336 return -EOPNOTSUPP; 1337 } 1338 1339 /* check that we do not decrement ttl twice */ 1340 if (!efx_tc_flower_action_order_ok(act, 1341 EFX_TC_AO_DEC_TTL)) { 1342 NL_SET_ERR_MSG_MOD(extack, 1343 "multiple dec ttl is not supported"); 1344 return -EOPNOTSUPP; 1345 } 1346 1347 /* check pedit can be achieved with decrement action */ 1348 tr_ttl = match->value.ip_ttl - 1; 1349 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) { 1350 act->do_ttl_dec = 1; 1351 return 0; 1352 } 1353 1354 fallthrough; 1355 default: 1356 NL_SET_ERR_MSG_FMT_MOD(extack, 1357 "only support mangle on the ttl field (offset is %u)", 1358 fa->mangle.offset); 1359 return -EOPNOTSUPP; 1360 } 1361 break; 1362 case FLOW_ACT_MANGLE_HDR_TYPE_IP6: 1363 switch (fa->mangle.offset) { 1364 case round_down(offsetof(struct ipv6hdr, hop_limit), 4): 1365 /* we currently only support pedit IP6 when it applies 1366 * to the hoplimit and then only when it can be achieved 1367 * with a decrement hoplimit action 1368 */ 1369 1370 /* check that pedit applies to ttl only */ 1371 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) { 1372 NL_SET_ERR_MSG_FMT_MOD(extack, 1373 "mask (%#x) out of range, only support mangle action on ipv6.hop_limit", 1374 fa->mangle.mask); 1375 1376 return -EOPNOTSUPP; 1377 } 1378 1379 /* we can only convert to a dec ttl when we have an 1380 * exact match on the ttl field 1381 */ 1382 if (match->mask.ip_ttl != U8_MAX) { 1383 NL_SET_ERR_MSG_FMT_MOD(extack, 1384 "only support hop_limit when we have an exact match, current mask (%#x)", 1385 match->mask.ip_ttl); 1386 return -EOPNOTSUPP; 1387 } 1388 1389 /* check that we don't try to decrement 0, which equates 1390 * to setting the ttl to 0xff 1391 */ 1392 if (match->value.ip_ttl == 0) { 1393 NL_SET_ERR_MSG_MOD(extack, 1394 "decrementing hop_limit past 0 is not supported"); 1395 return -EOPNOTSUPP; 1396 } 1397 1398 /* check that we do not decrement hoplimit twice */ 1399 if (!efx_tc_flower_action_order_ok(act, 1400 EFX_TC_AO_DEC_TTL)) { 1401 NL_SET_ERR_MSG_MOD(extack, 1402 "multiple dec ttl is not supported"); 1403 return -EOPNOTSUPP; 1404 } 1405 1406 /* check pedit can be achieved with decrement action */ 1407 tr_ttl = match->value.ip_ttl - 1; 1408 if ((fa->mangle.val >> 24) == tr_ttl) { 1409 act->do_ttl_dec = 1; 1410 return 0; 1411 } 1412 1413 fallthrough; 1414 default: 1415 NL_SET_ERR_MSG_FMT_MOD(extack, 1416 "only support mangle on the hop_limit field"); 1417 return -EOPNOTSUPP; 1418 } 1419 default: 1420 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule", 1421 fa->mangle.htype); 1422 return -EOPNOTSUPP; 1423 } 1424 return 0; 1425 } 1426 1427 /** 1428 * efx_tc_incomplete_mangle() - check for leftover partial pedits 1429 * @mung: accumulator for partial mangles 1430 * @extack: netlink extended ack for reporting errors 1431 * 1432 * Since the MAE can only overwrite whole fields, any partial 1433 * field mangle left over on reaching packet delivery (mirred or 1434 * end of TC actions) cannot be offloaded. Check for any such 1435 * and reject them with -%EOPNOTSUPP. 1436 */ 1437 1438 static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung, 1439 struct netlink_ext_ack *extack) 1440 { 1441 if (mung->dst_mac_32 || mung->dst_mac_16) { 1442 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of destination MAC address"); 1443 return -EOPNOTSUPP; 1444 } 1445 if (mung->src_mac_16 || mung->src_mac_32) { 1446 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of source MAC address"); 1447 return -EOPNOTSUPP; 1448 } 1449 return 0; 1450 } 1451 1452 static int efx_tc_flower_replace_foreign_lhs_ar(struct efx_nic *efx, 1453 struct flow_cls_offload *tc, 1454 struct flow_rule *fr, 1455 struct efx_tc_match *match, 1456 struct net_device *net_dev) 1457 { 1458 struct netlink_ext_ack *extack = tc->common.extack; 1459 struct efx_tc_lhs_rule *rule, *old; 1460 enum efx_encap_type type; 1461 int rc; 1462 1463 type = efx_tc_indr_netdev_type(net_dev); 1464 if (type == EFX_ENCAP_TYPE_NONE) { 1465 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device"); 1466 return -EOPNOTSUPP; 1467 } 1468 1469 rc = efx_mae_check_encap_type_supported(efx, type); 1470 if (rc) { 1471 NL_SET_ERR_MSG_FMT_MOD(extack, 1472 "Firmware reports no support for %s encap match", 1473 efx_tc_encap_type_name(type)); 1474 return rc; 1475 } 1476 /* This is an Action Rule, so it needs a separate Encap Match in the 1477 * Outer Rule table. Insert that now. 1478 */ 1479 rc = efx_tc_flower_record_encap_match(efx, match, type, 1480 EFX_TC_EM_DIRECT, 0, 0, extack); 1481 if (rc) 1482 return rc; 1483 1484 match->mask.recirc_id = 0xff; 1485 if (match->mask.ct_state_trk && match->value.ct_state_trk) { 1486 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk"); 1487 rc = -EOPNOTSUPP; 1488 goto release_encap_match; 1489 } 1490 /* LHS rules are always -trk, so we don't need to match on that */ 1491 match->mask.ct_state_trk = 0; 1492 match->value.ct_state_trk = 0; 1493 /* We must inhibit match on TCP SYN/FIN/RST, so that SW can see 1494 * the packet and update the conntrack table. 1495 * Outer Rules will do that with CT_TCP_FLAGS_INHIBIT, but Action 1496 * Rules don't have that; instead they support matching on 1497 * TCP_SYN_FIN_RST (aka TCP_INTERESTING_FLAGS), so use that. 1498 * This is only strictly needed if there will be a DO_CT action, 1499 * which we don't know yet, but typically there will be and it's 1500 * simpler not to bother checking here. 1501 */ 1502 match->mask.tcp_syn_fin_rst = true; 1503 1504 rc = efx_mae_match_check_caps(efx, &match->mask, extack); 1505 if (rc) 1506 goto release_encap_match; 1507 1508 rule = kzalloc_obj(*rule, GFP_USER); 1509 if (!rule) { 1510 rc = -ENOMEM; 1511 goto release_encap_match; 1512 } 1513 rule->cookie = tc->cookie; 1514 rule->is_ar = true; 1515 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, 1516 &rule->linkage, 1517 efx_tc_lhs_rule_ht_params); 1518 if (old) { 1519 netif_dbg(efx, drv, efx->net_dev, 1520 "Already offloaded rule (cookie %lx)\n", tc->cookie); 1521 rc = -EEXIST; 1522 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded"); 1523 goto release; 1524 } 1525 1526 /* Parse actions */ 1527 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule); 1528 if (rc) 1529 goto release; 1530 1531 rule->match = *match; 1532 rule->lhs_act.tun_type = type; 1533 1534 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC); 1535 if (rc) { 1536 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); 1537 goto release; 1538 } 1539 netif_dbg(efx, drv, efx->net_dev, 1540 "Successfully parsed lhs rule (cookie %lx)\n", 1541 tc->cookie); 1542 return 0; 1543 1544 release: 1545 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); 1546 if (!old) 1547 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, 1548 efx_tc_lhs_rule_ht_params); 1549 kfree(rule); 1550 release_encap_match: 1551 if (match->encap) 1552 efx_tc_flower_release_encap_match(efx, match->encap); 1553 return rc; 1554 } 1555 1556 static int efx_tc_flower_replace_foreign_lhs(struct efx_nic *efx, 1557 struct flow_cls_offload *tc, 1558 struct flow_rule *fr, 1559 struct efx_tc_match *match, 1560 struct net_device *net_dev) 1561 { 1562 struct netlink_ext_ack *extack = tc->common.extack; 1563 struct efx_tc_lhs_rule *rule, *old; 1564 enum efx_encap_type type; 1565 int rc; 1566 1567 if (tc->common.chain_index) { 1568 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0"); 1569 return -EOPNOTSUPP; 1570 } 1571 1572 if (!efx_tc_match_is_encap(&match->mask)) { 1573 /* This is not a tunnel decap rule, ignore it */ 1574 netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n"); 1575 return -EOPNOTSUPP; 1576 } 1577 1578 if (efx_tc_flower_flhs_needs_ar(match)) 1579 return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match, 1580 net_dev); 1581 1582 type = efx_tc_indr_netdev_type(net_dev); 1583 if (type == EFX_ENCAP_TYPE_NONE) { 1584 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device"); 1585 return -EOPNOTSUPP; 1586 } 1587 1588 rc = efx_mae_check_encap_type_supported(efx, type); 1589 if (rc) { 1590 NL_SET_ERR_MSG_FMT_MOD(extack, 1591 "Firmware reports no support for %s encap match", 1592 efx_tc_encap_type_name(type)); 1593 return rc; 1594 } 1595 /* Reserve the outer tuple with a pseudo Encap Match */ 1596 rc = efx_tc_flower_record_encap_match(efx, match, type, 1597 EFX_TC_EM_PSEUDO_OR, 0, 0, 1598 extack); 1599 if (rc) 1600 return rc; 1601 1602 if (match->mask.ct_state_trk && match->value.ct_state_trk) { 1603 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk"); 1604 rc = -EOPNOTSUPP; 1605 goto release_encap_match; 1606 } 1607 /* LHS rules are always -trk, so we don't need to match on that */ 1608 match->mask.ct_state_trk = 0; 1609 match->value.ct_state_trk = 0; 1610 1611 rc = efx_tc_flower_translate_flhs_match(match); 1612 if (rc) { 1613 NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields"); 1614 goto release_encap_match; 1615 } 1616 1617 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack); 1618 if (rc) 1619 goto release_encap_match; 1620 1621 rule = kzalloc_obj(*rule, GFP_USER); 1622 if (!rule) { 1623 rc = -ENOMEM; 1624 goto release_encap_match; 1625 } 1626 rule->cookie = tc->cookie; 1627 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, 1628 &rule->linkage, 1629 efx_tc_lhs_rule_ht_params); 1630 if (old) { 1631 netif_dbg(efx, drv, efx->net_dev, 1632 "Already offloaded rule (cookie %lx)\n", tc->cookie); 1633 rc = -EEXIST; 1634 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded"); 1635 goto release; 1636 } 1637 1638 /* Parse actions */ 1639 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule); 1640 if (rc) 1641 goto release; 1642 1643 rule->match = *match; 1644 rule->lhs_act.tun_type = type; 1645 1646 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC); 1647 if (rc) { 1648 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); 1649 goto release; 1650 } 1651 netif_dbg(efx, drv, efx->net_dev, 1652 "Successfully parsed lhs rule (cookie %lx)\n", 1653 tc->cookie); 1654 return 0; 1655 1656 release: 1657 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); 1658 if (!old) 1659 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, 1660 efx_tc_lhs_rule_ht_params); 1661 kfree(rule); 1662 release_encap_match: 1663 if (match->encap) 1664 efx_tc_flower_release_encap_match(efx, match->encap); 1665 return rc; 1666 } 1667 1668 static int efx_tc_flower_replace_foreign(struct efx_nic *efx, 1669 struct net_device *net_dev, 1670 struct flow_cls_offload *tc) 1671 { 1672 struct flow_rule *fr = flow_cls_offload_flow_rule(tc); 1673 struct netlink_ext_ack *extack = tc->common.extack; 1674 struct efx_tc_flow_rule *rule = NULL, *old = NULL; 1675 struct efx_tc_action_set *act = NULL; 1676 bool found = false, uplinked = false; 1677 const struct flow_action_entry *fa; 1678 struct efx_tc_match match; 1679 struct efx_rep *to_efv; 1680 s64 rc; 1681 int i; 1682 1683 /* Parse match */ 1684 memset(&match, 0, sizeof(match)); 1685 rc = efx_tc_flower_parse_match(efx, fr, &match, extack); 1686 if (rc) 1687 return rc; 1688 /* The rule as given to us doesn't specify a source netdevice. 1689 * But, determining whether packets from a VF should match it is 1690 * complicated, so leave those to the software slowpath: qualify 1691 * the filter with source m-port == wire. 1692 */ 1693 rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF); 1694 if (rc < 0) { 1695 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter"); 1696 return rc; 1697 } 1698 match.value.ingress_port = rc; 1699 match.mask.ingress_port = ~0; 1700 1701 if (efx_tc_rule_is_lhs_rule(fr, &match)) 1702 return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match, 1703 net_dev); 1704 1705 if (tc->common.chain_index) { 1706 struct efx_tc_recirc_id *rid; 1707 1708 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev); 1709 if (IS_ERR(rid)) { 1710 NL_SET_ERR_MSG_FMT_MOD(extack, 1711 "Failed to allocate a hardware recirculation ID for chain_index %u", 1712 tc->common.chain_index); 1713 return PTR_ERR(rid); 1714 } 1715 match.rid = rid; 1716 match.value.recirc_id = rid->fw_id; 1717 } 1718 match.mask.recirc_id = 0xff; 1719 1720 /* AR table can't match on DO_CT (+trk). But a commonly used pattern is 1721 * +trk+est, which is strictly implied by +est, so rewrite it to that. 1722 */ 1723 if (match.mask.ct_state_trk && match.value.ct_state_trk && 1724 match.mask.ct_state_est && match.value.ct_state_est) 1725 match.mask.ct_state_trk = 0; 1726 /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could 1727 * match +trk-est (CT_HIT=0) despite being on an established connection. 1728 * So make -est imply -tcp_syn_fin_rst match to ensure these packets 1729 * still hit the software path. 1730 */ 1731 if (match.mask.ct_state_est && !match.value.ct_state_est) { 1732 if (match.value.tcp_syn_fin_rst) { 1733 /* Can't offload this combination */ 1734 NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload"); 1735 rc = -EOPNOTSUPP; 1736 goto release; 1737 } 1738 match.mask.tcp_syn_fin_rst = true; 1739 } 1740 1741 flow_action_for_each(i, fa, &fr->action) { 1742 switch (fa->id) { 1743 case FLOW_ACTION_REDIRECT: 1744 case FLOW_ACTION_MIRRED: /* mirred means mirror here */ 1745 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); 1746 if (IS_ERR(to_efv)) 1747 continue; 1748 found = true; 1749 break; 1750 default: 1751 break; 1752 } 1753 } 1754 if (!found) { /* We don't care. */ 1755 netif_dbg(efx, drv, efx->net_dev, 1756 "Ignoring foreign filter that doesn't egdev us\n"); 1757 rc = -EOPNOTSUPP; 1758 goto release; 1759 } 1760 1761 rc = efx_mae_match_check_caps(efx, &match.mask, extack); 1762 if (rc) 1763 goto release; 1764 1765 if (efx_tc_match_is_encap(&match.mask)) { 1766 enum efx_encap_type type; 1767 1768 type = efx_tc_indr_netdev_type(net_dev); 1769 if (type == EFX_ENCAP_TYPE_NONE) { 1770 NL_SET_ERR_MSG_MOD(extack, 1771 "Egress encap match on unsupported tunnel device"); 1772 rc = -EOPNOTSUPP; 1773 goto release; 1774 } 1775 1776 rc = efx_mae_check_encap_type_supported(efx, type); 1777 if (rc) { 1778 NL_SET_ERR_MSG_FMT_MOD(extack, 1779 "Firmware reports no support for %s encap match", 1780 efx_tc_encap_type_name(type)); 1781 goto release; 1782 } 1783 1784 rc = efx_tc_flower_record_encap_match(efx, &match, type, 1785 EFX_TC_EM_DIRECT, 0, 0, 1786 extack); 1787 if (rc) 1788 goto release; 1789 } else if (!tc->common.chain_index) { 1790 /* This is not a tunnel decap rule, ignore it */ 1791 netif_dbg(efx, drv, efx->net_dev, 1792 "Ignoring foreign filter without encap match\n"); 1793 rc = -EOPNOTSUPP; 1794 goto release; 1795 } 1796 1797 rule = kzalloc_obj(*rule, GFP_USER); 1798 if (!rule) { 1799 rc = -ENOMEM; 1800 goto release; 1801 } 1802 INIT_LIST_HEAD(&rule->acts.list); 1803 rule->cookie = tc->cookie; 1804 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, 1805 &rule->linkage, 1806 efx_tc_match_action_ht_params); 1807 if (IS_ERR(old)) { 1808 rc = PTR_ERR(old); 1809 goto release; 1810 } else if (old) { 1811 netif_dbg(efx, drv, efx->net_dev, 1812 "Ignoring already-offloaded rule (cookie %lx)\n", 1813 tc->cookie); 1814 rc = -EEXIST; 1815 goto release; 1816 } 1817 1818 act = kzalloc_obj(*act, GFP_USER); 1819 if (!act) { 1820 rc = -ENOMEM; 1821 goto release; 1822 } 1823 1824 /* Parse actions. For foreign rules we only support decap & redirect. 1825 * See corresponding code in efx_tc_flower_replace() for theory of 1826 * operation & how 'act' cursor is used. 1827 */ 1828 flow_action_for_each(i, fa, &fr->action) { 1829 struct efx_tc_action_set save; 1830 1831 if (!act) { 1832 /* more actions after a non-pipe action */ 1833 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); 1834 rc = -EINVAL; 1835 goto release; 1836 } 1837 1838 switch (fa->id) { 1839 case FLOW_ACTION_REDIRECT: 1840 case FLOW_ACTION_MIRRED: 1841 /* See corresponding code in efx_tc_flower_replace() for 1842 * long explanations of what's going on here. 1843 */ 1844 save = *act; 1845 if (fa->hw_stats) { 1846 struct efx_tc_counter_index *ctr; 1847 1848 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { 1849 NL_SET_ERR_MSG_FMT_MOD(extack, 1850 "hw_stats_type %u not supported (only 'delayed')", 1851 fa->hw_stats); 1852 rc = -EOPNOTSUPP; 1853 goto release; 1854 } 1855 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) { 1856 NL_SET_ERR_MSG_MOD(extack, "Count action violates action order (can't happen)"); 1857 rc = -EOPNOTSUPP; 1858 goto release; 1859 } 1860 1861 ctr = efx_tc_flower_get_counter_index(efx, 1862 tc->cookie, 1863 EFX_TC_COUNTER_TYPE_AR); 1864 if (IS_ERR(ctr)) { 1865 rc = PTR_ERR(ctr); 1866 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter"); 1867 goto release; 1868 } 1869 act->count = ctr; 1870 INIT_LIST_HEAD(&act->count_user); 1871 } 1872 1873 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) { 1874 /* can't happen */ 1875 rc = -EOPNOTSUPP; 1876 NL_SET_ERR_MSG_MOD(extack, 1877 "Deliver action violates action order (can't happen)"); 1878 goto release; 1879 } 1880 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); 1881 /* PF implies egdev is us, in which case we really 1882 * want to deliver to the uplink (because this is an 1883 * ingress filter). If we don't recognise the egdev 1884 * at all, then we'd better trap so SW can handle it. 1885 */ 1886 if (IS_ERR(to_efv)) 1887 to_efv = EFX_EFV_PF; 1888 if (to_efv == EFX_EFV_PF) { 1889 if (uplinked) 1890 break; 1891 uplinked = true; 1892 } 1893 rc = efx_tc_flower_internal_mport(efx, to_efv); 1894 if (rc < 0) { 1895 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); 1896 goto release; 1897 } 1898 act->dest_mport = rc; 1899 act->deliver = 1; 1900 rc = efx_mae_alloc_action_set(efx, act); 1901 if (rc) { 1902 NL_SET_ERR_MSG_MOD(extack, 1903 "Failed to write action set to hw (mirred)"); 1904 goto release; 1905 } 1906 list_add_tail(&act->list, &rule->acts.list); 1907 act = NULL; 1908 if (fa->id == FLOW_ACTION_REDIRECT) 1909 break; /* end of the line */ 1910 /* Mirror, so continue on with saved act */ 1911 act = kzalloc_obj(*act, GFP_USER); 1912 if (!act) { 1913 rc = -ENOMEM; 1914 goto release; 1915 } 1916 *act = save; 1917 break; 1918 case FLOW_ACTION_TUNNEL_DECAP: 1919 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) { 1920 rc = -EINVAL; 1921 NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order"); 1922 goto release; 1923 } 1924 act->decap = 1; 1925 /* If we previously delivered/trapped to uplink, now 1926 * that we've decapped we'll want another copy if we 1927 * try to deliver/trap to uplink again. 1928 */ 1929 uplinked = false; 1930 break; 1931 default: 1932 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u", 1933 fa->id); 1934 rc = -EOPNOTSUPP; 1935 goto release; 1936 } 1937 } 1938 1939 if (act) { 1940 if (!uplinked) { 1941 /* Not shot/redirected, so deliver to default dest (which is 1942 * the uplink, as this is an ingress filter) 1943 */ 1944 efx_mae_mport_uplink(efx, &act->dest_mport); 1945 act->deliver = 1; 1946 } 1947 rc = efx_mae_alloc_action_set(efx, act); 1948 if (rc) { 1949 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)"); 1950 goto release; 1951 } 1952 list_add_tail(&act->list, &rule->acts.list); 1953 act = NULL; /* Prevent double-free in error path */ 1954 } 1955 1956 rule->match = match; 1957 1958 netif_dbg(efx, drv, efx->net_dev, 1959 "Successfully parsed foreign filter (cookie %lx)\n", 1960 tc->cookie); 1961 1962 rc = efx_mae_alloc_action_set_list(efx, &rule->acts); 1963 if (rc) { 1964 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw"); 1965 goto release; 1966 } 1967 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, 1968 rule->acts.fw_id, &rule->fw_id); 1969 if (rc) { 1970 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); 1971 goto release_acts; 1972 } 1973 return 0; 1974 1975 release_acts: 1976 efx_mae_free_action_set_list(efx, &rule->acts); 1977 release: 1978 /* We failed to insert the rule, so free up any entries we created in 1979 * subsidiary tables. 1980 */ 1981 if (match.rid) 1982 efx_tc_put_recirc_id(efx, match.rid); 1983 if (act) 1984 efx_tc_free_action_set(efx, act, false); 1985 if (rule) { 1986 if (!old) 1987 rhashtable_remove_fast(&efx->tc->match_action_ht, 1988 &rule->linkage, 1989 efx_tc_match_action_ht_params); 1990 efx_tc_free_action_set_list(efx, &rule->acts, false); 1991 } 1992 kfree(rule); 1993 if (match.encap) 1994 efx_tc_flower_release_encap_match(efx, match.encap); 1995 return rc; 1996 } 1997 1998 static int efx_tc_flower_replace_lhs(struct efx_nic *efx, 1999 struct flow_cls_offload *tc, 2000 struct flow_rule *fr, 2001 struct efx_tc_match *match, 2002 struct efx_rep *efv, 2003 struct net_device *net_dev) 2004 { 2005 struct netlink_ext_ack *extack = tc->common.extack; 2006 struct efx_tc_lhs_rule *rule, *old; 2007 int rc; 2008 2009 if (tc->common.chain_index) { 2010 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0"); 2011 return -EOPNOTSUPP; 2012 } 2013 2014 if (match->mask.ct_state_trk && match->value.ct_state_trk) { 2015 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk"); 2016 return -EOPNOTSUPP; 2017 } 2018 /* LHS rules are always -trk, so we don't need to match on that */ 2019 match->mask.ct_state_trk = 0; 2020 match->value.ct_state_trk = 0; 2021 2022 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack); 2023 if (rc) 2024 return rc; 2025 2026 rule = kzalloc_obj(*rule, GFP_USER); 2027 if (!rule) 2028 return -ENOMEM; 2029 rule->cookie = tc->cookie; 2030 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, 2031 &rule->linkage, 2032 efx_tc_lhs_rule_ht_params); 2033 if (IS_ERR(old)) { 2034 rc = PTR_ERR(old); 2035 goto release; 2036 } else if (old) { 2037 netif_dbg(efx, drv, efx->net_dev, 2038 "Already offloaded rule (cookie %lx)\n", tc->cookie); 2039 rc = -EEXIST; 2040 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded"); 2041 goto release; 2042 } 2043 2044 /* Parse actions */ 2045 /* See note in efx_tc_flower_replace() regarding passed net_dev 2046 * (used for efx_tc_get_recirc_id()). 2047 */ 2048 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule); 2049 if (rc) 2050 goto release; 2051 2052 rule->match = *match; 2053 2054 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC); 2055 if (rc) { 2056 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); 2057 goto release; 2058 } 2059 netif_dbg(efx, drv, efx->net_dev, 2060 "Successfully parsed lhs rule (cookie %lx)\n", 2061 tc->cookie); 2062 return 0; 2063 2064 release: 2065 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); 2066 if (!old) 2067 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, 2068 efx_tc_lhs_rule_ht_params); 2069 kfree(rule); 2070 return rc; 2071 } 2072 2073 static int efx_tc_flower_replace(struct efx_nic *efx, 2074 struct net_device *net_dev, 2075 struct flow_cls_offload *tc, 2076 struct efx_rep *efv) 2077 { 2078 struct flow_rule *fr = flow_cls_offload_flow_rule(tc); 2079 struct netlink_ext_ack *extack = tc->common.extack; 2080 const struct ip_tunnel_info *encap_info = NULL; 2081 struct efx_tc_flow_rule *rule = NULL, *old; 2082 struct efx_tc_mangler_state mung = {}; 2083 struct efx_tc_action_set *act = NULL; 2084 const struct flow_action_entry *fa; 2085 struct efx_rep *from_efv, *to_efv; 2086 struct efx_tc_match match; 2087 u32 acts_id; 2088 s64 rc; 2089 int i; 2090 2091 if (!tc_can_offload_extack(efx->net_dev, extack)) 2092 return -EOPNOTSUPP; 2093 if (WARN_ON(!efx->tc)) 2094 return -ENETDOWN; 2095 if (WARN_ON(!efx->tc->up)) 2096 return -ENETDOWN; 2097 2098 from_efv = efx_tc_flower_lookup_efv(efx, net_dev); 2099 if (IS_ERR(from_efv)) { 2100 /* Not from our PF or representors, so probably a tunnel dev */ 2101 return efx_tc_flower_replace_foreign(efx, net_dev, tc); 2102 } 2103 2104 if (efv != from_efv) { 2105 /* can't happen */ 2106 NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)", 2107 netdev_name(net_dev), efv ? "non-" : "", 2108 from_efv ? "non-" : ""); 2109 return -EINVAL; 2110 } 2111 2112 /* Parse match */ 2113 memset(&match, 0, sizeof(match)); 2114 rc = efx_tc_flower_external_mport(efx, from_efv); 2115 if (rc < 0) { 2116 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port"); 2117 return rc; 2118 } 2119 match.value.ingress_port = rc; 2120 match.mask.ingress_port = ~0; 2121 rc = efx_tc_flower_parse_match(efx, fr, &match, extack); 2122 if (rc) 2123 return rc; 2124 if (efx_tc_match_is_encap(&match.mask)) { 2125 NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported"); 2126 return -EOPNOTSUPP; 2127 } 2128 2129 if (efx_tc_rule_is_lhs_rule(fr, &match)) 2130 return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv, 2131 net_dev); 2132 2133 /* chain_index 0 is always recirc_id 0 (and does not appear in recirc_ht). 2134 * Conveniently, match.rid == NULL and match.value.recirc_id == 0 owing 2135 * to the initial memset(), so we don't need to do anything in that case. 2136 */ 2137 if (tc->common.chain_index) { 2138 struct efx_tc_recirc_id *rid; 2139 2140 /* Note regarding passed net_dev: 2141 * VFreps and PF can share chain namespace, as they have 2142 * distinct ingress_mports. So we don't need to burn an 2143 * extra recirc_id if both use the same chain_index. 2144 * (Strictly speaking, we could give each VFrep its own 2145 * recirc_id namespace that doesn't take IDs away from the 2146 * PF, but that would require a bunch of additional IDAs - 2147 * one for each representor - and that's not likely to be 2148 * the main cause of recirc_id exhaustion anyway.) 2149 */ 2150 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, 2151 efx->net_dev); 2152 if (IS_ERR(rid)) { 2153 NL_SET_ERR_MSG_FMT_MOD(extack, 2154 "Failed to allocate a hardware recirculation ID for chain_index %u", 2155 tc->common.chain_index); 2156 return PTR_ERR(rid); 2157 } 2158 match.rid = rid; 2159 match.value.recirc_id = rid->fw_id; 2160 } 2161 match.mask.recirc_id = 0xff; 2162 2163 /* AR table can't match on DO_CT (+trk). But a commonly used pattern is 2164 * +trk+est, which is strictly implied by +est, so rewrite it to that. 2165 */ 2166 if (match.mask.ct_state_trk && match.value.ct_state_trk && 2167 match.mask.ct_state_est && match.value.ct_state_est) 2168 match.mask.ct_state_trk = 0; 2169 /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could 2170 * match +trk-est (CT_HIT=0) despite being on an established connection. 2171 * So make -est imply -tcp_syn_fin_rst match to ensure these packets 2172 * still hit the software path. 2173 */ 2174 if (match.mask.ct_state_est && !match.value.ct_state_est) { 2175 if (match.value.tcp_syn_fin_rst) { 2176 /* Can't offload this combination */ 2177 rc = -EOPNOTSUPP; 2178 goto release; 2179 } 2180 match.mask.tcp_syn_fin_rst = true; 2181 } 2182 2183 rc = efx_mae_match_check_caps(efx, &match.mask, extack); 2184 if (rc) 2185 goto release; 2186 2187 rule = kzalloc_obj(*rule, GFP_USER); 2188 if (!rule) { 2189 rc = -ENOMEM; 2190 goto release; 2191 } 2192 INIT_LIST_HEAD(&rule->acts.list); 2193 rule->cookie = tc->cookie; 2194 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, 2195 &rule->linkage, 2196 efx_tc_match_action_ht_params); 2197 if (IS_ERR(old)) { 2198 rc = PTR_ERR(old); 2199 goto release; 2200 } else if (old) { 2201 netif_dbg(efx, drv, efx->net_dev, 2202 "Already offloaded rule (cookie %lx)\n", tc->cookie); 2203 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded"); 2204 rc = -EEXIST; 2205 goto release; 2206 } 2207 2208 /* Parse actions */ 2209 act = kzalloc_obj(*act, GFP_USER); 2210 if (!act) { 2211 rc = -ENOMEM; 2212 goto release; 2213 } 2214 2215 /** 2216 * DOC: TC action translation 2217 * 2218 * Actions in TC are sequential and cumulative, with delivery actions 2219 * potentially anywhere in the order. The EF100 MAE, however, takes 2220 * an 'action set list' consisting of 'action sets', each of which is 2221 * applied to the _original_ packet, and consists of a set of optional 2222 * actions in a fixed order with delivery at the end. 2223 * To translate between these two models, we maintain a 'cursor', @act, 2224 * which describes the cumulative effect of all the packet-mutating 2225 * actions encountered so far; on handling a delivery (mirred or drop) 2226 * action, once the action-set has been inserted into hardware, we 2227 * append @act to the action-set list (@rule->acts); if this is a pipe 2228 * action (mirred mirror) we then allocate a new @act with a copy of 2229 * the cursor state _before_ the delivery action, otherwise we set @act 2230 * to %NULL. 2231 * This ensures that every allocated action-set is either attached to 2232 * @rule->acts or pointed to by @act (and never both), and that only 2233 * those action-sets in @rule->acts exist in hardware. Consequently, 2234 * in the failure path, @act only needs to be freed in memory, whereas 2235 * for @rule->acts we remove each action-set from hardware before 2236 * freeing it (efx_tc_free_action_set_list()), even if the action-set 2237 * list itself is not in hardware. 2238 */ 2239 flow_action_for_each(i, fa, &fr->action) { 2240 struct efx_tc_action_set save; 2241 u16 tci; 2242 2243 if (!act) { 2244 /* more actions after a non-pipe action */ 2245 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); 2246 rc = -EINVAL; 2247 goto release; 2248 } 2249 2250 if ((fa->id == FLOW_ACTION_REDIRECT || 2251 fa->id == FLOW_ACTION_MIRRED || 2252 fa->id == FLOW_ACTION_DROP) && fa->hw_stats) { 2253 struct efx_tc_counter_index *ctr; 2254 2255 /* Currently the only actions that want stats are 2256 * mirred and gact (ok, shot, trap, goto-chain), which 2257 * means we want stats just before delivery. Also, 2258 * note that tunnel_key set shouldn't change the length 2259 * — it's only the subsequent mirred that does that, 2260 * and the stats are taken _before_ the mirred action 2261 * happens. 2262 */ 2263 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) { 2264 /* All supported actions that count either steal 2265 * (gact shot, mirred redirect) or clone act 2266 * (mirred mirror), so we should never get two 2267 * count actions on one action_set. 2268 */ 2269 NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)"); 2270 rc = -EOPNOTSUPP; 2271 goto release; 2272 } 2273 2274 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { 2275 NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')", 2276 fa->hw_stats); 2277 rc = -EOPNOTSUPP; 2278 goto release; 2279 } 2280 2281 ctr = efx_tc_flower_get_counter_index(efx, tc->cookie, 2282 EFX_TC_COUNTER_TYPE_AR); 2283 if (IS_ERR(ctr)) { 2284 rc = PTR_ERR(ctr); 2285 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter"); 2286 goto release; 2287 } 2288 act->count = ctr; 2289 INIT_LIST_HEAD(&act->count_user); 2290 } 2291 2292 switch (fa->id) { 2293 case FLOW_ACTION_DROP: 2294 rc = efx_mae_alloc_action_set(efx, act); 2295 if (rc) { 2296 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)"); 2297 goto release; 2298 } 2299 list_add_tail(&act->list, &rule->acts.list); 2300 act = NULL; /* end of the line */ 2301 break; 2302 case FLOW_ACTION_REDIRECT: 2303 case FLOW_ACTION_MIRRED: 2304 save = *act; 2305 2306 if (encap_info) { 2307 struct efx_tc_encap_action *encap; 2308 2309 if (!efx_tc_flower_action_order_ok(act, 2310 EFX_TC_AO_ENCAP)) { 2311 rc = -EOPNOTSUPP; 2312 NL_SET_ERR_MSG_MOD(extack, "Encap action violates action order"); 2313 goto release; 2314 } 2315 encap = efx_tc_flower_create_encap_md( 2316 efx, encap_info, fa->dev, extack); 2317 if (IS_ERR_OR_NULL(encap)) { 2318 rc = PTR_ERR(encap); 2319 if (!rc) 2320 rc = -EIO; /* arbitrary */ 2321 goto release; 2322 } 2323 act->encap_md = encap; 2324 list_add_tail(&act->encap_user, &encap->users); 2325 act->dest_mport = encap->dest_mport; 2326 act->deliver = 1; 2327 if (act->count && !WARN_ON(!act->count->cnt)) { 2328 /* This counter is used by an encap 2329 * action, which needs a reference back 2330 * so it can prod neighbouring whenever 2331 * traffic is seen. 2332 */ 2333 spin_lock_bh(&act->count->cnt->lock); 2334 list_add_tail(&act->count_user, 2335 &act->count->cnt->users); 2336 spin_unlock_bh(&act->count->cnt->lock); 2337 } 2338 rc = efx_mae_alloc_action_set(efx, act); 2339 if (rc) { 2340 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (encap)"); 2341 goto release; 2342 } 2343 list_add_tail(&act->list, &rule->acts.list); 2344 act->user = &rule->acts; 2345 act = NULL; 2346 if (fa->id == FLOW_ACTION_REDIRECT) 2347 break; /* end of the line */ 2348 /* Mirror, so continue on with saved act */ 2349 save.count = NULL; 2350 act = kzalloc_obj(*act, GFP_USER); 2351 if (!act) { 2352 rc = -ENOMEM; 2353 goto release; 2354 } 2355 *act = save; 2356 break; 2357 } 2358 2359 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) { 2360 /* can't happen */ 2361 rc = -EOPNOTSUPP; 2362 NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)"); 2363 goto release; 2364 } 2365 2366 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); 2367 if (IS_ERR(to_efv)) { 2368 NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch"); 2369 rc = PTR_ERR(to_efv); 2370 goto release; 2371 } 2372 rc = efx_tc_flower_external_mport(efx, to_efv); 2373 if (rc < 0) { 2374 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); 2375 goto release; 2376 } 2377 act->dest_mport = rc; 2378 act->deliver = 1; 2379 rc = efx_mae_alloc_action_set(efx, act); 2380 if (rc) { 2381 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)"); 2382 goto release; 2383 } 2384 list_add_tail(&act->list, &rule->acts.list); 2385 act = NULL; 2386 if (fa->id == FLOW_ACTION_REDIRECT) 2387 break; /* end of the line */ 2388 /* Mirror, so continue on with saved act */ 2389 save.count = NULL; 2390 act = kzalloc_obj(*act, GFP_USER); 2391 if (!act) { 2392 rc = -ENOMEM; 2393 goto release; 2394 } 2395 *act = save; 2396 break; 2397 case FLOW_ACTION_VLAN_POP: 2398 if (act->vlan_push) { 2399 act->vlan_push--; 2400 } else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) { 2401 act->vlan_pop++; 2402 } else { 2403 NL_SET_ERR_MSG_MOD(extack, 2404 "More than two VLAN pops, or action order violated"); 2405 rc = -EINVAL; 2406 goto release; 2407 } 2408 break; 2409 case FLOW_ACTION_VLAN_PUSH: 2410 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) { 2411 rc = -EINVAL; 2412 NL_SET_ERR_MSG_MOD(extack, 2413 "More than two VLAN pushes, or action order violated"); 2414 goto release; 2415 } 2416 tci = fa->vlan.vid & VLAN_VID_MASK; 2417 tci |= fa->vlan.prio << VLAN_PRIO_SHIFT; 2418 act->vlan_tci[act->vlan_push] = cpu_to_be16(tci); 2419 act->vlan_proto[act->vlan_push] = fa->vlan.proto; 2420 act->vlan_push++; 2421 break; 2422 case FLOW_ACTION_ADD: 2423 rc = efx_tc_pedit_add(efx, act, fa, extack); 2424 if (rc < 0) 2425 goto release; 2426 break; 2427 case FLOW_ACTION_MANGLE: 2428 rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match); 2429 if (rc < 0) 2430 goto release; 2431 break; 2432 case FLOW_ACTION_TUNNEL_ENCAP: 2433 if (encap_info) { 2434 /* Can't specify encap multiple times. 2435 * If you want to overwrite an existing 2436 * encap_info, use an intervening 2437 * FLOW_ACTION_TUNNEL_DECAP to clear it. 2438 */ 2439 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set"); 2440 rc = -EINVAL; 2441 goto release; 2442 } 2443 if (!fa->tunnel) { 2444 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key"); 2445 rc = -EOPNOTSUPP; 2446 goto release; 2447 } 2448 encap_info = fa->tunnel; 2449 break; 2450 case FLOW_ACTION_TUNNEL_DECAP: 2451 if (encap_info) { 2452 encap_info = NULL; 2453 break; 2454 } 2455 /* Since we don't support enc_key matches on ingress 2456 * (and if we did there'd be no tunnel-device to give 2457 * us a type), we can't offload a decap that's not 2458 * just undoing a previous encap action. 2459 */ 2460 NL_SET_ERR_MSG_MOD(extack, "Cannot offload tunnel decap action without tunnel device"); 2461 rc = -EOPNOTSUPP; 2462 goto release; 2463 case FLOW_ACTION_CT: 2464 if (fa->ct.action != TCA_CT_ACT_NAT) { 2465 rc = -EOPNOTSUPP; 2466 NL_SET_ERR_MSG_FMT_MOD(extack, "Can only offload CT 'nat' action in RHS rules, not %d", fa->ct.action); 2467 goto release; 2468 } 2469 act->do_nat = 1; 2470 break; 2471 default: 2472 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u", 2473 fa->id); 2474 rc = -EOPNOTSUPP; 2475 goto release; 2476 } 2477 } 2478 2479 rc = efx_tc_incomplete_mangle(&mung, extack); 2480 if (rc < 0) 2481 goto release; 2482 if (act) { 2483 /* Not shot/redirected, so deliver to default dest */ 2484 if (from_efv == EFX_EFV_PF) 2485 /* Rule applies to traffic from the wire, 2486 * and default dest is thus the PF 2487 */ 2488 efx_mae_mport_uplink(efx, &act->dest_mport); 2489 else 2490 /* Representor, so rule applies to traffic from 2491 * representee, and default dest is thus the rep. 2492 * All reps use the same mport for delivery 2493 */ 2494 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, 2495 &act->dest_mport); 2496 act->deliver = 1; 2497 rc = efx_mae_alloc_action_set(efx, act); 2498 if (rc) { 2499 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)"); 2500 goto release; 2501 } 2502 list_add_tail(&act->list, &rule->acts.list); 2503 act = NULL; /* Prevent double-free in error path */ 2504 } 2505 2506 netif_dbg(efx, drv, efx->net_dev, 2507 "Successfully parsed filter (cookie %lx)\n", 2508 tc->cookie); 2509 2510 rule->match = match; 2511 2512 rc = efx_mae_alloc_action_set_list(efx, &rule->acts); 2513 if (rc) { 2514 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw"); 2515 goto release; 2516 } 2517 if (from_efv == EFX_EFV_PF) 2518 /* PF netdev, so rule applies to traffic from wire */ 2519 rule->fallback = &efx->tc->facts.pf; 2520 else 2521 /* repdev, so rule applies to traffic from representee */ 2522 rule->fallback = &efx->tc->facts.reps; 2523 if (!efx_tc_check_ready(efx, rule)) { 2524 netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n"); 2525 acts_id = rule->fallback->fw_id; 2526 } else { 2527 netif_dbg(efx, drv, efx->net_dev, "ready for hw\n"); 2528 acts_id = rule->acts.fw_id; 2529 } 2530 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, 2531 acts_id, &rule->fw_id); 2532 if (rc) { 2533 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); 2534 goto release_acts; 2535 } 2536 return 0; 2537 2538 release_acts: 2539 efx_mae_free_action_set_list(efx, &rule->acts); 2540 release: 2541 /* We failed to insert the rule, so free up any entries we created in 2542 * subsidiary tables. 2543 */ 2544 if (match.rid) 2545 efx_tc_put_recirc_id(efx, match.rid); 2546 if (act) 2547 efx_tc_free_action_set(efx, act, false); 2548 if (rule) { 2549 if (!old) 2550 rhashtable_remove_fast(&efx->tc->match_action_ht, 2551 &rule->linkage, 2552 efx_tc_match_action_ht_params); 2553 efx_tc_free_action_set_list(efx, &rule->acts, false); 2554 } 2555 kfree(rule); 2556 return rc; 2557 } 2558 2559 static int efx_tc_flower_destroy(struct efx_nic *efx, 2560 struct net_device *net_dev, 2561 struct flow_cls_offload *tc) 2562 { 2563 struct netlink_ext_ack *extack = tc->common.extack; 2564 struct efx_tc_lhs_rule *lhs_rule; 2565 struct efx_tc_flow_rule *rule; 2566 2567 lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie, 2568 efx_tc_lhs_rule_ht_params); 2569 if (lhs_rule) { 2570 /* Remove it from HW */ 2571 efx_mae_remove_lhs_rule(efx, lhs_rule); 2572 /* Delete it from SW */ 2573 efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act); 2574 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage, 2575 efx_tc_lhs_rule_ht_params); 2576 if (lhs_rule->match.encap) 2577 efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap); 2578 netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n", 2579 lhs_rule->cookie); 2580 kfree(lhs_rule); 2581 return 0; 2582 } 2583 2584 rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie, 2585 efx_tc_match_action_ht_params); 2586 if (!rule) { 2587 /* Only log a message if we're the ingress device. Otherwise 2588 * it's a foreign filter and we might just not have been 2589 * interested (e.g. we might not have been the egress device 2590 * either). 2591 */ 2592 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev))) 2593 netif_warn(efx, drv, efx->net_dev, 2594 "Filter %lx not found to remove\n", tc->cookie); 2595 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules"); 2596 return -ENOENT; 2597 } 2598 2599 /* Remove it from HW */ 2600 efx_tc_delete_rule(efx, rule); 2601 /* Delete it from SW */ 2602 rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage, 2603 efx_tc_match_action_ht_params); 2604 netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie); 2605 kfree(rule); 2606 return 0; 2607 } 2608 2609 static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev, 2610 struct flow_cls_offload *tc) 2611 { 2612 struct netlink_ext_ack *extack = tc->common.extack; 2613 struct efx_tc_counter_index *ctr; 2614 struct efx_tc_counter *cnt; 2615 u64 packets, bytes; 2616 2617 ctr = efx_tc_flower_find_counter_index(efx, tc->cookie); 2618 if (!ctr) { 2619 /* See comment in efx_tc_flower_destroy() */ 2620 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev))) 2621 if (net_ratelimit()) 2622 netif_warn(efx, drv, efx->net_dev, 2623 "Filter %lx not found for stats\n", 2624 tc->cookie); 2625 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules"); 2626 return -ENOENT; 2627 } 2628 if (WARN_ON(!ctr->cnt)) /* can't happen */ 2629 return -EIO; 2630 cnt = ctr->cnt; 2631 2632 spin_lock_bh(&cnt->lock); 2633 /* Report only new pkts/bytes since last time TC asked */ 2634 packets = cnt->packets; 2635 bytes = cnt->bytes; 2636 flow_stats_update(&tc->stats, bytes - cnt->old_bytes, 2637 packets - cnt->old_packets, 0, cnt->touched, 2638 FLOW_ACTION_HW_STATS_DELAYED); 2639 cnt->old_packets = packets; 2640 cnt->old_bytes = bytes; 2641 spin_unlock_bh(&cnt->lock); 2642 return 0; 2643 } 2644 2645 int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev, 2646 struct flow_cls_offload *tc, struct efx_rep *efv) 2647 { 2648 int rc; 2649 2650 if (!efx->tc) 2651 return -EOPNOTSUPP; 2652 2653 mutex_lock(&efx->tc->mutex); 2654 switch (tc->command) { 2655 case FLOW_CLS_REPLACE: 2656 rc = efx_tc_flower_replace(efx, net_dev, tc, efv); 2657 break; 2658 case FLOW_CLS_DESTROY: 2659 rc = efx_tc_flower_destroy(efx, net_dev, tc); 2660 break; 2661 case FLOW_CLS_STATS: 2662 rc = efx_tc_flower_stats(efx, net_dev, tc); 2663 break; 2664 default: 2665 rc = -EOPNOTSUPP; 2666 break; 2667 } 2668 mutex_unlock(&efx->tc->mutex); 2669 return rc; 2670 } 2671 2672 static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port, 2673 u32 eg_port, struct efx_tc_flow_rule *rule) 2674 { 2675 struct efx_tc_action_set_list *acts = &rule->acts; 2676 struct efx_tc_match *match = &rule->match; 2677 struct efx_tc_action_set *act; 2678 int rc; 2679 2680 match->value.ingress_port = ing_port; 2681 match->mask.ingress_port = ~0; 2682 act = kzalloc_obj(*act); 2683 if (!act) 2684 return -ENOMEM; 2685 act->deliver = 1; 2686 act->dest_mport = eg_port; 2687 rc = efx_mae_alloc_action_set(efx, act); 2688 if (rc) 2689 goto fail1; 2690 EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); 2691 list_add_tail(&act->list, &acts->list); 2692 rc = efx_mae_alloc_action_set_list(efx, acts); 2693 if (rc) 2694 goto fail2; 2695 rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT, 2696 acts->fw_id, &rule->fw_id); 2697 if (rc) 2698 goto fail3; 2699 return 0; 2700 fail3: 2701 efx_mae_free_action_set_list(efx, acts); 2702 fail2: 2703 list_del(&act->list); 2704 efx_mae_free_action_set(efx, act->fw_id); 2705 fail1: 2706 kfree(act); 2707 return rc; 2708 } 2709 2710 static int efx_tc_configure_default_rule_pf(struct efx_nic *efx) 2711 { 2712 struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf; 2713 u32 ing_port, eg_port; 2714 2715 efx_mae_mport_uplink(efx, &ing_port); 2716 efx_mae_mport_wire(efx, &eg_port); 2717 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); 2718 } 2719 2720 static int efx_tc_configure_default_rule_wire(struct efx_nic *efx) 2721 { 2722 struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire; 2723 u32 ing_port, eg_port; 2724 2725 efx_mae_mport_wire(efx, &ing_port); 2726 efx_mae_mport_uplink(efx, &eg_port); 2727 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); 2728 } 2729 2730 int efx_tc_configure_default_rule_rep(struct efx_rep *efv) 2731 { 2732 struct efx_tc_flow_rule *rule = &efv->dflt; 2733 struct efx_nic *efx = efv->parent; 2734 u32 ing_port, eg_port; 2735 2736 efx_mae_mport_mport(efx, efv->mport, &ing_port); 2737 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); 2738 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); 2739 } 2740 2741 void efx_tc_deconfigure_default_rule(struct efx_nic *efx, 2742 struct efx_tc_flow_rule *rule) 2743 { 2744 if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL) 2745 efx_tc_delete_rule(efx, rule); 2746 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; 2747 } 2748 2749 static int efx_tc_configure_fallback_acts(struct efx_nic *efx, u32 eg_port, 2750 struct efx_tc_action_set_list *acts) 2751 { 2752 struct efx_tc_action_set *act; 2753 int rc; 2754 2755 act = kzalloc_obj(*act); 2756 if (!act) 2757 return -ENOMEM; 2758 act->deliver = 1; 2759 act->dest_mport = eg_port; 2760 rc = efx_mae_alloc_action_set(efx, act); 2761 if (rc) 2762 goto fail1; 2763 EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); 2764 list_add_tail(&act->list, &acts->list); 2765 rc = efx_mae_alloc_action_set_list(efx, acts); 2766 if (rc) 2767 goto fail2; 2768 return 0; 2769 fail2: 2770 list_del(&act->list); 2771 efx_mae_free_action_set(efx, act->fw_id); 2772 fail1: 2773 kfree(act); 2774 return rc; 2775 } 2776 2777 static int efx_tc_configure_fallback_acts_pf(struct efx_nic *efx) 2778 { 2779 struct efx_tc_action_set_list *acts = &efx->tc->facts.pf; 2780 u32 eg_port; 2781 2782 efx_mae_mport_uplink(efx, &eg_port); 2783 return efx_tc_configure_fallback_acts(efx, eg_port, acts); 2784 } 2785 2786 static int efx_tc_configure_fallback_acts_reps(struct efx_nic *efx) 2787 { 2788 struct efx_tc_action_set_list *acts = &efx->tc->facts.reps; 2789 u32 eg_port; 2790 2791 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); 2792 return efx_tc_configure_fallback_acts(efx, eg_port, acts); 2793 } 2794 2795 static void efx_tc_deconfigure_fallback_acts(struct efx_nic *efx, 2796 struct efx_tc_action_set_list *acts) 2797 { 2798 efx_tc_free_action_set_list(efx, acts, true); 2799 } 2800 2801 static int efx_tc_configure_rep_mport(struct efx_nic *efx) 2802 { 2803 u32 rep_mport_label; 2804 int rc; 2805 2806 rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label); 2807 if (rc) 2808 return rc; 2809 pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n", 2810 efx->tc->reps_mport_id, rep_mport_label); 2811 /* Use mport *selector* as vport ID */ 2812 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, 2813 &efx->tc->reps_mport_vport_id); 2814 return 0; 2815 } 2816 2817 static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx) 2818 { 2819 efx_mae_free_mport(efx, efx->tc->reps_mport_id); 2820 efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL; 2821 } 2822 2823 int efx_tc_insert_rep_filters(struct efx_nic *efx) 2824 { 2825 struct efx_filter_spec promisc, allmulti; 2826 int rc; 2827 2828 if (efx->type->is_vf) 2829 return 0; 2830 if (!efx->tc) 2831 return 0; 2832 efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0); 2833 efx_filter_set_uc_def(&promisc); 2834 efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id); 2835 rc = efx_filter_insert_filter(efx, &promisc, false); 2836 if (rc < 0) 2837 return rc; 2838 efx->tc->reps_filter_uc = rc; 2839 efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0); 2840 efx_filter_set_mc_def(&allmulti); 2841 efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id); 2842 rc = efx_filter_insert_filter(efx, &allmulti, false); 2843 if (rc < 0) 2844 return rc; 2845 efx->tc->reps_filter_mc = rc; 2846 return 0; 2847 } 2848 2849 void efx_tc_remove_rep_filters(struct efx_nic *efx) 2850 { 2851 if (efx->type->is_vf) 2852 return; 2853 if (!efx->tc) 2854 return; 2855 if (efx->tc->reps_filter_mc >= 0) 2856 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc); 2857 efx->tc->reps_filter_mc = -1; 2858 if (efx->tc->reps_filter_uc >= 0) 2859 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc); 2860 efx->tc->reps_filter_uc = -1; 2861 } 2862 2863 int efx_init_tc(struct efx_nic *efx) 2864 { 2865 int rc; 2866 2867 rc = efx_mae_get_caps(efx, efx->tc->caps); 2868 if (rc) 2869 return rc; 2870 if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS) 2871 /* Firmware supports some match fields the driver doesn't know 2872 * about. Not fatal, unless any of those fields are required 2873 * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know. 2874 */ 2875 netif_warn(efx, probe, efx->net_dev, 2876 "FW reports additional match fields %u\n", 2877 efx->tc->caps->match_field_count); 2878 if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) { 2879 netif_err(efx, probe, efx->net_dev, 2880 "Too few action prios supported (have %u, need %u)\n", 2881 efx->tc->caps->action_prios, EFX_TC_PRIO__NUM); 2882 return -EIO; 2883 } 2884 rc = efx_tc_configure_default_rule_pf(efx); 2885 if (rc) 2886 return rc; 2887 rc = efx_tc_configure_default_rule_wire(efx); 2888 if (rc) 2889 return rc; 2890 rc = efx_tc_configure_rep_mport(efx); 2891 if (rc) 2892 return rc; 2893 rc = efx_tc_configure_fallback_acts_pf(efx); 2894 if (rc) 2895 return rc; 2896 rc = efx_tc_configure_fallback_acts_reps(efx); 2897 if (rc) 2898 return rc; 2899 rc = efx_mae_get_tables(efx); 2900 if (rc) 2901 return rc; 2902 rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx); 2903 if (rc) 2904 goto out_free; 2905 efx->tc->up = true; 2906 return 0; 2907 out_free: 2908 efx_mae_free_tables(efx); 2909 return rc; 2910 } 2911 2912 void efx_fini_tc(struct efx_nic *efx) 2913 { 2914 /* We can get called even if efx_init_struct_tc() failed */ 2915 if (!efx->tc) 2916 return; 2917 if (efx->tc->up) 2918 flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind); 2919 efx_tc_deconfigure_rep_mport(efx); 2920 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf); 2921 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire); 2922 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf); 2923 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps); 2924 efx->tc->up = false; 2925 efx_mae_free_tables(efx); 2926 } 2927 2928 /* At teardown time, all TC filter rules (and thus all resources they created) 2929 * should already have been removed. If we find any in our hashtables, make a 2930 * cursory attempt to clean up the software side. 2931 */ 2932 static void efx_tc_encap_match_free(void *ptr, void *__unused) 2933 { 2934 struct efx_tc_encap_match *encap = ptr; 2935 2936 WARN_ON(refcount_read(&encap->ref)); 2937 kfree(encap); 2938 } 2939 2940 static void efx_tc_recirc_free(void *ptr, void *arg) 2941 { 2942 struct efx_tc_recirc_id *rid = ptr; 2943 struct efx_nic *efx = arg; 2944 2945 WARN_ON(refcount_read(&rid->ref)); 2946 ida_free(&efx->tc->recirc_ida, rid->fw_id); 2947 kfree(rid); 2948 } 2949 2950 static void efx_tc_lhs_free(void *ptr, void *arg) 2951 { 2952 struct efx_tc_lhs_rule *rule = ptr; 2953 struct efx_nic *efx = arg; 2954 2955 netif_err(efx, drv, efx->net_dev, 2956 "tc lhs_rule %lx still present at teardown, removing\n", 2957 rule->cookie); 2958 2959 if (rule->lhs_act.zone) 2960 efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone); 2961 if (rule->lhs_act.count) 2962 efx_tc_flower_put_counter_index(efx, rule->lhs_act.count); 2963 efx_mae_remove_lhs_rule(efx, rule); 2964 2965 kfree(rule); 2966 } 2967 2968 static void efx_tc_mac_free(void *ptr, void *__unused) 2969 { 2970 struct efx_tc_mac_pedit_action *ped = ptr; 2971 2972 WARN_ON(refcount_read(&ped->ref)); 2973 kfree(ped); 2974 } 2975 2976 static void efx_tc_flow_free(void *ptr, void *arg) 2977 { 2978 struct efx_tc_flow_rule *rule = ptr; 2979 struct efx_nic *efx = arg; 2980 2981 netif_err(efx, drv, efx->net_dev, 2982 "tc rule %lx still present at teardown, removing\n", 2983 rule->cookie); 2984 2985 /* Also releases entries in subsidiary tables */ 2986 efx_tc_delete_rule(efx, rule); 2987 2988 kfree(rule); 2989 } 2990 2991 int efx_init_struct_tc(struct efx_nic *efx) 2992 { 2993 int rc; 2994 2995 if (efx->type->is_vf) 2996 return 0; 2997 2998 efx->tc = kzalloc_obj(*efx->tc); 2999 if (!efx->tc) 3000 return -ENOMEM; 3001 efx->tc->caps = kzalloc_obj(struct mae_caps); 3002 if (!efx->tc->caps) { 3003 rc = -ENOMEM; 3004 goto fail_alloc_caps; 3005 } 3006 INIT_LIST_HEAD(&efx->tc->block_list); 3007 3008 mutex_init(&efx->tc->mutex); 3009 init_waitqueue_head(&efx->tc->flush_wq); 3010 rc = efx_tc_init_encap_actions(efx); 3011 if (rc < 0) 3012 goto fail_encap_actions; 3013 rc = efx_tc_init_counters(efx); 3014 if (rc < 0) 3015 goto fail_counters; 3016 rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params); 3017 if (rc < 0) 3018 goto fail_mac_ht; 3019 rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params); 3020 if (rc < 0) 3021 goto fail_encap_match_ht; 3022 rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params); 3023 if (rc < 0) 3024 goto fail_match_action_ht; 3025 rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params); 3026 if (rc < 0) 3027 goto fail_lhs_rule_ht; 3028 rc = efx_tc_init_conntrack(efx); 3029 if (rc < 0) 3030 goto fail_conntrack; 3031 rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params); 3032 if (rc < 0) 3033 goto fail_recirc_ht; 3034 ida_init(&efx->tc->recirc_ida); 3035 efx->tc->reps_filter_uc = -1; 3036 efx->tc->reps_filter_mc = -1; 3037 INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list); 3038 efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; 3039 INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list); 3040 efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; 3041 INIT_LIST_HEAD(&efx->tc->facts.pf.list); 3042 efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; 3043 INIT_LIST_HEAD(&efx->tc->facts.reps.list); 3044 efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; 3045 efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type; 3046 return 0; 3047 fail_recirc_ht: 3048 efx_tc_destroy_conntrack(efx); 3049 fail_conntrack: 3050 rhashtable_destroy(&efx->tc->lhs_rule_ht); 3051 fail_lhs_rule_ht: 3052 rhashtable_destroy(&efx->tc->match_action_ht); 3053 fail_match_action_ht: 3054 rhashtable_destroy(&efx->tc->encap_match_ht); 3055 fail_encap_match_ht: 3056 rhashtable_destroy(&efx->tc->mac_ht); 3057 fail_mac_ht: 3058 efx_tc_destroy_counters(efx); 3059 fail_counters: 3060 efx_tc_destroy_encap_actions(efx); 3061 fail_encap_actions: 3062 mutex_destroy(&efx->tc->mutex); 3063 kfree(efx->tc->caps); 3064 fail_alloc_caps: 3065 kfree(efx->tc); 3066 efx->tc = NULL; 3067 return rc; 3068 } 3069 3070 void efx_fini_struct_tc(struct efx_nic *efx) 3071 { 3072 if (!efx->tc) 3073 return; 3074 3075 mutex_lock(&efx->tc->mutex); 3076 EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id != 3077 MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); 3078 EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id != 3079 MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); 3080 EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id != 3081 MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL); 3082 EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id != 3083 MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL); 3084 rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx); 3085 rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free, 3086 efx); 3087 rhashtable_free_and_destroy(&efx->tc->encap_match_ht, 3088 efx_tc_encap_match_free, NULL); 3089 efx_tc_fini_conntrack(efx); 3090 rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx); 3091 WARN_ON(!ida_is_empty(&efx->tc->recirc_ida)); 3092 ida_destroy(&efx->tc->recirc_ida); 3093 rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL); 3094 efx_tc_fini_counters(efx); 3095 efx_tc_fini_encap_actions(efx); 3096 mutex_unlock(&efx->tc->mutex); 3097 mutex_destroy(&efx->tc->mutex); 3098 kfree(efx->tc->caps); 3099 kfree(efx->tc); 3100 efx->tc = NULL; 3101 } 3102