1 /*- 2 * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "opt_rss.h" 27 #include "opt_ratelimit.h" 28 29 #include <dev/mlx5/mlx5_en/en.h> 30 31 #include <linux/list.h> 32 #include <dev/mlx5/fs.h> 33 #include <dev/mlx5/mpfs.h> 34 #include <dev/mlx5/mlx5_core/fs_tcp.h> 35 36 /* 37 * The flow tables with rules define the packet processing on receive. 38 * Currently the following structure is set up to handle different 39 * offloads like TLS RX offload, VLAN decapsulation, packet 40 * classification, RSS hashing, VxLAN checksum offloading: 41 * 42 * +=========+ +=========+ +=================+ 43 * |TCP/IPv4 | |TCP/IPv4 | |TCP/IPv4 Match | 44 * |Flowtable|------>| |----->|Outer Proto Match|=====> TLS TIR n 45 * | | |Catch-all|\ | | 46 * +=========+ +=========+| +=================+ 47 * | 48 * +------------------------+ 49 * V 50 * +=========+ +=========+ +=================+ 51 * |TCP/IPv6 | |TCP/IPv6 | |TCP/IPv6 Match | 52 * |Flowtable|------>| |----->|Outer Proto Match|=====> TLS TIR n 53 * | | |Catch-all|\ | | 54 * +=========+ +=========+| +=================+ 55 * | 56 * +------------------------+ 57 * V 58 * +=========+ +=========+ +=================+ 59 * |VLAN ft: | |VxLAN | |VxLAN Main | 60 * |CTAG/STAG|------>| VNI|----->|Inner Proto Match|=====> Inner TIR n 61 * |VID/noVID|/ |Catch-all|\ | | 62 * +=========+ +=========+| +=================+ 63 * | 64 * | 65 * | 66 * v 67 * +=================+ 68 * |Main | 69 * |Outer Proto Match|=====> TIR n 70 * | | 71 * +=================+ 72 * 73 * The path through flow rules directs each packet into an appropriate TIR, 74 * according to the: 75 * - VLAN encapsulation 76 * - Outer protocol 77 * - Presence of inner protocol 78 */ 79 80 #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v) 81 82 enum { 83 MLX5E_FULLMATCH = 0, 84 MLX5E_ALLMULTI = 1, 85 MLX5E_PROMISC = 2, 86 }; 87 88 enum { 89 MLX5E_UC = 0, 90 MLX5E_MC_IPV4 = 1, 91 MLX5E_MC_IPV6 = 2, 92 MLX5E_MC_OTHER = 3, 93 }; 94 95 enum { 96 MLX5E_ACTION_NONE = 0, 97 MLX5E_ACTION_ADD = 1, 98 MLX5E_ACTION_DEL = 2, 99 }; 100 101 struct mlx5e_eth_addr_hash_node { 102 LIST_ENTRY(mlx5e_eth_addr_hash_node) hlist; 103 u8 action; 104 u32 mpfs_index; 105 struct mlx5e_eth_addr_info ai; 106 }; 107 108 static void mlx5e_del_all_vlan_rules(struct mlx5e_priv *); 109 110 static inline int 111 mlx5e_hash_eth_addr(const u8 * addr) 112 { 113 return (addr[5]); 114 } 115 116 static bool 117 mlx5e_add_eth_addr_to_hash(struct mlx5e_eth_addr_hash_head *hash, 118 struct mlx5e_eth_addr_hash_node *hn_new) 119 { 120 struct mlx5e_eth_addr_hash_node *hn; 121 u32 ix = mlx5e_hash_eth_addr(hn_new->ai.addr); 122 123 LIST_FOREACH(hn, &hash[ix], hlist) { 124 if (bcmp(hn->ai.addr, hn_new->ai.addr, ETHER_ADDR_LEN) == 0) { 125 if (hn->action == MLX5E_ACTION_DEL) 126 hn->action = MLX5E_ACTION_NONE; 127 free(hn_new, M_MLX5EN); 128 return (false); 129 } 130 } 131 LIST_INSERT_HEAD(&hash[ix], hn_new, hlist); 132 return (true); 133 } 134 135 static void 136 mlx5e_del_eth_addr_from_hash(struct mlx5e_eth_addr_hash_node *hn) 137 { 138 LIST_REMOVE(hn, hlist); 139 free(hn, M_MLX5EN); 140 } 141 142 static void 143 mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv, 144 struct mlx5e_eth_addr_info *ai) 145 { 146 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]); 147 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]); 148 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]); 149 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]); 150 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_TCP]); 151 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_TCP]); 152 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_UDP]); 153 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_UDP]); 154 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6]); 155 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4]); 156 mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_ANY]); 157 } 158 159 static int 160 mlx5e_get_eth_addr_type(const u8 * addr) 161 { 162 if (ETHER_IS_MULTICAST(addr) == 0) 163 return (MLX5E_UC); 164 165 if ((addr[0] == 0x01) && 166 (addr[1] == 0x00) && 167 (addr[2] == 0x5e) && 168 !(addr[3] & 0x80)) 169 return (MLX5E_MC_IPV4); 170 171 if ((addr[0] == 0x33) && 172 (addr[1] == 0x33)) 173 return (MLX5E_MC_IPV6); 174 175 return (MLX5E_MC_OTHER); 176 } 177 178 static u32 179 mlx5e_get_tt_vec(struct mlx5e_eth_addr_info *ai, int type) 180 { 181 int eth_addr_type; 182 u32 ret; 183 184 switch (type) { 185 case MLX5E_FULLMATCH: 186 eth_addr_type = mlx5e_get_eth_addr_type(ai->addr); 187 switch (eth_addr_type) { 188 case MLX5E_UC: 189 ret = 190 (1 << MLX5E_TT_IPV4_TCP) | 191 (1 << MLX5E_TT_IPV6_TCP) | 192 (1 << MLX5E_TT_IPV4_UDP) | 193 (1 << MLX5E_TT_IPV6_UDP) | 194 (1 << MLX5E_TT_IPV4) | 195 (1 << MLX5E_TT_IPV6) | 196 (1 << MLX5E_TT_ANY) | 197 0; 198 break; 199 200 case MLX5E_MC_IPV4: 201 ret = 202 (1 << MLX5E_TT_IPV4_UDP) | 203 (1 << MLX5E_TT_IPV4) | 204 0; 205 break; 206 207 case MLX5E_MC_IPV6: 208 ret = 209 (1 << MLX5E_TT_IPV6_UDP) | 210 (1 << MLX5E_TT_IPV6) | 211 0; 212 break; 213 214 default: 215 ret = 216 (1 << MLX5E_TT_ANY) | 217 0; 218 break; 219 } 220 break; 221 222 case MLX5E_ALLMULTI: 223 ret = 224 (1 << MLX5E_TT_IPV4_UDP) | 225 (1 << MLX5E_TT_IPV6_UDP) | 226 (1 << MLX5E_TT_IPV4) | 227 (1 << MLX5E_TT_IPV6) | 228 (1 << MLX5E_TT_ANY) | 229 0; 230 break; 231 232 default: /* MLX5E_PROMISC */ 233 ret = 234 (1 << MLX5E_TT_IPV4_TCP) | 235 (1 << MLX5E_TT_IPV6_TCP) | 236 (1 << MLX5E_TT_IPV4_UDP) | 237 (1 << MLX5E_TT_IPV6_UDP) | 238 (1 << MLX5E_TT_IPV4) | 239 (1 << MLX5E_TT_IPV6) | 240 (1 << MLX5E_TT_ANY) | 241 0; 242 break; 243 } 244 245 return (ret); 246 } 247 248 static int 249 mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv, 250 struct mlx5e_eth_addr_info *ai, int type, 251 u32 *mc, u32 *mv) 252 { 253 struct mlx5_flow_destination dest = {}; 254 u8 mc_enable = 0; 255 struct mlx5_flow_rule **rule_p; 256 struct mlx5_flow_table *ft = priv->fts.main.t; 257 u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, mc, 258 outer_headers.dmac_47_16); 259 u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, mv, 260 outer_headers.dmac_47_16); 261 u32 *tirn = priv->tirn; 262 u32 tt_vec; 263 int err = 0; 264 struct mlx5_flow_act flow_act = { 265 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 266 .flow_tag = MLX5_FS_ETH_FLOW_TAG, 267 }; 268 269 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 270 271 switch (type) { 272 case MLX5E_FULLMATCH: 273 mc_enable = MLX5_MATCH_OUTER_HEADERS; 274 memset(mc_dmac, 0xff, ETH_ALEN); 275 ether_addr_copy(mv_dmac, ai->addr); 276 break; 277 278 case MLX5E_ALLMULTI: 279 mc_enable = MLX5_MATCH_OUTER_HEADERS; 280 mc_dmac[0] = 0x01; 281 mv_dmac[0] = 0x01; 282 break; 283 284 case MLX5E_PROMISC: 285 break; 286 default: 287 break; 288 } 289 290 tt_vec = mlx5e_get_tt_vec(ai, type); 291 292 if (tt_vec & BIT(MLX5E_TT_ANY)) { 293 rule_p = &ai->ft_rule[MLX5E_TT_ANY]; 294 dest.tir_num = tirn[MLX5E_TT_ANY]; 295 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 296 MLX5_FLOW_RULE_FWD_ACTION_DEST, 297 &flow_act, &dest); 298 if (IS_ERR_OR_NULL(*rule_p)) 299 goto err_del_ai; 300 } 301 302 mc_enable = MLX5_MATCH_OUTER_HEADERS; 303 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 304 305 if (tt_vec & BIT(MLX5E_TT_IPV4)) { 306 rule_p = &ai->ft_rule[MLX5E_TT_IPV4]; 307 dest.tir_num = tirn[MLX5E_TT_IPV4]; 308 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 309 ETHERTYPE_IP); 310 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 311 MLX5_FLOW_RULE_FWD_ACTION_DEST, 312 &flow_act, &dest); 313 if (IS_ERR_OR_NULL(*rule_p)) 314 goto err_del_ai; 315 } 316 317 if (tt_vec & BIT(MLX5E_TT_IPV6)) { 318 rule_p = &ai->ft_rule[MLX5E_TT_IPV6]; 319 dest.tir_num = tirn[MLX5E_TT_IPV6]; 320 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 321 ETHERTYPE_IPV6); 322 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 323 MLX5_FLOW_RULE_FWD_ACTION_DEST, 324 &flow_act, &dest); 325 if (IS_ERR_OR_NULL(*rule_p)) 326 goto err_del_ai; 327 } 328 329 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); 330 MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_UDP); 331 332 if (tt_vec & BIT(MLX5E_TT_IPV4_UDP)) { 333 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_UDP]; 334 dest.tir_num = tirn[MLX5E_TT_IPV4_UDP]; 335 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 336 ETHERTYPE_IP); 337 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 338 MLX5_FLOW_RULE_FWD_ACTION_DEST, 339 &flow_act, &dest); 340 if (IS_ERR_OR_NULL(*rule_p)) 341 goto err_del_ai; 342 } 343 344 if (tt_vec & BIT(MLX5E_TT_IPV6_UDP)) { 345 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_UDP]; 346 dest.tir_num = tirn[MLX5E_TT_IPV6_UDP]; 347 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 348 ETHERTYPE_IPV6); 349 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 350 MLX5_FLOW_RULE_FWD_ACTION_DEST, 351 &flow_act, &dest); 352 if (IS_ERR_OR_NULL(*rule_p)) 353 goto err_del_ai; 354 } 355 356 MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_TCP); 357 358 if (tt_vec & BIT(MLX5E_TT_IPV4_TCP)) { 359 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_TCP]; 360 dest.tir_num = tirn[MLX5E_TT_IPV4_TCP]; 361 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 362 ETHERTYPE_IP); 363 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 364 MLX5_FLOW_RULE_FWD_ACTION_DEST, 365 &flow_act, &dest); 366 if (IS_ERR_OR_NULL(*rule_p)) 367 goto err_del_ai; 368 } 369 370 if (tt_vec & BIT(MLX5E_TT_IPV6_TCP)) { 371 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_TCP]; 372 dest.tir_num = tirn[MLX5E_TT_IPV6_TCP]; 373 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 374 ETHERTYPE_IPV6); 375 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 376 MLX5_FLOW_RULE_FWD_ACTION_DEST, 377 &flow_act, &dest); 378 if (IS_ERR_OR_NULL(*rule_p)) 379 goto err_del_ai; 380 } 381 382 MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_AH); 383 384 if (tt_vec & BIT(MLX5E_TT_IPV4_IPSEC_AH)) { 385 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]; 386 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH]; 387 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 388 ETHERTYPE_IP); 389 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 390 MLX5_FLOW_RULE_FWD_ACTION_DEST, 391 &flow_act, &dest); 392 if (IS_ERR_OR_NULL(*rule_p)) 393 goto err_del_ai; 394 } 395 396 if (tt_vec & BIT(MLX5E_TT_IPV6_IPSEC_AH)) { 397 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]; 398 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH]; 399 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 400 ETHERTYPE_IPV6); 401 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 402 MLX5_FLOW_RULE_FWD_ACTION_DEST, 403 &flow_act, &dest); 404 if (IS_ERR_OR_NULL(*rule_p)) 405 goto err_del_ai; 406 } 407 408 MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_ESP); 409 410 if (tt_vec & BIT(MLX5E_TT_IPV4_IPSEC_ESP)) { 411 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]; 412 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP]; 413 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 414 ETHERTYPE_IP); 415 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 416 MLX5_FLOW_RULE_FWD_ACTION_DEST, 417 &flow_act, &dest); 418 if (IS_ERR_OR_NULL(*rule_p)) 419 goto err_del_ai; 420 } 421 422 if (tt_vec & BIT(MLX5E_TT_IPV6_IPSEC_ESP)) { 423 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]; 424 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP]; 425 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, 426 ETHERTYPE_IPV6); 427 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 428 MLX5_FLOW_RULE_FWD_ACTION_DEST, 429 &flow_act, &dest); 430 if (IS_ERR_OR_NULL(*rule_p)) 431 goto err_del_ai; 432 } 433 434 return 0; 435 436 err_del_ai: 437 err = PTR_ERR(*rule_p); 438 *rule_p = NULL; 439 mlx5e_del_eth_addr_from_flow_table(priv, ai); 440 441 return err; 442 } 443 444 static int 445 mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv, 446 struct mlx5e_eth_addr_info *ai, int type) 447 { 448 u32 *match_criteria; 449 u32 *match_value; 450 int err = 0; 451 452 match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 453 match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 454 if (!match_value || !match_criteria) { 455 mlx5_en_err(priv->ifp, "alloc failed\n"); 456 err = -ENOMEM; 457 goto add_eth_addr_rule_out; 458 } 459 err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, match_criteria, 460 match_value); 461 462 add_eth_addr_rule_out: 463 kvfree(match_criteria); 464 kvfree(match_value); 465 466 return (err); 467 } 468 469 static void 470 mlx5e_del_main_vxlan_rules(struct mlx5e_priv *priv) 471 { 472 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]); 473 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]); 474 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]); 475 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]); 476 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]); 477 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]); 478 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]); 479 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]); 480 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]); 481 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]); 482 mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]); 483 } 484 485 static int 486 mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv) 487 { 488 struct mlx5_flow_destination dest = {}; 489 u8 mc_enable = 0; 490 struct mlx5_flow_rule **rule_p; 491 struct mlx5_flow_table *ft = priv->fts.main_vxlan.t; 492 u32 *tirn = priv->tirn_inner_vxlan; 493 struct mlx5_flow_act flow_act = { 494 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 495 .flow_tag = MLX5_FS_ETH_FLOW_TAG, 496 }; 497 int err = 0; 498 499 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 500 501 mc_enable = MLX5_MATCH_INNER_HEADERS; 502 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype); 503 504 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]; 505 dest.tir_num = tirn[MLX5E_TT_IPV4]; 506 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP); 507 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 508 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 509 if (IS_ERR_OR_NULL(*rule_p)) 510 goto err_del_ai; 511 512 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]; 513 dest.tir_num = tirn[MLX5E_TT_IPV6]; 514 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6); 515 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 516 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 517 if (IS_ERR_OR_NULL(*rule_p)) 518 goto err_del_ai; 519 520 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol); 521 MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_UDP); 522 523 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]; 524 dest.tir_num = tirn[MLX5E_TT_IPV4_UDP]; 525 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP); 526 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 527 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 528 if (IS_ERR_OR_NULL(*rule_p)) 529 goto err_del_ai; 530 531 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]; 532 dest.tir_num = tirn[MLX5E_TT_IPV6_UDP]; 533 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6); 534 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 535 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 536 if (IS_ERR_OR_NULL(*rule_p)) 537 goto err_del_ai; 538 539 MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_TCP); 540 541 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]; 542 dest.tir_num = tirn[MLX5E_TT_IPV4_TCP]; 543 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP); 544 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 545 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 546 if (IS_ERR_OR_NULL(*rule_p)) 547 goto err_del_ai; 548 549 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]; 550 dest.tir_num = tirn[MLX5E_TT_IPV6_TCP]; 551 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6); 552 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 553 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 554 if (IS_ERR_OR_NULL(*rule_p)) 555 goto err_del_ai; 556 557 MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_AH); 558 559 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]; 560 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH]; 561 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP); 562 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 563 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 564 if (IS_ERR_OR_NULL(*rule_p)) 565 goto err_del_ai; 566 567 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]; 568 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH]; 569 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6); 570 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 571 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 572 if (IS_ERR_OR_NULL(*rule_p)) 573 goto err_del_ai; 574 575 MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_ESP); 576 577 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]; 578 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP]; 579 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP); 580 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 581 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 582 if (IS_ERR_OR_NULL(*rule_p)) 583 goto err_del_ai; 584 585 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]; 586 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP]; 587 MLX5_SET(fte_match_param, mv, inner_headers.ethertype, 588 ETHERTYPE_IPV6); 589 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 590 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 591 if (IS_ERR_OR_NULL(*rule_p)) 592 goto err_del_ai; 593 594 mc_enable = 0; 595 memset(mv, 0, MLX5_ST_SZ_BYTES(fte_match_param)); 596 memset(mc, 0, MLX5_ST_SZ_BYTES(fte_match_param)); 597 rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_ANY]; 598 dest.tir_num = tirn[MLX5E_TT_ANY]; 599 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 600 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 601 if (IS_ERR_OR_NULL(*rule_p)) 602 goto err_del_ai; 603 604 return (0); 605 606 err_del_ai: 607 err = PTR_ERR(*rule_p); 608 *rule_p = NULL; 609 mlx5e_del_main_vxlan_rules(priv); 610 611 return (err); 612 } 613 614 static int 615 mlx5e_add_main_vxlan_rules(struct mlx5e_priv *priv) 616 { 617 u32 *match_criteria; 618 u32 *match_value; 619 int err = 0; 620 621 match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 622 match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 623 if (match_value == NULL || match_criteria == NULL) { 624 mlx5_en_err(priv->ifp, "alloc failed\n"); 625 err = -ENOMEM; 626 goto add_main_vxlan_rules_out; 627 } 628 err = mlx5e_add_main_vxlan_rules_sub(priv, match_criteria, match_value); 629 630 add_main_vxlan_rules_out: 631 kvfree(match_criteria); 632 kvfree(match_value); 633 634 return (err); 635 } 636 637 static int mlx5e_vport_context_update_vlans(struct mlx5e_priv *priv) 638 { 639 if_t ifp = priv->ifp; 640 int max_list_size; 641 int list_size; 642 u16 *vlans; 643 int vlan; 644 int err; 645 int i; 646 647 list_size = 0; 648 for_each_set_bit(vlan, priv->vlan.active_vlans, VLAN_N_VID) 649 list_size++; 650 651 max_list_size = 1 << MLX5_CAP_GEN(priv->mdev, log_max_vlan_list); 652 653 if (list_size > max_list_size) { 654 mlx5_en_err(ifp, 655 "ifnet vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n", 656 list_size, max_list_size); 657 list_size = max_list_size; 658 } 659 660 vlans = kcalloc(list_size, sizeof(*vlans), GFP_KERNEL); 661 if (!vlans) 662 return -ENOMEM; 663 664 i = 0; 665 for_each_set_bit(vlan, priv->vlan.active_vlans, VLAN_N_VID) { 666 if (i >= list_size) 667 break; 668 vlans[i++] = vlan; 669 } 670 671 err = mlx5_modify_nic_vport_vlans(priv->mdev, vlans, list_size); 672 if (err) 673 mlx5_en_err(ifp, "Failed to modify vport vlans list err(%d)\n", 674 err); 675 676 kfree(vlans); 677 return err; 678 } 679 680 enum mlx5e_vlan_rule_type { 681 MLX5E_VLAN_RULE_TYPE_UNTAGGED, 682 MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 683 MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 684 MLX5E_VLAN_RULE_TYPE_MATCH_VID, 685 }; 686 687 static int 688 mlx5e_add_vlan_rule_sub(struct mlx5e_priv *priv, 689 enum mlx5e_vlan_rule_type rule_type, u16 vid, 690 u32 *mc, u32 *mv) 691 { 692 struct mlx5_flow_table *ft = priv->fts.vlan.t; 693 struct mlx5_flow_destination dest = {}; 694 u8 mc_enable = 0; 695 struct mlx5_flow_rule **rule_p; 696 int err = 0; 697 struct mlx5_flow_act flow_act = { 698 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 699 .flow_tag = MLX5_FS_ETH_FLOW_TAG, 700 }; 701 702 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 703 dest.ft = priv->fts.vxlan.t; 704 705 mc_enable = MLX5_MATCH_OUTER_HEADERS; 706 707 switch (rule_type) { 708 case MLX5E_VLAN_RULE_TYPE_UNTAGGED: 709 rule_p = &priv->vlan.untagged_ft_rule; 710 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag); 711 break; 712 case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID: 713 rule_p = &priv->vlan.any_cvlan_ft_rule; 714 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag); 715 MLX5_SET(fte_match_param, mv, outer_headers.cvlan_tag, 1); 716 break; 717 case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID: 718 rule_p = &priv->vlan.any_svlan_ft_rule; 719 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.svlan_tag); 720 MLX5_SET(fte_match_param, mv, outer_headers.svlan_tag, 1); 721 break; 722 default: /* MLX5E_VLAN_RULE_TYPE_MATCH_VID */ 723 rule_p = &priv->vlan.active_vlans_ft_rule[vid]; 724 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag); 725 MLX5_SET(fte_match_param, mv, outer_headers.cvlan_tag, 1); 726 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.first_vid); 727 MLX5_SET(fte_match_param, mv, outer_headers.first_vid, vid); 728 mlx5e_vport_context_update_vlans(priv); 729 break; 730 } 731 732 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 733 MLX5_FLOW_RULE_FWD_ACTION_DEST, 734 &flow_act, 735 &dest); 736 737 if (IS_ERR(*rule_p)) { 738 err = PTR_ERR(*rule_p); 739 *rule_p = NULL; 740 mlx5_en_err(priv->ifp, "add rule failed\n"); 741 } 742 743 return (err); 744 } 745 746 static int 747 mlx5e_add_vlan_rule(struct mlx5e_priv *priv, 748 enum mlx5e_vlan_rule_type rule_type, u16 vid) 749 { 750 u32 *match_criteria; 751 u32 *match_value; 752 int err = 0; 753 754 match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 755 match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 756 if (!match_value || !match_criteria) { 757 mlx5_en_err(priv->ifp, "alloc failed\n"); 758 err = -ENOMEM; 759 goto add_vlan_rule_out; 760 } 761 762 err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, match_criteria, 763 match_value); 764 765 add_vlan_rule_out: 766 kvfree(match_criteria); 767 kvfree(match_value); 768 769 return (err); 770 } 771 772 static void 773 mlx5e_del_vlan_rule(struct mlx5e_priv *priv, 774 enum mlx5e_vlan_rule_type rule_type, u16 vid) 775 { 776 switch (rule_type) { 777 case MLX5E_VLAN_RULE_TYPE_UNTAGGED: 778 mlx5_del_flow_rule(&priv->vlan.untagged_ft_rule); 779 break; 780 case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID: 781 mlx5_del_flow_rule(&priv->vlan.any_cvlan_ft_rule); 782 break; 783 case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID: 784 mlx5_del_flow_rule(&priv->vlan.any_svlan_ft_rule); 785 break; 786 case MLX5E_VLAN_RULE_TYPE_MATCH_VID: 787 mlx5_del_flow_rule(&priv->vlan.active_vlans_ft_rule[vid]); 788 mlx5e_vport_context_update_vlans(priv); 789 break; 790 default: 791 break; 792 } 793 } 794 795 static void 796 mlx5e_del_any_vid_rules(struct mlx5e_priv *priv) 797 { 798 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); 799 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0); 800 } 801 802 static int 803 mlx5e_add_any_vid_rules(struct mlx5e_priv *priv) 804 { 805 int err; 806 807 err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); 808 if (err) 809 return (err); 810 811 err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0); 812 if (err) 813 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); 814 815 return (err); 816 } 817 818 void 819 mlx5e_enable_vlan_filter(struct mlx5e_priv *priv) 820 { 821 if (priv->vlan.filter_disabled) { 822 priv->vlan.filter_disabled = false; 823 if (if_getflags(priv->ifp) & IFF_PROMISC) 824 return; 825 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state)) 826 mlx5e_del_any_vid_rules(priv); 827 } 828 } 829 830 void 831 mlx5e_disable_vlan_filter(struct mlx5e_priv *priv) 832 { 833 if (!priv->vlan.filter_disabled) { 834 priv->vlan.filter_disabled = true; 835 if (if_getflags(priv->ifp) & IFF_PROMISC) 836 return; 837 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state)) 838 mlx5e_add_any_vid_rules(priv); 839 } 840 } 841 842 void 843 mlx5e_vlan_rx_add_vid(void *arg, if_t ifp, u16 vid) 844 { 845 struct mlx5e_priv *priv = arg; 846 847 if (ifp != priv->ifp) 848 return; 849 850 PRIV_LOCK(priv); 851 if (!test_and_set_bit(vid, priv->vlan.active_vlans) && 852 test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state)) 853 mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid); 854 PRIV_UNLOCK(priv); 855 } 856 857 void 858 mlx5e_vlan_rx_kill_vid(void *arg, if_t ifp, u16 vid) 859 { 860 struct mlx5e_priv *priv = arg; 861 862 if (ifp != priv->ifp) 863 return; 864 865 PRIV_LOCK(priv); 866 clear_bit(vid, priv->vlan.active_vlans); 867 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state)) 868 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid); 869 PRIV_UNLOCK(priv); 870 } 871 872 static int 873 mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv) 874 { 875 int err; 876 int i; 877 878 set_bit(0, priv->vlan.active_vlans); 879 for_each_set_bit(i, priv->vlan.active_vlans, VLAN_N_VID) { 880 err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, 881 i); 882 if (err) 883 goto error; 884 } 885 886 err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); 887 if (err) 888 goto error; 889 890 if (priv->vlan.filter_disabled) { 891 err = mlx5e_add_any_vid_rules(priv); 892 if (err) 893 goto error; 894 } 895 return (0); 896 error: 897 mlx5e_del_all_vlan_rules(priv); 898 return (err); 899 } 900 901 static void 902 mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv) 903 { 904 int i; 905 906 if (priv->vlan.filter_disabled) 907 mlx5e_del_any_vid_rules(priv); 908 909 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); 910 911 for_each_set_bit(i, priv->vlan.active_vlans, VLAN_N_VID) 912 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, i); 913 clear_bit(0, priv->vlan.active_vlans); 914 } 915 916 #define mlx5e_for_each_hash_node(hn, tmp, hash, i) \ 917 for (i = 0; i < MLX5E_ETH_ADDR_HASH_SIZE; i++) \ 918 LIST_FOREACH_SAFE(hn, &(hash)[i], hlist, tmp) 919 920 static void 921 mlx5e_execute_action(struct mlx5e_priv *priv, 922 struct mlx5e_eth_addr_hash_node *hn) 923 { 924 switch (hn->action) { 925 case MLX5E_ACTION_ADD: 926 mlx5e_add_eth_addr_rule(priv, &hn->ai, MLX5E_FULLMATCH); 927 hn->action = MLX5E_ACTION_NONE; 928 break; 929 930 case MLX5E_ACTION_DEL: 931 mlx5e_del_eth_addr_from_flow_table(priv, &hn->ai); 932 if (hn->mpfs_index != -1U) 933 mlx5_mpfs_del_mac(priv->mdev, hn->mpfs_index); 934 mlx5e_del_eth_addr_from_hash(hn); 935 break; 936 937 default: 938 break; 939 } 940 } 941 942 static struct mlx5e_eth_addr_hash_node * 943 mlx5e_move_hn(struct mlx5e_eth_addr_hash_head *fh, struct mlx5e_eth_addr_hash_head *uh) 944 { 945 struct mlx5e_eth_addr_hash_node *hn; 946 947 hn = LIST_FIRST(fh); 948 if (hn != NULL) { 949 LIST_REMOVE(hn, hlist); 950 LIST_INSERT_HEAD(uh, hn, hlist); 951 } 952 return (hn); 953 } 954 955 static struct mlx5e_eth_addr_hash_node * 956 mlx5e_remove_hn(struct mlx5e_eth_addr_hash_head *fh) 957 { 958 struct mlx5e_eth_addr_hash_node *hn; 959 960 hn = LIST_FIRST(fh); 961 if (hn != NULL) 962 LIST_REMOVE(hn, hlist); 963 return (hn); 964 } 965 966 struct mlx5e_copy_addr_ctx { 967 struct mlx5e_eth_addr_hash_head *free; 968 struct mlx5e_eth_addr_hash_head *fill; 969 bool success; 970 }; 971 972 static u_int 973 mlx5e_copy_addr(void *arg, struct sockaddr_dl *sdl, u_int cnt) 974 { 975 struct mlx5e_copy_addr_ctx *ctx = arg; 976 struct mlx5e_eth_addr_hash_node *hn; 977 978 hn = mlx5e_move_hn(ctx->free, ctx->fill); 979 if (hn == NULL) { 980 ctx->success = false; 981 return (0); 982 } 983 ether_addr_copy(hn->ai.addr, LLADDR(sdl)); 984 985 return (1); 986 } 987 988 static void 989 mlx5e_sync_ifp_addr(struct mlx5e_priv *priv) 990 { 991 struct mlx5e_copy_addr_ctx ctx; 992 struct mlx5e_eth_addr_hash_head head_free; 993 struct mlx5e_eth_addr_hash_head head_uc; 994 struct mlx5e_eth_addr_hash_head head_mc; 995 struct mlx5e_eth_addr_hash_node *hn; 996 if_t ifp = priv->ifp; 997 size_t x; 998 size_t num; 999 1000 PRIV_ASSERT_LOCKED(priv); 1001 1002 retry: 1003 LIST_INIT(&head_free); 1004 LIST_INIT(&head_uc); 1005 LIST_INIT(&head_mc); 1006 num = 1 + if_lladdr_count(ifp) + if_llmaddr_count(ifp); 1007 1008 /* allocate place holders */ 1009 for (x = 0; x != num; x++) { 1010 hn = malloc(sizeof(*hn), M_MLX5EN, M_WAITOK | M_ZERO); 1011 hn->action = MLX5E_ACTION_ADD; 1012 hn->mpfs_index = -1U; 1013 LIST_INSERT_HEAD(&head_free, hn, hlist); 1014 } 1015 1016 hn = mlx5e_move_hn(&head_free, &head_uc); 1017 MPASS(hn != NULL); 1018 1019 ether_addr_copy(hn->ai.addr, if_getlladdr(ifp)); 1020 1021 ctx.free = &head_free; 1022 ctx.fill = &head_uc; 1023 ctx.success = true; 1024 if_foreach_lladdr(ifp, mlx5e_copy_addr, &ctx); 1025 if (ctx.success == false) 1026 goto cleanup; 1027 1028 ctx.fill = &head_mc; 1029 if_foreach_llmaddr(ifp, mlx5e_copy_addr, &ctx); 1030 if (ctx.success == false) 1031 goto cleanup; 1032 1033 /* insert L2 unicast addresses into hash list */ 1034 1035 while ((hn = mlx5e_remove_hn(&head_uc)) != NULL) { 1036 if (mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_uc, hn) == 0) 1037 continue; 1038 if (hn->mpfs_index == -1U) 1039 mlx5_mpfs_add_mac(priv->mdev, &hn->mpfs_index, 1040 hn->ai.addr, 0, 0); 1041 } 1042 1043 /* insert L2 multicast addresses into hash list */ 1044 1045 while ((hn = mlx5e_remove_hn(&head_mc)) != NULL) { 1046 if (mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_mc, hn) == 0) 1047 continue; 1048 } 1049 1050 cleanup: 1051 while ((hn = mlx5e_remove_hn(&head_uc)) != NULL) 1052 free(hn, M_MLX5EN); 1053 while ((hn = mlx5e_remove_hn(&head_mc)) != NULL) 1054 free(hn, M_MLX5EN); 1055 while ((hn = mlx5e_remove_hn(&head_free)) != NULL) 1056 free(hn, M_MLX5EN); 1057 1058 if (ctx.success == false) 1059 goto retry; 1060 } 1061 1062 static void mlx5e_fill_addr_array(struct mlx5e_priv *priv, int list_type, 1063 u8 addr_array[][ETH_ALEN], int size) 1064 { 1065 bool is_uc = (list_type == MLX5_NIC_VPORT_LIST_TYPE_UC); 1066 if_t ifp = priv->ifp; 1067 struct mlx5e_eth_addr_hash_node *hn; 1068 struct mlx5e_eth_addr_hash_head *addr_list; 1069 struct mlx5e_eth_addr_hash_node *tmp; 1070 int i = 0; 1071 int hi; 1072 1073 addr_list = is_uc ? priv->eth_addr.if_uc : priv->eth_addr.if_mc; 1074 1075 if (is_uc) /* Make sure our own address is pushed first */ 1076 ether_addr_copy(addr_array[i++], if_getlladdr(ifp)); 1077 else if (priv->eth_addr.broadcast_enabled) 1078 ether_addr_copy(addr_array[i++], if_getbroadcastaddr(ifp)); 1079 1080 mlx5e_for_each_hash_node(hn, tmp, addr_list, hi) { 1081 if (ether_addr_equal(if_getlladdr(ifp), hn->ai.addr)) 1082 continue; 1083 if (i >= size) 1084 break; 1085 ether_addr_copy(addr_array[i++], hn->ai.addr); 1086 } 1087 } 1088 1089 static void mlx5e_vport_context_update_addr_list(struct mlx5e_priv *priv, 1090 int list_type) 1091 { 1092 bool is_uc = (list_type == MLX5_NIC_VPORT_LIST_TYPE_UC); 1093 struct mlx5e_eth_addr_hash_node *hn; 1094 u8 (*addr_array)[ETH_ALEN] = NULL; 1095 struct mlx5e_eth_addr_hash_head *addr_list; 1096 struct mlx5e_eth_addr_hash_node *tmp; 1097 int max_size; 1098 int size; 1099 int err; 1100 int hi; 1101 1102 size = is_uc ? 0 : (priv->eth_addr.broadcast_enabled ? 1 : 0); 1103 max_size = is_uc ? 1104 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_uc_list) : 1105 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_mc_list); 1106 1107 addr_list = is_uc ? priv->eth_addr.if_uc : priv->eth_addr.if_mc; 1108 mlx5e_for_each_hash_node(hn, tmp, addr_list, hi) 1109 size++; 1110 1111 if (size > max_size) { 1112 mlx5_en_err(priv->ifp, 1113 "ifp %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n", 1114 is_uc ? "UC" : "MC", size, max_size); 1115 size = max_size; 1116 } 1117 1118 if (size) { 1119 addr_array = kcalloc(size, ETH_ALEN, GFP_KERNEL); 1120 if (!addr_array) { 1121 err = -ENOMEM; 1122 goto out; 1123 } 1124 mlx5e_fill_addr_array(priv, list_type, addr_array, size); 1125 } 1126 1127 err = mlx5_modify_nic_vport_mac_list(priv->mdev, list_type, addr_array, size); 1128 out: 1129 if (err) 1130 mlx5_en_err(priv->ifp, 1131 "Failed to modify vport %s list err(%d)\n", 1132 is_uc ? "UC" : "MC", err); 1133 kfree(addr_array); 1134 } 1135 1136 static void mlx5e_vport_context_update(struct mlx5e_priv *priv) 1137 { 1138 struct mlx5e_eth_addr_db *ea = &priv->eth_addr; 1139 1140 mlx5e_vport_context_update_addr_list(priv, MLX5_NIC_VPORT_LIST_TYPE_UC); 1141 mlx5e_vport_context_update_addr_list(priv, MLX5_NIC_VPORT_LIST_TYPE_MC); 1142 mlx5_modify_nic_vport_promisc(priv->mdev, 0, 1143 ea->allmulti_enabled, 1144 ea->promisc_enabled); 1145 } 1146 1147 static void 1148 mlx5e_apply_ifp_addr(struct mlx5e_priv *priv) 1149 { 1150 struct mlx5e_eth_addr_hash_node *hn; 1151 struct mlx5e_eth_addr_hash_node *tmp; 1152 int i; 1153 1154 mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i) 1155 mlx5e_execute_action(priv, hn); 1156 1157 mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i) 1158 mlx5e_execute_action(priv, hn); 1159 } 1160 1161 static void 1162 mlx5e_handle_ifp_addr(struct mlx5e_priv *priv, bool rx_mode_enable) 1163 { 1164 struct mlx5e_eth_addr_hash_node *hn; 1165 struct mlx5e_eth_addr_hash_node *tmp; 1166 int i; 1167 1168 mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i) 1169 hn->action = MLX5E_ACTION_DEL; 1170 mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i) 1171 hn->action = MLX5E_ACTION_DEL; 1172 1173 if (rx_mode_enable) 1174 mlx5e_sync_ifp_addr(priv); 1175 1176 mlx5e_apply_ifp_addr(priv); 1177 } 1178 1179 static void 1180 mlx5e_set_rx_mode_core(struct mlx5e_priv *priv, bool rx_mode_enable) 1181 { 1182 struct mlx5e_eth_addr_db *ea = &priv->eth_addr; 1183 if_t ndev = priv->ifp; 1184 int ndev_flags = if_getflags(ndev); 1185 1186 bool promisc_enabled = rx_mode_enable && (ndev_flags & IFF_PROMISC); 1187 bool allmulti_enabled = rx_mode_enable && (ndev_flags & IFF_ALLMULTI); 1188 bool broadcast_enabled = rx_mode_enable; 1189 1190 bool enable_promisc = !ea->promisc_enabled && promisc_enabled; 1191 bool disable_promisc = ea->promisc_enabled && !promisc_enabled; 1192 bool enable_allmulti = !ea->allmulti_enabled && allmulti_enabled; 1193 bool disable_allmulti = ea->allmulti_enabled && !allmulti_enabled; 1194 bool enable_broadcast = !ea->broadcast_enabled && broadcast_enabled; 1195 bool disable_broadcast = ea->broadcast_enabled && !broadcast_enabled; 1196 1197 /* update broadcast address */ 1198 ether_addr_copy(priv->eth_addr.broadcast.addr, 1199 if_getbroadcastaddr(priv->ifp)); 1200 1201 if (enable_promisc) { 1202 mlx5e_add_eth_addr_rule(priv, &ea->promisc, MLX5E_PROMISC); 1203 if (!priv->vlan.filter_disabled) 1204 mlx5e_add_any_vid_rules(priv); 1205 } 1206 if (enable_allmulti) 1207 mlx5e_add_eth_addr_rule(priv, &ea->allmulti, MLX5E_ALLMULTI); 1208 if (enable_broadcast) 1209 mlx5e_add_eth_addr_rule(priv, &ea->broadcast, MLX5E_FULLMATCH); 1210 1211 mlx5e_handle_ifp_addr(priv, rx_mode_enable); 1212 1213 if (disable_broadcast) 1214 mlx5e_del_eth_addr_from_flow_table(priv, &ea->broadcast); 1215 if (disable_allmulti) 1216 mlx5e_del_eth_addr_from_flow_table(priv, &ea->allmulti); 1217 if (disable_promisc) { 1218 if (!priv->vlan.filter_disabled) 1219 mlx5e_del_any_vid_rules(priv); 1220 mlx5e_del_eth_addr_from_flow_table(priv, &ea->promisc); 1221 } 1222 1223 ea->promisc_enabled = promisc_enabled; 1224 ea->allmulti_enabled = allmulti_enabled; 1225 ea->broadcast_enabled = broadcast_enabled; 1226 1227 mlx5e_vport_context_update(priv); 1228 } 1229 1230 void 1231 mlx5e_set_rx_mode_work(struct work_struct *work) 1232 { 1233 struct mlx5e_priv *priv = 1234 container_of(work, struct mlx5e_priv, set_rx_mode_work); 1235 1236 PRIV_LOCK(priv); 1237 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state)) 1238 mlx5e_set_rx_mode_core(priv, true); 1239 PRIV_UNLOCK(priv); 1240 } 1241 1242 static void 1243 mlx5e_destroy_groups(struct mlx5e_flow_table *ft) 1244 { 1245 int i; 1246 1247 for (i = ft->num_groups - 1; i >= 0; i--) { 1248 if (!IS_ERR_OR_NULL(ft->g[i])) 1249 mlx5_destroy_flow_group(ft->g[i]); 1250 ft->g[i] = NULL; 1251 } 1252 ft->num_groups = 0; 1253 } 1254 1255 static void 1256 mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft) 1257 { 1258 mlx5e_destroy_groups(ft); 1259 kfree(ft->g); 1260 mlx5_destroy_flow_table(ft->t); 1261 ft->t = NULL; 1262 } 1263 1264 #define MLX5E_NUM_MAIN_GROUPS 10 1265 #define MLX5E_MAIN_GROUP0_SIZE BIT(4) 1266 #define MLX5E_MAIN_GROUP1_SIZE BIT(3) 1267 #define MLX5E_MAIN_GROUP2_SIZE BIT(1) 1268 #define MLX5E_MAIN_GROUP3_SIZE BIT(0) 1269 #define MLX5E_MAIN_GROUP4_SIZE BIT(14) 1270 #define MLX5E_MAIN_GROUP5_SIZE BIT(13) 1271 #define MLX5E_MAIN_GROUP6_SIZE BIT(11) 1272 #define MLX5E_MAIN_GROUP7_SIZE BIT(2) 1273 #define MLX5E_MAIN_GROUP8_SIZE BIT(1) 1274 #define MLX5E_MAIN_GROUP9_SIZE BIT(0) 1275 #define MLX5E_MAIN_TABLE_SIZE (MLX5E_MAIN_GROUP0_SIZE +\ 1276 MLX5E_MAIN_GROUP1_SIZE +\ 1277 MLX5E_MAIN_GROUP2_SIZE +\ 1278 MLX5E_MAIN_GROUP3_SIZE +\ 1279 MLX5E_MAIN_GROUP4_SIZE +\ 1280 MLX5E_MAIN_GROUP5_SIZE +\ 1281 MLX5E_MAIN_GROUP6_SIZE +\ 1282 MLX5E_MAIN_GROUP7_SIZE +\ 1283 MLX5E_MAIN_GROUP8_SIZE +\ 1284 MLX5E_MAIN_GROUP9_SIZE +\ 1285 0) 1286 1287 static int 1288 mlx5e_create_main_groups_sub(struct mlx5e_flow_table *ft, u32 *in, 1289 int inlen) 1290 { 1291 u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1292 u8 *dmac = MLX5_ADDR_OF(create_flow_group_in, in, 1293 match_criteria.outer_headers.dmac_47_16); 1294 int err; 1295 int ix = 0; 1296 1297 /* Tunnel rules need to be first in this list of groups */ 1298 1299 /* Start tunnel rules */ 1300 memset(in, 0, inlen); 1301 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1302 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1303 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); 1304 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport); 1305 MLX5_SET_CFG(in, start_flow_index, ix); 1306 ix += MLX5E_MAIN_GROUP0_SIZE; 1307 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1308 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1309 if (IS_ERR(ft->g[ft->num_groups])) 1310 goto err_destory_groups; 1311 ft->num_groups++; 1312 /* End Tunnel Rules */ 1313 1314 memset(in, 0, inlen); 1315 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1316 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1317 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); 1318 MLX5_SET_CFG(in, start_flow_index, ix); 1319 ix += MLX5E_MAIN_GROUP1_SIZE; 1320 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1321 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1322 if (IS_ERR(ft->g[ft->num_groups])) 1323 goto err_destory_groups; 1324 ft->num_groups++; 1325 1326 memset(in, 0, inlen); 1327 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1328 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1329 MLX5_SET_CFG(in, start_flow_index, ix); 1330 ix += MLX5E_MAIN_GROUP2_SIZE; 1331 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1332 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1333 if (IS_ERR(ft->g[ft->num_groups])) 1334 goto err_destory_groups; 1335 ft->num_groups++; 1336 1337 memset(in, 0, inlen); 1338 MLX5_SET_CFG(in, start_flow_index, ix); 1339 ix += MLX5E_MAIN_GROUP3_SIZE; 1340 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1341 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1342 if (IS_ERR(ft->g[ft->num_groups])) 1343 goto err_destory_groups; 1344 ft->num_groups++; 1345 1346 memset(in, 0, inlen); 1347 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1348 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1349 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); 1350 memset(dmac, 0xff, ETH_ALEN); 1351 MLX5_SET_CFG(in, start_flow_index, ix); 1352 ix += MLX5E_MAIN_GROUP4_SIZE; 1353 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1354 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1355 if (IS_ERR(ft->g[ft->num_groups])) 1356 goto err_destory_groups; 1357 ft->num_groups++; 1358 1359 memset(in, 0, inlen); 1360 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1361 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1362 memset(dmac, 0xff, ETH_ALEN); 1363 MLX5_SET_CFG(in, start_flow_index, ix); 1364 ix += MLX5E_MAIN_GROUP5_SIZE; 1365 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1366 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1367 if (IS_ERR(ft->g[ft->num_groups])) 1368 goto err_destory_groups; 1369 ft->num_groups++; 1370 1371 memset(in, 0, inlen); 1372 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1373 memset(dmac, 0xff, ETH_ALEN); 1374 MLX5_SET_CFG(in, start_flow_index, ix); 1375 ix += MLX5E_MAIN_GROUP6_SIZE; 1376 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1377 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1378 if (IS_ERR(ft->g[ft->num_groups])) 1379 goto err_destory_groups; 1380 ft->num_groups++; 1381 1382 memset(in, 0, inlen); 1383 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1384 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1385 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); 1386 dmac[0] = 0x01; 1387 MLX5_SET_CFG(in, start_flow_index, ix); 1388 ix += MLX5E_MAIN_GROUP7_SIZE; 1389 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1390 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1391 if (IS_ERR(ft->g[ft->num_groups])) 1392 goto err_destory_groups; 1393 ft->num_groups++; 1394 1395 memset(in, 0, inlen); 1396 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1397 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1398 dmac[0] = 0x01; 1399 MLX5_SET_CFG(in, start_flow_index, ix); 1400 ix += MLX5E_MAIN_GROUP8_SIZE; 1401 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1402 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1403 if (IS_ERR(ft->g[ft->num_groups])) 1404 goto err_destory_groups; 1405 ft->num_groups++; 1406 1407 memset(in, 0, inlen); 1408 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1409 dmac[0] = 0x01; 1410 MLX5_SET_CFG(in, start_flow_index, ix); 1411 ix += MLX5E_MAIN_GROUP9_SIZE; 1412 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1413 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1414 if (IS_ERR(ft->g[ft->num_groups])) 1415 goto err_destory_groups; 1416 ft->num_groups++; 1417 1418 return (0); 1419 1420 err_destory_groups: 1421 err = PTR_ERR(ft->g[ft->num_groups]); 1422 ft->g[ft->num_groups] = NULL; 1423 mlx5e_destroy_groups(ft); 1424 1425 return (err); 1426 } 1427 1428 static int 1429 mlx5e_create_main_groups(struct mlx5e_flow_table *ft) 1430 { 1431 u32 *in; 1432 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1433 int err; 1434 1435 in = mlx5_vzalloc(inlen); 1436 if (!in) 1437 return (-ENOMEM); 1438 1439 err = mlx5e_create_main_groups_sub(ft, in, inlen); 1440 1441 kvfree(in); 1442 return (err); 1443 } 1444 1445 #define MLX5E_MAIN_VXLAN_GROUP0_SIZE BIT(3) 1446 #define MLX5E_MAIN_VXLAN_GROUP1_SIZE BIT(3) 1447 #define MLX5E_MAIN_VXLAN_GROUP2_SIZE BIT(0) 1448 static int 1449 mlx5e_create_main_vxlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in, 1450 int inlen) 1451 { 1452 u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1453 int err; 1454 int ix = 0; 1455 1456 memset(in, 0, inlen); 1457 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS); 1458 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype); 1459 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol); 1460 MLX5_SET_CFG(in, start_flow_index, ix); 1461 ix += MLX5E_MAIN_VXLAN_GROUP0_SIZE; 1462 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1463 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1464 if (IS_ERR(ft->g[ft->num_groups])) 1465 goto err_destory_groups; 1466 ft->num_groups++; 1467 1468 memset(in, 0, inlen); 1469 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS); 1470 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype); 1471 MLX5_SET_CFG(in, start_flow_index, ix); 1472 ix += MLX5E_MAIN_VXLAN_GROUP1_SIZE; 1473 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1474 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1475 if (IS_ERR(ft->g[ft->num_groups])) 1476 goto err_destory_groups; 1477 ft->num_groups++; 1478 1479 memset(in, 0, inlen); 1480 MLX5_SET_CFG(in, start_flow_index, ix); 1481 ix += MLX5E_MAIN_VXLAN_GROUP2_SIZE; 1482 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1483 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1484 if (IS_ERR(ft->g[ft->num_groups])) 1485 goto err_destory_groups; 1486 ft->num_groups++; 1487 1488 return (0); 1489 1490 err_destory_groups: 1491 err = PTR_ERR(ft->g[ft->num_groups]); 1492 ft->g[ft->num_groups] = NULL; 1493 mlx5e_destroy_groups(ft); 1494 1495 return (err); 1496 } 1497 1498 static int 1499 mlx5e_create_main_vxlan_groups(struct mlx5e_flow_table *ft) 1500 { 1501 u32 *in; 1502 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1503 int err; 1504 1505 in = mlx5_vzalloc(inlen); 1506 if (!in) 1507 return (-ENOMEM); 1508 1509 err = mlx5e_create_main_vxlan_groups_sub(ft, in, inlen); 1510 1511 kvfree(in); 1512 return (err); 1513 } 1514 1515 1516 static int 1517 mlx5e_create_main_flow_table(struct mlx5e_priv *priv, bool inner_vxlan) 1518 { 1519 struct mlx5e_flow_table *ft = inner_vxlan ? &priv->fts.main_vxlan : 1520 &priv->fts.main; 1521 int err; 1522 1523 ft->num_groups = 0; 1524 ft->t = mlx5_create_flow_table(priv->fts.ns, 0, 1525 inner_vxlan ? "vxlan_main" : "main", MLX5E_MAIN_TABLE_SIZE); 1526 1527 if (IS_ERR(ft->t)) { 1528 err = PTR_ERR(ft->t); 1529 ft->t = NULL; 1530 return (err); 1531 } 1532 ft->g = kcalloc(MLX5E_NUM_MAIN_GROUPS, sizeof(*ft->g), GFP_KERNEL); 1533 if (!ft->g) { 1534 err = -ENOMEM; 1535 goto err_destroy_main_flow_table; 1536 } 1537 1538 err = inner_vxlan ? mlx5e_create_main_vxlan_groups(ft) : 1539 mlx5e_create_main_groups(ft); 1540 if (err) 1541 goto err_free_g; 1542 return (0); 1543 1544 err_free_g: 1545 kfree(ft->g); 1546 1547 err_destroy_main_flow_table: 1548 mlx5_destroy_flow_table(ft->t); 1549 ft->t = NULL; 1550 1551 return (err); 1552 } 1553 1554 static void mlx5e_destroy_main_flow_table(struct mlx5e_priv *priv) 1555 { 1556 mlx5e_destroy_flow_table(&priv->fts.main); 1557 } 1558 1559 static void mlx5e_destroy_main_vxlan_flow_table(struct mlx5e_priv *priv) 1560 { 1561 mlx5e_destroy_flow_table(&priv->fts.main_vxlan); 1562 } 1563 1564 #define MLX5E_NUM_VLAN_GROUPS 3 1565 #define MLX5E_VLAN_GROUP0_SIZE BIT(12) 1566 #define MLX5E_VLAN_GROUP1_SIZE BIT(1) 1567 #define MLX5E_VLAN_GROUP2_SIZE BIT(0) 1568 #define MLX5E_VLAN_TABLE_SIZE (MLX5E_VLAN_GROUP0_SIZE +\ 1569 MLX5E_VLAN_GROUP1_SIZE +\ 1570 MLX5E_VLAN_GROUP2_SIZE +\ 1571 0) 1572 1573 static int 1574 mlx5e_create_vlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in, 1575 int inlen) 1576 { 1577 int err; 1578 int ix = 0; 1579 u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1580 1581 memset(in, 0, inlen); 1582 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1583 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag); 1584 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.first_vid); 1585 MLX5_SET_CFG(in, start_flow_index, ix); 1586 ix += MLX5E_VLAN_GROUP0_SIZE; 1587 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1588 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1589 if (IS_ERR(ft->g[ft->num_groups])) 1590 goto err_destory_groups; 1591 ft->num_groups++; 1592 1593 memset(in, 0, inlen); 1594 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1595 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag); 1596 MLX5_SET_CFG(in, start_flow_index, ix); 1597 ix += MLX5E_VLAN_GROUP1_SIZE; 1598 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1599 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1600 if (IS_ERR(ft->g[ft->num_groups])) 1601 goto err_destory_groups; 1602 ft->num_groups++; 1603 1604 memset(in, 0, inlen); 1605 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1606 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.svlan_tag); 1607 MLX5_SET_CFG(in, start_flow_index, ix); 1608 ix += MLX5E_VLAN_GROUP2_SIZE; 1609 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1610 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1611 if (IS_ERR(ft->g[ft->num_groups])) 1612 goto err_destory_groups; 1613 ft->num_groups++; 1614 1615 return (0); 1616 1617 err_destory_groups: 1618 err = PTR_ERR(ft->g[ft->num_groups]); 1619 ft->g[ft->num_groups] = NULL; 1620 mlx5e_destroy_groups(ft); 1621 1622 return (err); 1623 } 1624 1625 static int 1626 mlx5e_create_vlan_groups(struct mlx5e_flow_table *ft) 1627 { 1628 u32 *in; 1629 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1630 int err; 1631 1632 in = mlx5_vzalloc(inlen); 1633 if (!in) 1634 return (-ENOMEM); 1635 1636 err = mlx5e_create_vlan_groups_sub(ft, in, inlen); 1637 1638 kvfree(in); 1639 return (err); 1640 } 1641 1642 static int 1643 mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv) 1644 { 1645 struct mlx5e_flow_table *ft = &priv->fts.vlan; 1646 int err; 1647 1648 ft->num_groups = 0; 1649 ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vlan", 1650 MLX5E_VLAN_TABLE_SIZE); 1651 1652 if (IS_ERR(ft->t)) { 1653 err = PTR_ERR(ft->t); 1654 ft->t = NULL; 1655 return (err); 1656 } 1657 ft->g = kcalloc(MLX5E_NUM_VLAN_GROUPS, sizeof(*ft->g), GFP_KERNEL); 1658 if (!ft->g) { 1659 err = -ENOMEM; 1660 goto err_destroy_vlan_flow_table; 1661 } 1662 1663 err = mlx5e_create_vlan_groups(ft); 1664 if (err) 1665 goto err_free_g; 1666 1667 return (0); 1668 1669 err_free_g: 1670 kfree(ft->g); 1671 1672 err_destroy_vlan_flow_table: 1673 mlx5_destroy_flow_table(ft->t); 1674 ft->t = NULL; 1675 1676 return (err); 1677 } 1678 1679 static void 1680 mlx5e_destroy_vlan_flow_table(struct mlx5e_priv *priv) 1681 { 1682 mlx5e_destroy_flow_table(&priv->fts.vlan); 1683 } 1684 1685 static int 1686 mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv, 1687 struct mlx5e_vxlan_db_el *el) 1688 { 1689 struct mlx5_flow_table *ft = priv->fts.vxlan.t; 1690 struct mlx5_flow_destination dest = {}; 1691 u8 mc_enable; 1692 struct mlx5_flow_rule **rule_p; 1693 int err = 0; 1694 struct mlx5_flow_act flow_act = { 1695 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 1696 .flow_tag = MLX5_FS_ETH_FLOW_TAG, 1697 }; 1698 1699 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1700 dest.ft = priv->fts.main_vxlan.t; 1701 1702 mc_enable = MLX5_MATCH_OUTER_HEADERS; 1703 rule_p = &el->vxlan_ft_rule; 1704 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1705 MLX5_SET(fte_match_param, mv, outer_headers.ethertype, el->proto); 1706 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); 1707 MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_UDP); 1708 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport); 1709 MLX5_SET(fte_match_param, mv, outer_headers.udp_dport, el->port); 1710 1711 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 1712 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 1713 1714 if (IS_ERR(*rule_p)) { 1715 err = PTR_ERR(*rule_p); 1716 *rule_p = NULL; 1717 mlx5_en_err(priv->ifp, "add rule failed\n"); 1718 } 1719 1720 return (err); 1721 } 1722 1723 static struct mlx5e_vxlan_db_el * 1724 mlx5e_vxlan_find_db_el(struct mlx5e_priv *priv, u_int proto, u_int port) 1725 { 1726 struct mlx5e_vxlan_db_el *el; 1727 1728 TAILQ_FOREACH(el, &priv->vxlan.head, link) { 1729 if (el->proto == proto && el->port == port) 1730 return (el); 1731 } 1732 return (NULL); 1733 } 1734 1735 static struct mlx5e_vxlan_db_el * 1736 mlx5e_vxlan_alloc_db_el(struct mlx5e_priv *priv, u_int proto, u_int port) 1737 { 1738 struct mlx5e_vxlan_db_el *el; 1739 1740 el = mlx5_vzalloc(sizeof(*el)); 1741 el->refcount = 1; 1742 el->proto = proto; 1743 el->port = port; 1744 el->vxlan_ft_rule = NULL; 1745 return (el); 1746 } 1747 1748 static int 1749 mlx5e_vxlan_family_to_proto(sa_family_t family, u_int *proto) 1750 { 1751 switch (family) { 1752 case AF_INET: 1753 *proto = ETHERTYPE_IP; 1754 return (0); 1755 case AF_INET6: 1756 *proto = ETHERTYPE_IPV6; 1757 return (0); 1758 default: 1759 return (-EINVAL); 1760 } 1761 } 1762 1763 static int 1764 mlx5e_add_vxlan_rule_from_db(struct mlx5e_priv *priv, 1765 struct mlx5e_vxlan_db_el *el) 1766 { 1767 u32 *match_criteria; 1768 u32 *match_value; 1769 int err; 1770 1771 match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 1772 match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 1773 if (match_value == NULL || match_criteria == NULL) { 1774 mlx5_en_err(priv->ifp, "alloc failed\n"); 1775 err = -ENOMEM; 1776 goto add_vxlan_rule_out; 1777 } 1778 1779 err = mlx5e_add_vxlan_rule_sub(priv, match_criteria, match_value, el); 1780 1781 add_vxlan_rule_out: 1782 kvfree(match_criteria); 1783 kvfree(match_value); 1784 1785 return (err); 1786 } 1787 1788 static int 1789 mlx5e_add_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port) 1790 { 1791 struct mlx5e_vxlan_db_el *el; 1792 u_int proto; 1793 int err; 1794 1795 err = mlx5e_vxlan_family_to_proto(family, &proto); 1796 if (err != 0) 1797 return (err); 1798 1799 el = mlx5e_vxlan_find_db_el(priv, proto, port); 1800 if (el != NULL) { 1801 el->refcount++; 1802 if (el->installed) 1803 return (0); 1804 } 1805 el = mlx5e_vxlan_alloc_db_el(priv, proto, port); 1806 1807 if ((if_getcapenable(priv->ifp) & IFCAP_VXLAN_HWCSUM) != 0) { 1808 err = mlx5e_add_vxlan_rule_from_db(priv, el); 1809 if (err == 0) 1810 el->installed = true; 1811 } 1812 if (err == 0) 1813 TAILQ_INSERT_TAIL(&priv->vxlan.head, el, link); 1814 else 1815 kvfree(el); 1816 1817 return (err); 1818 } 1819 1820 static int 1821 mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv) 1822 { 1823 struct mlx5_flow_table *ft = priv->fts.vxlan.t; 1824 struct mlx5_flow_destination dest = {}; 1825 u8 mc_enable = 0; 1826 struct mlx5_flow_rule **rule_p; 1827 int err = 0; 1828 struct mlx5_flow_act flow_act = { 1829 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG, 1830 .flow_tag = MLX5_FS_ETH_FLOW_TAG, 1831 }; 1832 1833 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1834 dest.ft = priv->fts.main.t; 1835 1836 rule_p = &priv->fts.vxlan_catchall_ft_rule; 1837 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv, 1838 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest); 1839 1840 if (IS_ERR(*rule_p)) { 1841 err = PTR_ERR(*rule_p); 1842 *rule_p = NULL; 1843 mlx5_en_err(priv->ifp, "add rule failed\n"); 1844 } 1845 1846 return (err); 1847 } 1848 1849 1850 static int 1851 mlx5e_add_vxlan_catchall_rule(struct mlx5e_priv *priv) 1852 { 1853 u32 *match_criteria; 1854 u32 *match_value; 1855 int err; 1856 1857 match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 1858 match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); 1859 if (match_value == NULL || match_criteria == NULL) { 1860 mlx5_en_err(priv->ifp, "alloc failed\n"); 1861 err = -ENOMEM; 1862 goto add_vxlan_rule_out; 1863 } 1864 1865 err = mlx5e_add_vxlan_catchall_rule_sub(priv, match_criteria, 1866 match_value); 1867 1868 add_vxlan_rule_out: 1869 kvfree(match_criteria); 1870 kvfree(match_value); 1871 1872 return (err); 1873 } 1874 1875 int 1876 mlx5e_add_all_vxlan_rules(struct mlx5e_priv *priv) 1877 { 1878 struct mlx5e_vxlan_db_el *el; 1879 int err; 1880 1881 err = 0; 1882 TAILQ_FOREACH(el, &priv->vxlan.head, link) { 1883 if (el->installed) 1884 continue; 1885 err = mlx5e_add_vxlan_rule_from_db(priv, el); 1886 if (err != 0) 1887 break; 1888 el->installed = true; 1889 } 1890 1891 return (err); 1892 } 1893 1894 static int 1895 mlx5e_del_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port) 1896 { 1897 struct mlx5e_vxlan_db_el *el; 1898 u_int proto; 1899 int err; 1900 1901 err = mlx5e_vxlan_family_to_proto(family, &proto); 1902 if (err != 0) 1903 return (err); 1904 1905 el = mlx5e_vxlan_find_db_el(priv, proto, port); 1906 if (el == NULL) 1907 return (0); 1908 if (el->refcount > 1) { 1909 el->refcount--; 1910 return (0); 1911 } 1912 1913 if (el->installed) 1914 mlx5_del_flow_rule(&el->vxlan_ft_rule); 1915 TAILQ_REMOVE(&priv->vxlan.head, el, link); 1916 kvfree(el); 1917 return (0); 1918 } 1919 1920 void 1921 mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv) 1922 { 1923 struct mlx5e_vxlan_db_el *el; 1924 1925 TAILQ_FOREACH(el, &priv->vxlan.head, link) { 1926 if (!el->installed) 1927 continue; 1928 mlx5_del_flow_rule(&el->vxlan_ft_rule); 1929 el->installed = false; 1930 } 1931 } 1932 1933 static void 1934 mlx5e_del_vxlan_catchall_rule(struct mlx5e_priv *priv) 1935 { 1936 mlx5_del_flow_rule(&priv->fts.vxlan_catchall_ft_rule); 1937 } 1938 1939 void 1940 mlx5e_vxlan_start(void *arg, if_t ifp __unused, sa_family_t family, 1941 u_int port) 1942 { 1943 struct mlx5e_priv *priv = arg; 1944 int err; 1945 1946 PRIV_LOCK(priv); 1947 err = mlx5_vxlan_udp_port_add(priv->mdev, port); 1948 if (err == 0 && test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state)) 1949 mlx5e_add_vxlan_rule(priv, family, port); 1950 PRIV_UNLOCK(priv); 1951 } 1952 1953 void 1954 mlx5e_vxlan_stop(void *arg, if_t ifp __unused, sa_family_t family, 1955 u_int port) 1956 { 1957 struct mlx5e_priv *priv = arg; 1958 1959 PRIV_LOCK(priv); 1960 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state)) 1961 mlx5e_del_vxlan_rule(priv, family, port); 1962 (void)mlx5_vxlan_udp_port_delete(priv->mdev, port); 1963 PRIV_UNLOCK(priv); 1964 } 1965 1966 #define MLX5E_VXLAN_GROUP0_SIZE BIT(3) /* XXXKIB */ 1967 #define MLX5E_VXLAN_GROUP1_SIZE BIT(0) 1968 #define MLX5E_NUM_VXLAN_GROUPS BIT(1) 1969 #define MLX5E_VXLAN_TABLE_SIZE \ 1970 (MLX5E_VXLAN_GROUP0_SIZE + MLX5E_VXLAN_GROUP1_SIZE) 1971 1972 static int 1973 mlx5e_create_vxlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in, 1974 int inlen) 1975 { 1976 int err; 1977 int ix = 0; 1978 u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1979 1980 memset(in, 0, inlen); 1981 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1982 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1983 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); 1984 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport); 1985 MLX5_SET_CFG(in, start_flow_index, ix); 1986 ix += MLX5E_VXLAN_GROUP0_SIZE; 1987 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1988 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1989 if (IS_ERR(ft->g[ft->num_groups])) 1990 goto err_destory_groups; 1991 ft->num_groups++; 1992 1993 memset(in, 0, inlen); 1994 MLX5_SET_CFG(in, start_flow_index, ix); 1995 ix += MLX5E_VXLAN_GROUP1_SIZE; 1996 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1997 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1998 if (IS_ERR(ft->g[ft->num_groups])) 1999 goto err_destory_groups; 2000 ft->num_groups++; 2001 2002 return (0); 2003 2004 err_destory_groups: 2005 err = PTR_ERR(ft->g[ft->num_groups]); 2006 ft->g[ft->num_groups] = NULL; 2007 mlx5e_destroy_groups(ft); 2008 2009 return (err); 2010 } 2011 2012 static int 2013 mlx5e_create_vxlan_groups(struct mlx5e_flow_table *ft) 2014 { 2015 u32 *in; 2016 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2017 int err; 2018 2019 in = mlx5_vzalloc(inlen); 2020 if (!in) 2021 return (-ENOMEM); 2022 2023 err = mlx5e_create_vxlan_groups_sub(ft, in, inlen); 2024 2025 kvfree(in); 2026 return (err); 2027 } 2028 2029 static int 2030 mlx5e_create_vxlan_flow_table(struct mlx5e_priv *priv) 2031 { 2032 struct mlx5e_flow_table *ft = &priv->fts.vxlan; 2033 int err; 2034 2035 ft->num_groups = 0; 2036 ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vxlan", 2037 MLX5E_VXLAN_TABLE_SIZE); 2038 2039 if (IS_ERR(ft->t)) { 2040 err = PTR_ERR(ft->t); 2041 ft->t = NULL; 2042 return (err); 2043 } 2044 ft->g = kcalloc(MLX5E_NUM_VXLAN_GROUPS, sizeof(*ft->g), GFP_KERNEL); 2045 if (!ft->g) { 2046 err = -ENOMEM; 2047 goto err_destroy_vxlan_flow_table; 2048 } 2049 2050 err = mlx5e_create_vxlan_groups(ft); 2051 if (err) 2052 goto err_free_g; 2053 2054 TAILQ_INIT(&priv->vxlan.head); 2055 return (0); 2056 2057 err_free_g: 2058 kfree(ft->g); 2059 2060 err_destroy_vxlan_flow_table: 2061 mlx5_destroy_flow_table(ft->t); 2062 ft->t = NULL; 2063 2064 return (err); 2065 } 2066 2067 #define MLX5E_NUM_INNER_RSS_GROUPS 3 2068 #define MLX5E_INNER_RSS_GROUP0_SIZE BIT(3) 2069 #define MLX5E_INNER_RSS_GROUP1_SIZE BIT(1) 2070 #define MLX5E_INNER_RSS_GROUP2_SIZE BIT(0) 2071 #define MLX5E_INNER_RSS_TABLE_SIZE (MLX5E_INNER_RSS_GROUP0_SIZE +\ 2072 MLX5E_INNER_RSS_GROUP1_SIZE +\ 2073 MLX5E_INNER_RSS_GROUP2_SIZE +\ 2074 0) 2075 2076 static int 2077 mlx5e_create_inner_rss_groups_sub(struct mlx5e_flow_table *ft, u32 *in, 2078 int inlen) 2079 { 2080 u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 2081 int err; 2082 int ix = 0; 2083 2084 memset(in, 0, inlen); 2085 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS); 2086 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype); 2087 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol); 2088 MLX5_SET_CFG(in, start_flow_index, ix); 2089 ix += MLX5E_INNER_RSS_GROUP0_SIZE; 2090 MLX5_SET_CFG(in, end_flow_index, ix - 1); 2091 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 2092 if (IS_ERR(ft->g[ft->num_groups])) 2093 goto err_destory_groups; 2094 ft->num_groups++; 2095 2096 memset(in, 0, inlen); 2097 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS); 2098 MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype); 2099 MLX5_SET_CFG(in, start_flow_index, ix); 2100 ix += MLX5E_INNER_RSS_GROUP1_SIZE; 2101 MLX5_SET_CFG(in, end_flow_index, ix - 1); 2102 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 2103 if (IS_ERR(ft->g[ft->num_groups])) 2104 goto err_destory_groups; 2105 ft->num_groups++; 2106 2107 memset(in, 0, inlen); 2108 MLX5_SET_CFG(in, start_flow_index, ix); 2109 ix += MLX5E_INNER_RSS_GROUP2_SIZE; 2110 MLX5_SET_CFG(in, end_flow_index, ix - 1); 2111 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 2112 if (IS_ERR(ft->g[ft->num_groups])) 2113 goto err_destory_groups; 2114 ft->num_groups++; 2115 2116 return (0); 2117 2118 err_destory_groups: 2119 err = PTR_ERR(ft->g[ft->num_groups]); 2120 ft->g[ft->num_groups] = NULL; 2121 mlx5e_destroy_groups(ft); 2122 2123 return (err); 2124 } 2125 2126 static int 2127 mlx5e_create_inner_rss_groups(struct mlx5e_flow_table *ft) 2128 { 2129 u32 *in; 2130 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2131 int err; 2132 2133 in = mlx5_vzalloc(inlen); 2134 if (!in) 2135 return (-ENOMEM); 2136 2137 err = mlx5e_create_inner_rss_groups_sub(ft, in, inlen); 2138 2139 kvfree(in); 2140 return (err); 2141 } 2142 2143 static int 2144 mlx5e_create_inner_rss_flow_table(struct mlx5e_priv *priv) 2145 { 2146 struct mlx5e_flow_table *ft = &priv->fts.inner_rss; 2147 int err; 2148 2149 ft->num_groups = 0; 2150 ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "inner_rss", 2151 MLX5E_INNER_RSS_TABLE_SIZE); 2152 2153 if (IS_ERR(ft->t)) { 2154 err = PTR_ERR(ft->t); 2155 ft->t = NULL; 2156 return (err); 2157 } 2158 ft->g = kcalloc(MLX5E_NUM_INNER_RSS_GROUPS, sizeof(*ft->g), 2159 GFP_KERNEL); 2160 if (!ft->g) { 2161 err = -ENOMEM; 2162 goto err_destroy_inner_rss_flow_table; 2163 } 2164 2165 err = mlx5e_create_inner_rss_groups(ft); 2166 if (err) 2167 goto err_free_g; 2168 2169 return (0); 2170 2171 err_free_g: 2172 kfree(ft->g); 2173 2174 err_destroy_inner_rss_flow_table: 2175 mlx5_destroy_flow_table(ft->t); 2176 ft->t = NULL; 2177 2178 return (err); 2179 } 2180 2181 static void mlx5e_destroy_inner_rss_flow_table(struct mlx5e_priv *priv) 2182 { 2183 mlx5e_destroy_flow_table(&priv->fts.inner_rss); 2184 } 2185 2186 static void 2187 mlx5e_destroy_vxlan_flow_table(struct mlx5e_priv *priv) 2188 { 2189 mlx5e_destroy_flow_table(&priv->fts.vxlan); 2190 } 2191 2192 int 2193 mlx5e_open_flow_tables(struct mlx5e_priv *priv) 2194 { 2195 int err; 2196 2197 /* setup namespace pointer */ 2198 priv->fts.ns = mlx5_get_flow_namespace( 2199 priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL); 2200 2201 err = mlx5e_create_vlan_flow_table(priv); 2202 if (err) 2203 return (err); 2204 2205 err = mlx5e_create_vxlan_flow_table(priv); 2206 if (err) 2207 goto err_destroy_vlan_flow_table; 2208 2209 err = mlx5e_create_main_flow_table(priv, true); 2210 if (err) 2211 goto err_destroy_vxlan_flow_table; 2212 2213 err = mlx5e_create_inner_rss_flow_table(priv); 2214 if (err) 2215 goto err_destroy_main_flow_table_true; 2216 2217 err = mlx5e_create_main_flow_table(priv, false); 2218 if (err) 2219 goto err_destroy_inner_rss_flow_table; 2220 2221 err = mlx5e_add_vxlan_catchall_rule(priv); 2222 if (err) 2223 goto err_destroy_main_flow_table_false; 2224 2225 err = mlx5e_accel_fs_tcp_create(priv); 2226 if (err) 2227 goto err_del_vxlan_catchall_rule; 2228 2229 return (0); 2230 2231 err_del_vxlan_catchall_rule: 2232 mlx5e_del_vxlan_catchall_rule(priv); 2233 err_destroy_main_flow_table_false: 2234 mlx5e_destroy_main_flow_table(priv); 2235 err_destroy_inner_rss_flow_table: 2236 mlx5e_destroy_inner_rss_flow_table(priv); 2237 err_destroy_main_flow_table_true: 2238 mlx5e_destroy_main_vxlan_flow_table(priv); 2239 err_destroy_vxlan_flow_table: 2240 mlx5e_destroy_vxlan_flow_table(priv); 2241 err_destroy_vlan_flow_table: 2242 mlx5e_destroy_vlan_flow_table(priv); 2243 2244 return (err); 2245 } 2246 2247 void 2248 mlx5e_close_flow_tables(struct mlx5e_priv *priv) 2249 { 2250 mlx5e_accel_fs_tcp_destroy(priv); 2251 mlx5e_del_vxlan_catchall_rule(priv); 2252 mlx5e_destroy_main_flow_table(priv); 2253 mlx5e_destroy_inner_rss_flow_table(priv); 2254 mlx5e_destroy_main_vxlan_flow_table(priv); 2255 mlx5e_destroy_vxlan_flow_table(priv); 2256 mlx5e_destroy_vlan_flow_table(priv); 2257 } 2258 2259 int 2260 mlx5e_open_flow_rules(struct mlx5e_priv *priv) 2261 { 2262 int err; 2263 2264 err = mlx5e_add_all_vlan_rules(priv); 2265 if (err) 2266 return (err); 2267 2268 err = mlx5e_add_main_vxlan_rules(priv); 2269 if (err) 2270 goto err_del_all_vlan_rules; 2271 2272 err = mlx5e_add_all_vxlan_rules(priv); 2273 if (err) 2274 goto err_del_main_vxlan_rules; 2275 2276 mlx5e_set_rx_mode_core(priv, true); 2277 2278 set_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state); 2279 2280 return (0); 2281 2282 err_del_main_vxlan_rules: 2283 mlx5e_del_main_vxlan_rules(priv); 2284 2285 err_del_all_vlan_rules: 2286 mlx5e_del_all_vlan_rules(priv); 2287 2288 return (err); 2289 } 2290 2291 void 2292 mlx5e_close_flow_rules(struct mlx5e_priv *priv) 2293 { 2294 clear_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state); 2295 2296 mlx5e_set_rx_mode_core(priv, false); 2297 mlx5e_del_all_vxlan_rules(priv); 2298 mlx5e_del_main_vxlan_rules(priv); 2299 mlx5e_del_all_vlan_rules(priv); 2300 } 2301