1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/indir_table.h" 42 #include "esw/acl/ofld.h" 43 #include "rdma.h" 44 #include "en.h" 45 #include "fs_core.h" 46 #include "lib/mlx5.h" 47 #include "lib/devcom.h" 48 #include "lib/eq.h" 49 #include "lib/fs_chains.h" 50 #include "en_tc.h" 51 #include "en/mapping.h" 52 #include "devlink.h" 53 #include "lag/lag.h" 54 #include "en/tc/post_meter.h" 55 #include "fw_reset.h" 56 57 /* There are two match-all miss flows, one for unicast dst mac and 58 * one for multicast. 59 */ 60 #define MLX5_ESW_MISS_FLOWS (2) 61 #define UPLINK_REP_INDEX 0 62 63 #define MLX5_ESW_VPORT_TBL_SIZE 128 64 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 65 66 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 67 68 #define MLX5_ESW_MAX_CTRL_EQS 4 69 #define MLX5_ESW_DEFAULT_SF_COMP_EQS 8 70 71 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 72 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 73 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 74 .flags = 0, 75 }; 76 77 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 78 u16 vport_num) 79 { 80 return xa_load(&esw->offloads.vport_reps, vport_num); 81 } 82 83 static void 84 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 85 struct mlx5_flow_spec *spec, 86 struct mlx5_esw_flow_attr *attr) 87 { 88 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 89 return; 90 91 if (attr->int_port) { 92 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 93 94 return; 95 } 96 97 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 98 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 99 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 100 } 101 102 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 103 * are not needed as well in the following process. So clear them all for simplicity. 104 */ 105 void 106 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 107 { 108 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 109 void *misc2; 110 111 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 112 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 113 114 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 115 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 116 117 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 118 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 119 } 120 } 121 122 static void 123 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 124 struct mlx5_flow_spec *spec, 125 struct mlx5_flow_attr *attr, 126 struct mlx5_eswitch *src_esw, 127 u16 vport) 128 { 129 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 130 u32 metadata; 131 void *misc2; 132 void *misc; 133 134 /* Use metadata matching because vport is not represented by single 135 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 136 */ 137 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 138 if (mlx5_esw_indir_table_decap_vport(attr)) 139 vport = mlx5_esw_indir_table_decap_vport(attr); 140 141 if (!attr->chain && esw_attr && esw_attr->int_port) 142 metadata = 143 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 144 else 145 metadata = 146 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 147 148 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 149 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 150 151 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 152 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 153 mlx5_eswitch_get_vport_metadata_mask()); 154 155 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 156 } else { 157 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 158 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 159 160 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 161 MLX5_SET(fte_match_set_misc, misc, 162 source_eswitch_owner_vhca_id, 163 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 164 165 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 166 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 167 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 168 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 169 source_eswitch_owner_vhca_id); 170 171 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 172 } 173 } 174 175 static int 176 esw_setup_decap_indir(struct mlx5_eswitch *esw, 177 struct mlx5_flow_attr *attr) 178 { 179 struct mlx5_flow_table *ft; 180 181 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 182 return -EOPNOTSUPP; 183 184 ft = mlx5_esw_indir_table_get(esw, attr, 185 mlx5_esw_indir_table_decap_vport(attr), true); 186 return PTR_ERR_OR_ZERO(ft); 187 } 188 189 static void 190 esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 191 struct mlx5_flow_attr *attr) 192 { 193 if (mlx5_esw_indir_table_decap_vport(attr)) 194 mlx5_esw_indir_table_put(esw, 195 mlx5_esw_indir_table_decap_vport(attr), 196 true); 197 } 198 199 static int 200 esw_setup_mtu_dest(struct mlx5_flow_destination *dest, 201 struct mlx5e_meter_attr *meter, 202 int i) 203 { 204 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; 205 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; 206 dest[i].range.min = 0; 207 dest[i].range.max = meter->params.mtu; 208 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); 209 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); 210 211 return 0; 212 } 213 214 static int 215 esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 216 struct mlx5_flow_act *flow_act, 217 u32 sampler_id, 218 int i) 219 { 220 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 221 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 222 dest[i].sampler_id = sampler_id; 223 224 return 0; 225 } 226 227 static int 228 esw_setup_ft_dest(struct mlx5_flow_destination *dest, 229 struct mlx5_flow_act *flow_act, 230 struct mlx5_eswitch *esw, 231 struct mlx5_flow_attr *attr, 232 int i) 233 { 234 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 235 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 236 dest[i].ft = attr->dest_ft; 237 238 if (mlx5_esw_indir_table_decap_vport(attr)) 239 return esw_setup_decap_indir(esw, attr); 240 return 0; 241 } 242 243 static void 244 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 245 struct mlx5_fs_chains *chains, int i) 246 { 247 if (mlx5_chains_ignore_flow_level_supported(chains)) 248 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 249 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 250 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 251 } 252 253 static void 254 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 255 struct mlx5_eswitch *esw, int i) 256 { 257 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 258 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 259 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 260 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); 261 } 262 263 static int 264 esw_setup_chain_dest(struct mlx5_flow_destination *dest, 265 struct mlx5_flow_act *flow_act, 266 struct mlx5_fs_chains *chains, 267 u32 chain, u32 prio, u32 level, 268 int i) 269 { 270 struct mlx5_flow_table *ft; 271 272 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 273 ft = mlx5_chains_get_table(chains, chain, prio, level); 274 if (IS_ERR(ft)) 275 return PTR_ERR(ft); 276 277 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 278 dest[i].ft = ft; 279 return 0; 280 } 281 282 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 283 int from, int to) 284 { 285 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 286 struct mlx5_fs_chains *chains = esw_chains(esw); 287 int i; 288 289 for (i = from; i < to; i++) 290 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 291 mlx5_chains_put_table(chains, 0, 1, 0); 292 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 293 esw_attr->dests[i].mdev)) 294 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false); 295 } 296 297 static bool 298 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 299 { 300 int i; 301 302 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 303 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 304 return true; 305 return false; 306 } 307 308 static int 309 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 310 struct mlx5_flow_act *flow_act, 311 struct mlx5_eswitch *esw, 312 struct mlx5_fs_chains *chains, 313 struct mlx5_flow_attr *attr, 314 int *i) 315 { 316 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 317 int err; 318 319 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 320 return -EOPNOTSUPP; 321 322 /* flow steering cannot handle more than one dest with the same ft 323 * in a single flow 324 */ 325 if (esw_attr->out_count - esw_attr->split_count > 1) 326 return -EOPNOTSUPP; 327 328 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 329 if (err) 330 return err; 331 332 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 333 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 334 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 335 } 336 (*i)++; 337 338 return 0; 339 } 340 341 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 342 struct mlx5_flow_attr *attr) 343 { 344 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 345 346 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 347 } 348 349 static bool 350 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 351 { 352 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 353 bool result = false; 354 int i; 355 356 /* Indirect table is supported only for flows with in_port uplink 357 * and the destination is vport on the same eswitch as the uplink, 358 * return false in case at least one of destinations doesn't meet 359 * this criteria. 360 */ 361 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 362 if (esw_attr->dests[i].vport_valid && 363 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 364 esw_attr->dests[i].mdev)) { 365 result = true; 366 } else { 367 result = false; 368 break; 369 } 370 } 371 return result; 372 } 373 374 static int 375 esw_setup_indir_table(struct mlx5_flow_destination *dest, 376 struct mlx5_flow_act *flow_act, 377 struct mlx5_eswitch *esw, 378 struct mlx5_flow_attr *attr, 379 int *i) 380 { 381 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 382 int j, err; 383 384 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 385 return -EOPNOTSUPP; 386 387 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 388 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 389 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 390 391 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, 392 esw_attr->dests[j].vport, false); 393 if (IS_ERR(dest[*i].ft)) { 394 err = PTR_ERR(dest[*i].ft); 395 goto err_indir_tbl_get; 396 } 397 } 398 399 if (mlx5_esw_indir_table_decap_vport(attr)) { 400 err = esw_setup_decap_indir(esw, attr); 401 if (err) 402 goto err_indir_tbl_get; 403 } 404 405 return 0; 406 407 err_indir_tbl_get: 408 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 409 return err; 410 } 411 412 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 413 { 414 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 415 416 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 417 esw_cleanup_decap_indir(esw, attr); 418 } 419 420 static void 421 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 422 { 423 mlx5_chains_put_table(chains, chain, prio, level); 424 } 425 426 static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2) 427 { 428 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id); 429 } 430 431 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw, 432 struct mlx5_esw_flow_attr *esw_attr, 433 int attr_idx) 434 { 435 if (esw->offloads.ft_ipsec_tx_pol && 436 esw_attr->dests[attr_idx].vport_valid && 437 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK && 438 /* To be aligned with software, encryption is needed only for tunnel device */ 439 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) && 440 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport && 441 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev)) 442 return true; 443 444 return false; 445 } 446 447 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw, 448 struct mlx5_esw_flow_attr *esw_attr) 449 { 450 int i; 451 452 if (!esw->offloads.ft_ipsec_tx_pol) 453 return true; 454 455 for (i = 0; i < esw_attr->split_count; i++) 456 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i)) 457 return false; 458 459 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 460 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) && 461 (esw_attr->out_count - esw_attr->split_count > 1)) 462 return false; 463 464 return true; 465 } 466 467 static void 468 esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 469 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 470 int attr_idx, int dest_idx, bool pkt_reformat) 471 { 472 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 473 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport; 474 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 475 dest[dest_idx].vport.vhca_id = 476 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 477 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 478 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 479 mlx5_lag_is_mpesw(esw->dev)) 480 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 481 } 482 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 483 if (pkt_reformat) { 484 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 485 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 486 } 487 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 488 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 489 } 490 } 491 492 static void 493 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 494 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 495 int attr_idx, int dest_idx, bool pkt_reformat) 496 { 497 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol; 498 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 499 if (pkt_reformat && 500 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 501 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 502 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 503 } 504 } 505 506 static void 507 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 508 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 509 int attr_idx, int dest_idx, bool pkt_reformat) 510 { 511 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 512 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr, 513 attr_idx, dest_idx, pkt_reformat); 514 else 515 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr, 516 attr_idx, dest_idx, pkt_reformat); 517 } 518 519 static int 520 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 521 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 522 int i) 523 { 524 int j; 525 526 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 527 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 528 return i; 529 } 530 531 static bool 532 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 533 { 534 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 535 mlx5_eswitch_vport_match_metadata_enabled(esw) && 536 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 537 } 538 539 static bool 540 esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest) 541 { 542 bool internal_dest = false, external_dest = false; 543 int i; 544 545 for (i = 0; i < max_dest; i++) { 546 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT && 547 dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK) 548 continue; 549 550 /* Uplink dest is external, but considered as internal 551 * if there is reformat because firmware uses LB+hairpin to support it. 552 */ 553 if (dests[i].vport.num == MLX5_VPORT_UPLINK && 554 !(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)) 555 external_dest = true; 556 else 557 internal_dest = true; 558 559 if (internal_dest && external_dest) 560 return true; 561 } 562 563 return false; 564 } 565 566 static int 567 esw_setup_dests(struct mlx5_flow_destination *dest, 568 struct mlx5_flow_act *flow_act, 569 struct mlx5_eswitch *esw, 570 struct mlx5_flow_attr *attr, 571 struct mlx5_flow_spec *spec, 572 int *i) 573 { 574 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 575 struct mlx5_fs_chains *chains = esw_chains(esw); 576 int err = 0; 577 578 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 579 esw_src_port_rewrite_supported(esw)) 580 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 581 582 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 583 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 584 (*i)++; 585 goto out; 586 } 587 588 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { 589 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 590 (*i)++; 591 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 592 esw_setup_accept_dest(dest, flow_act, chains, *i); 593 (*i)++; 594 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { 595 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); 596 (*i)++; 597 } else if (esw_is_indir_table(esw, attr)) { 598 err = esw_setup_indir_table(dest, flow_act, esw, attr, i); 599 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 600 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 601 } else { 602 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 603 604 if (attr->dest_ft) { 605 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i); 606 (*i)++; 607 } else if (attr->dest_chain) { 608 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 609 1, 0, *i); 610 (*i)++; 611 } 612 } 613 614 if (attr->extra_split_ft) { 615 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 616 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 617 dest[*i].ft = attr->extra_split_ft; 618 (*i)++; 619 } 620 621 out: 622 return err; 623 } 624 625 static void 626 esw_cleanup_dests(struct mlx5_eswitch *esw, 627 struct mlx5_flow_attr *attr) 628 { 629 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 630 struct mlx5_fs_chains *chains = esw_chains(esw); 631 632 if (attr->dest_ft) { 633 esw_cleanup_decap_indir(esw, attr); 634 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 635 if (attr->dest_chain) 636 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 637 else if (esw_is_indir_table(esw, attr)) 638 esw_cleanup_indir_table(esw, attr); 639 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 640 esw_cleanup_chain_src_port_rewrite(esw, attr); 641 } 642 } 643 644 static void 645 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 646 { 647 struct mlx5e_flow_meter_handle *meter; 648 649 meter = attr->meter_attr.meter; 650 flow_act->exe_aso.type = attr->exe_aso_type; 651 flow_act->exe_aso.object_id = meter->obj_id; 652 flow_act->exe_aso.base_id = mlx5e_flow_meter_get_base_id(meter); 653 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 654 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 655 /* use metadata reg 5 for packet color */ 656 flow_act->exe_aso.return_reg_id = 5; 657 } 658 659 struct mlx5_flow_handle * 660 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 661 struct mlx5_flow_spec *spec, 662 struct mlx5_flow_attr *attr) 663 { 664 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 665 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 666 struct mlx5_fs_chains *chains = esw_chains(esw); 667 bool split = !!(esw_attr->split_count); 668 struct mlx5_vport_tbl_attr fwd_attr; 669 struct mlx5_flow_destination *dest; 670 struct mlx5_flow_handle *rule; 671 struct mlx5_flow_table *fdb; 672 int i = 0; 673 674 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 675 return ERR_PTR(-EOPNOTSUPP); 676 677 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 678 return ERR_PTR(-EOPNOTSUPP); 679 680 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr)) 681 return ERR_PTR(-EOPNOTSUPP); 682 683 dest = kzalloc_objs(*dest, MLX5_MAX_FLOW_FWD_VPORTS + 1); 684 if (!dest) 685 return ERR_PTR(-ENOMEM); 686 687 flow_act.action = attr->action; 688 689 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 690 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 691 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 692 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 693 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 694 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 695 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 696 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 697 } 698 } 699 700 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 701 702 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 703 int err; 704 705 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 706 if (err) { 707 rule = ERR_PTR(err); 708 goto err_create_goto_table; 709 } 710 711 /* Header rewrite with combined wire+loopback in FDB is not allowed */ 712 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) && 713 esw_dests_to_int_external(dest, i)) { 714 esw_warn(esw->dev, 715 "FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n"); 716 rule = ERR_PTR(-EINVAL); 717 goto err_esw_get; 718 } 719 } 720 721 if (esw_attr->decap_pkt_reformat) 722 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 723 724 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 725 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 726 dest[i].counter = attr->counter; 727 i++; 728 } 729 730 if (attr->outer_match_level != MLX5_MATCH_NONE) 731 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 732 if (attr->inner_match_level != MLX5_MATCH_NONE) 733 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 734 735 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 736 flow_act.modify_hdr = attr->modify_hdr; 737 738 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 739 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 740 esw_setup_meter(attr, &flow_act); 741 742 if (split) { 743 fwd_attr.chain = attr->chain; 744 fwd_attr.prio = attr->prio; 745 fwd_attr.vport = esw_attr->in_rep->vport; 746 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 747 748 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 749 } else { 750 if (attr->chain || attr->prio) 751 fdb = mlx5_chains_get_table(chains, attr->chain, 752 attr->prio, 0); 753 else 754 fdb = attr->ft; 755 756 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 757 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 758 esw_attr->in_mdev->priv.eswitch, 759 esw_attr->in_rep->vport); 760 } 761 if (IS_ERR(fdb)) { 762 rule = ERR_CAST(fdb); 763 goto err_esw_get; 764 } 765 766 if (!i) { 767 kfree(dest); 768 dest = NULL; 769 } 770 771 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 772 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 773 &flow_act, dest, i); 774 else 775 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 776 if (IS_ERR(rule)) 777 goto err_add_rule; 778 else 779 atomic64_inc(&esw->offloads.num_flows); 780 781 kfree(dest); 782 return rule; 783 784 err_add_rule: 785 if (split) 786 mlx5_esw_vporttbl_put(esw, &fwd_attr); 787 else if (attr->chain || attr->prio) 788 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 789 err_esw_get: 790 esw_cleanup_dests(esw, attr); 791 err_create_goto_table: 792 kfree(dest); 793 return rule; 794 } 795 796 struct mlx5_flow_handle * 797 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 798 struct mlx5_flow_spec *spec, 799 struct mlx5_flow_attr *attr) 800 { 801 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 802 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 803 struct mlx5_fs_chains *chains = esw_chains(esw); 804 struct mlx5_vport_tbl_attr fwd_attr; 805 struct mlx5_flow_destination *dest; 806 struct mlx5_flow_table *fast_fdb; 807 struct mlx5_flow_table *fwd_fdb; 808 struct mlx5_flow_handle *rule; 809 int i, err = 0; 810 811 dest = kzalloc_objs(*dest, MLX5_MAX_FLOW_FWD_VPORTS + 1); 812 if (!dest) 813 return ERR_PTR(-ENOMEM); 814 815 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 816 if (IS_ERR(fast_fdb)) { 817 rule = ERR_CAST(fast_fdb); 818 goto err_get_fast; 819 } 820 821 fwd_attr.chain = attr->chain; 822 fwd_attr.prio = attr->prio; 823 fwd_attr.vport = esw_attr->in_rep->vport; 824 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 825 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 826 if (IS_ERR(fwd_fdb)) { 827 rule = ERR_CAST(fwd_fdb); 828 goto err_get_fwd; 829 } 830 831 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 832 for (i = 0; i < esw_attr->split_count; i++) { 833 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 834 /* Source port rewrite (forward to ovs internal port or statck device) isn't 835 * supported in the rule of split action. 836 */ 837 err = -EOPNOTSUPP; 838 else 839 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 840 841 if (err) { 842 rule = ERR_PTR(err); 843 goto err_chain_src_rewrite; 844 } 845 } 846 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 847 dest[i].ft = fwd_fdb; 848 i++; 849 850 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 851 esw_attr->in_mdev->priv.eswitch, 852 esw_attr->in_rep->vport); 853 854 if (attr->outer_match_level != MLX5_MATCH_NONE) 855 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 856 857 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 858 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 859 860 if (IS_ERR(rule)) { 861 i = esw_attr->split_count; 862 goto err_chain_src_rewrite; 863 } 864 865 atomic64_inc(&esw->offloads.num_flows); 866 867 kfree(dest); 868 return rule; 869 err_chain_src_rewrite: 870 mlx5_esw_vporttbl_put(esw, &fwd_attr); 871 err_get_fwd: 872 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 873 err_get_fast: 874 kfree(dest); 875 return rule; 876 } 877 878 static void 879 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 880 struct mlx5_flow_handle *rule, 881 struct mlx5_flow_attr *attr, 882 bool fwd_rule) 883 { 884 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 885 struct mlx5_fs_chains *chains = esw_chains(esw); 886 bool split = (esw_attr->split_count > 0); 887 struct mlx5_vport_tbl_attr fwd_attr; 888 int i; 889 890 mlx5_del_flow_rules(rule); 891 892 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 893 /* unref the term table */ 894 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 895 if (esw_attr->dests[i].termtbl) 896 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 897 } 898 } 899 900 atomic64_dec(&esw->offloads.num_flows); 901 902 if (fwd_rule || split) { 903 fwd_attr.chain = attr->chain; 904 fwd_attr.prio = attr->prio; 905 fwd_attr.vport = esw_attr->in_rep->vport; 906 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 907 } 908 909 if (fwd_rule) { 910 mlx5_esw_vporttbl_put(esw, &fwd_attr); 911 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 912 } else { 913 if (split) 914 mlx5_esw_vporttbl_put(esw, &fwd_attr); 915 else if (attr->chain || attr->prio) 916 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 917 esw_cleanup_dests(esw, attr); 918 } 919 } 920 921 void 922 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 923 struct mlx5_flow_handle *rule, 924 struct mlx5_flow_attr *attr) 925 { 926 __mlx5_eswitch_del_rule(esw, rule, attr, false); 927 } 928 929 void 930 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 931 struct mlx5_flow_handle *rule, 932 struct mlx5_flow_attr *attr) 933 { 934 __mlx5_eswitch_del_rule(esw, rule, attr, true); 935 } 936 937 struct mlx5_flow_handle * 938 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 939 struct mlx5_eswitch *from_esw, 940 struct mlx5_eswitch_rep *rep, 941 u32 sqn) 942 { 943 struct mlx5_flow_act flow_act = {0}; 944 struct mlx5_flow_destination dest = {}; 945 struct mlx5_flow_handle *flow_rule; 946 struct mlx5_flow_spec *spec; 947 void *misc; 948 u16 vport; 949 950 spec = kvzalloc_obj(*spec); 951 if (!spec) { 952 flow_rule = ERR_PTR(-ENOMEM); 953 goto out; 954 } 955 956 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 957 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 958 959 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 960 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 961 962 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 963 964 /* source vport is the esw manager */ 965 vport = from_esw->manager_vport; 966 967 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) { 968 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 969 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 970 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport)); 971 972 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 973 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 974 mlx5_eswitch_get_vport_metadata_mask()); 975 976 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 977 } else { 978 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 979 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 980 981 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 982 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 983 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 984 985 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 986 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 987 988 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 989 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 990 source_eswitch_owner_vhca_id); 991 992 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 993 } 994 995 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 996 dest.vport.num = rep->vport; 997 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 998 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 999 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1000 1001 if (rep->vport == MLX5_VPORT_UPLINK && 1002 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) { 1003 dest.ft = on_esw->offloads.ft_ipsec_tx_pol; 1004 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; 1005 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1006 } else { 1007 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1008 dest.vport.num = rep->vport; 1009 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 1010 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1011 } 1012 1013 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) && 1014 rep->vport == MLX5_VPORT_UPLINK) 1015 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 1016 1017 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), 1018 spec, &flow_act, &dest, 1); 1019 if (IS_ERR(flow_rule)) 1020 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %pe\n", 1021 flow_rule); 1022 out: 1023 kvfree(spec); 1024 return flow_rule; 1025 } 1026 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1027 1028 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1029 { 1030 mlx5_del_flow_rules(rule); 1031 } 1032 1033 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) 1034 { 1035 if (rule) 1036 mlx5_del_flow_rules(rule); 1037 } 1038 1039 struct mlx5_flow_handle * 1040 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) 1041 { 1042 struct mlx5_flow_destination dest = {}; 1043 struct mlx5_flow_act flow_act = {0}; 1044 struct mlx5_flow_handle *flow_rule; 1045 struct mlx5_flow_spec *spec; 1046 1047 spec = kvzalloc_obj(*spec); 1048 if (!spec) 1049 return ERR_PTR(-ENOMEM); 1050 1051 MLX5_SET(fte_match_param, spec->match_criteria, 1052 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1053 MLX5_SET(fte_match_param, spec->match_criteria, 1054 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1055 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1056 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1057 1058 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1059 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1060 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1061 1062 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1063 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1064 dest.vport.num = vport_num; 1065 1066 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1067 spec, &flow_act, &dest, 1); 1068 if (IS_ERR(flow_rule)) 1069 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %pe\n", 1070 vport_num, flow_rule); 1071 1072 kvfree(spec); 1073 return flow_rule; 1074 } 1075 1076 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1077 { 1078 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1079 MLX5_FDB_TO_VPORT_REG_C_1; 1080 } 1081 1082 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1083 { 1084 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1085 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1086 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1087 u8 curr, wanted; 1088 int err; 1089 1090 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1091 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1092 return 0; 1093 1094 MLX5_SET(query_esw_vport_context_in, in, opcode, 1095 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1096 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1097 if (err) 1098 return err; 1099 1100 curr = MLX5_GET(query_esw_vport_context_out, out, 1101 esw_vport_context.fdb_to_vport_reg_c_id); 1102 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1103 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1104 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1105 1106 if (enable) 1107 curr |= wanted; 1108 else 1109 curr &= ~wanted; 1110 1111 MLX5_SET(modify_esw_vport_context_in, min, 1112 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1113 MLX5_SET(modify_esw_vport_context_in, min, 1114 field_select.fdb_to_vport_reg_c_id, 1); 1115 1116 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1117 if (!err) { 1118 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1119 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1120 else 1121 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1122 } 1123 1124 return err; 1125 } 1126 1127 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1128 struct mlx5_core_dev *peer_dev, 1129 struct mlx5_flow_spec *spec, 1130 struct mlx5_flow_destination *dest) 1131 { 1132 void *misc; 1133 1134 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1135 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1136 misc_parameters_2); 1137 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1138 mlx5_eswitch_get_vport_metadata_mask()); 1139 1140 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1141 } else { 1142 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1143 misc_parameters); 1144 1145 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1146 MLX5_CAP_GEN(peer_dev, vhca_id)); 1147 1148 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1149 1150 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1151 misc_parameters); 1152 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1153 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1154 source_eswitch_owner_vhca_id); 1155 } 1156 1157 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1158 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1159 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1160 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1161 } 1162 1163 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1164 struct mlx5_eswitch *peer_esw, 1165 struct mlx5_flow_spec *spec, 1166 u16 vport) 1167 { 1168 void *misc; 1169 1170 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1171 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1172 misc_parameters_2); 1173 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1174 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1175 vport)); 1176 } else { 1177 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1178 misc_parameters); 1179 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1180 } 1181 } 1182 1183 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1184 struct mlx5_core_dev *peer_dev) 1185 { 1186 struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch; 1187 struct mlx5_flow_destination dest = {}; 1188 struct mlx5_flow_act flow_act = {0}; 1189 struct mlx5_flow_handle **flows; 1190 struct mlx5_flow_handle *flow; 1191 struct mlx5_vport *peer_vport; 1192 struct mlx5_flow_spec *spec; 1193 int err; 1194 unsigned long i; 1195 void *misc; 1196 1197 if (!MLX5_VPORT_MANAGER(peer_dev) && 1198 !mlx5_core_is_ecpf_esw_manager(peer_dev)) 1199 return 0; 1200 1201 spec = kvzalloc_obj(*spec); 1202 if (!spec) 1203 return -ENOMEM; 1204 1205 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1206 1207 flows = kvzalloc_objs(*flows, peer_esw->total_vports); 1208 if (!flows) { 1209 err = -ENOMEM; 1210 goto alloc_flows_err; 1211 } 1212 1213 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1214 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1215 misc_parameters); 1216 1217 if (mlx5_core_is_ecpf_esw_manager(peer_dev) && 1218 mlx5_esw_host_functions_enabled(peer_dev)) { 1219 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); 1220 esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, 1221 MLX5_VPORT_PF); 1222 1223 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1224 spec, &flow_act, &dest, 1); 1225 if (IS_ERR(flow)) { 1226 err = PTR_ERR(flow); 1227 goto add_pf_flow_err; 1228 } 1229 flows[peer_vport->index] = flow; 1230 } 1231 1232 if (mlx5_ecpf_vport_exists(peer_dev)) { 1233 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); 1234 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1235 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1236 spec, &flow_act, &dest, 1); 1237 if (IS_ERR(flow)) { 1238 err = PTR_ERR(flow); 1239 goto add_ecpf_flow_err; 1240 } 1241 flows[peer_vport->index] = flow; 1242 } 1243 1244 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, 1245 mlx5_core_max_vfs(peer_dev)) { 1246 esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, 1247 peer_vport->vport); 1248 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1249 spec, &flow_act, &dest, 1); 1250 if (IS_ERR(flow)) { 1251 err = PTR_ERR(flow); 1252 goto add_vf_flow_err; 1253 } 1254 flows[peer_vport->index] = flow; 1255 } 1256 1257 if (mlx5_core_ec_sriov_enabled(peer_dev)) { 1258 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, 1259 mlx5_core_max_ec_vfs(peer_dev)) { 1260 esw_set_peer_miss_rule_source_port(esw, peer_esw, 1261 spec, 1262 peer_vport->vport); 1263 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1264 spec, &flow_act, &dest, 1); 1265 if (IS_ERR(flow)) { 1266 err = PTR_ERR(flow); 1267 goto add_ec_vf_flow_err; 1268 } 1269 flows[peer_vport->index] = flow; 1270 } 1271 } 1272 1273 err = xa_insert(&esw->fdb_table.offloads.peer_miss_rules, 1274 MLX5_CAP_GEN(peer_dev, vhca_id), flows, GFP_KERNEL); 1275 if (err) 1276 goto add_ec_vf_flow_err; 1277 1278 kvfree(spec); 1279 return 0; 1280 1281 add_ec_vf_flow_err: 1282 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, 1283 mlx5_core_max_ec_vfs(peer_dev)) { 1284 if (!flows[peer_vport->index]) 1285 continue; 1286 mlx5_del_flow_rules(flows[peer_vport->index]); 1287 } 1288 add_vf_flow_err: 1289 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, 1290 mlx5_core_max_vfs(peer_dev)) { 1291 if (!flows[peer_vport->index]) 1292 continue; 1293 mlx5_del_flow_rules(flows[peer_vport->index]); 1294 } 1295 if (mlx5_ecpf_vport_exists(peer_dev)) { 1296 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); 1297 mlx5_del_flow_rules(flows[peer_vport->index]); 1298 } 1299 add_ecpf_flow_err: 1300 1301 if (mlx5_core_is_ecpf_esw_manager(peer_dev) && 1302 mlx5_esw_host_functions_enabled(peer_dev)) { 1303 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); 1304 mlx5_del_flow_rules(flows[peer_vport->index]); 1305 } 1306 add_pf_flow_err: 1307 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1308 kvfree(flows); 1309 alloc_flows_err: 1310 kvfree(spec); 1311 return err; 1312 } 1313 1314 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1315 struct mlx5_core_dev *peer_dev) 1316 { 1317 struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch; 1318 u16 peer_vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1319 struct mlx5_flow_handle **flows; 1320 struct mlx5_vport *peer_vport; 1321 unsigned long i; 1322 1323 flows = xa_erase(&esw->fdb_table.offloads.peer_miss_rules, 1324 peer_vhca_id); 1325 if (!flows) 1326 return; 1327 1328 if (mlx5_core_ec_sriov_enabled(peer_dev)) { 1329 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, 1330 mlx5_core_max_ec_vfs(peer_dev)) 1331 mlx5_del_flow_rules(flows[peer_vport->index]); 1332 } 1333 1334 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, 1335 mlx5_core_max_vfs(peer_dev)) 1336 mlx5_del_flow_rules(flows[peer_vport->index]); 1337 1338 if (mlx5_ecpf_vport_exists(peer_dev)) { 1339 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); 1340 mlx5_del_flow_rules(flows[peer_vport->index]); 1341 } 1342 1343 if (mlx5_core_is_ecpf_esw_manager(peer_dev) && 1344 mlx5_esw_host_functions_enabled(peer_dev)) { 1345 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF); 1346 mlx5_del_flow_rules(flows[peer_vport->index]); 1347 } 1348 1349 kvfree(flows); 1350 } 1351 1352 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1353 { 1354 struct mlx5_flow_act flow_act = {0}; 1355 struct mlx5_flow_destination dest = {}; 1356 struct mlx5_flow_handle *flow_rule = NULL; 1357 struct mlx5_flow_spec *spec; 1358 void *headers_c; 1359 void *headers_v; 1360 int err = 0; 1361 u8 *dmac_c; 1362 u8 *dmac_v; 1363 1364 spec = kvzalloc_obj(*spec); 1365 if (!spec) { 1366 err = -ENOMEM; 1367 goto out; 1368 } 1369 1370 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1371 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1372 outer_headers); 1373 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1374 outer_headers.dmac_47_16); 1375 dmac_c[0] = 0x01; 1376 1377 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1378 dest.vport.num = esw->manager_vport; 1379 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1380 1381 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1382 spec, &flow_act, &dest, 1); 1383 if (IS_ERR(flow_rule)) { 1384 err = PTR_ERR(flow_rule); 1385 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1386 goto out; 1387 } 1388 1389 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1390 1391 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1392 outer_headers); 1393 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1394 outer_headers.dmac_47_16); 1395 dmac_v[0] = 0x01; 1396 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1397 spec, &flow_act, &dest, 1); 1398 if (IS_ERR(flow_rule)) { 1399 err = PTR_ERR(flow_rule); 1400 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1401 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1402 goto out; 1403 } 1404 1405 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1406 1407 out: 1408 kvfree(spec); 1409 return err; 1410 } 1411 1412 struct mlx5_flow_handle * 1413 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1414 { 1415 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1416 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1417 struct mlx5_flow_context *flow_context; 1418 struct mlx5_flow_handle *flow_rule; 1419 struct mlx5_flow_destination dest; 1420 struct mlx5_flow_spec *spec; 1421 void *misc; 1422 1423 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1424 return ERR_PTR(-EOPNOTSUPP); 1425 1426 spec = kvzalloc_obj(*spec); 1427 if (!spec) 1428 return ERR_PTR(-ENOMEM); 1429 1430 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1431 misc_parameters_2); 1432 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1433 ESW_REG_C0_USER_DATA_METADATA_MASK); 1434 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1435 misc_parameters_2); 1436 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1437 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1438 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1439 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1440 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1441 1442 flow_context = &spec->flow_context; 1443 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1444 flow_context->flow_tag = tag; 1445 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1446 dest.ft = esw->offloads.ft_offloads; 1447 1448 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1449 kvfree(spec); 1450 1451 if (IS_ERR(flow_rule)) 1452 esw_warn(esw->dev, 1453 "Failed to create restore rule for tag: %d, err(%d)\n", 1454 tag, (int)PTR_ERR(flow_rule)); 1455 1456 return flow_rule; 1457 } 1458 1459 struct mlx5_flow_group * 1460 mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw, 1461 struct mlx5_flow_table *ft) 1462 { 1463 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1464 struct mlx5_flow_group *fg; 1465 void *match_criteria; 1466 void *flow_group_in; 1467 1468 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 1469 return ERR_PTR(-EOPNOTSUPP); 1470 1471 if (IS_ERR(ft)) 1472 return ERR_CAST(ft); 1473 1474 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1475 if (!flow_group_in) 1476 return ERR_PTR(-ENOMEM); 1477 1478 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1479 match_criteria); 1480 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1481 MLX5_MATCH_MISC_PARAMETERS_2); 1482 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1483 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1484 ft->max_fte - 1); 1485 1486 MLX5_SET(fte_match_param, match_criteria, 1487 misc_parameters_2.metadata_reg_c_0, 1488 mlx5_eswitch_get_vport_metadata_mask()); 1489 1490 fg = mlx5_create_flow_group(ft, flow_group_in); 1491 kvfree(flow_group_in); 1492 if (IS_ERR(fg)) 1493 esw_warn(esw->dev, "Can't create LAG demux flow group\n"); 1494 1495 return fg; 1496 } 1497 1498 struct mlx5_flow_handle * 1499 mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num, 1500 struct mlx5_flow_table *lag_ft) 1501 { 1502 struct mlx5_flow_spec *spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1503 struct mlx5_flow_destination dest = {}; 1504 struct mlx5_flow_act flow_act = {}; 1505 struct mlx5_flow_handle *ret; 1506 void *misc; 1507 1508 if (!spec) 1509 return ERR_PTR(-ENOMEM); 1510 1511 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1512 kvfree(spec); 1513 return ERR_PTR(-EOPNOTSUPP); 1514 } 1515 1516 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1517 misc_parameters_2); 1518 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1519 mlx5_eswitch_get_vport_metadata_mask()); 1520 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1521 1522 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1523 misc_parameters_2); 1524 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1525 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1526 1527 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1528 dest.type = MLX5_FLOW_DESTINATION_TYPE_VHCA_RX; 1529 dest.vhca.id = MLX5_CAP_GEN(esw->dev, vhca_id); 1530 1531 ret = mlx5_add_flow_rules(lag_ft, spec, &flow_act, &dest, 1); 1532 kvfree(spec); 1533 return ret; 1534 } 1535 1536 #define MAX_PF_SQ 256 1537 #define MAX_SQ_NVPORTS 32 1538 1539 void 1540 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1541 u32 *flow_group_in, 1542 int match_params) 1543 { 1544 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1545 flow_group_in, 1546 match_criteria); 1547 1548 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1549 MLX5_SET(create_flow_group_in, flow_group_in, 1550 match_criteria_enable, 1551 MLX5_MATCH_MISC_PARAMETERS_2 | match_params); 1552 1553 MLX5_SET(fte_match_param, match_criteria, 1554 misc_parameters_2.metadata_reg_c_0, 1555 mlx5_eswitch_get_vport_metadata_mask()); 1556 } else { 1557 MLX5_SET(create_flow_group_in, flow_group_in, 1558 match_criteria_enable, 1559 MLX5_MATCH_MISC_PARAMETERS | match_params); 1560 1561 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1562 misc_parameters.source_port); 1563 } 1564 } 1565 1566 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1567 static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1568 { 1569 struct mlx5_vport_tbl_attr attr; 1570 struct mlx5_vport *vport; 1571 unsigned long i; 1572 1573 attr.chain = 0; 1574 attr.prio = 1; 1575 mlx5_esw_for_each_vport(esw, i, vport) { 1576 attr.vport = vport->vport; 1577 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1578 mlx5_esw_vporttbl_put(esw, &attr); 1579 } 1580 } 1581 1582 static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1583 { 1584 struct mlx5_vport_tbl_attr attr; 1585 struct mlx5_flow_table *fdb; 1586 struct mlx5_vport *vport; 1587 unsigned long i; 1588 1589 attr.chain = 0; 1590 attr.prio = 1; 1591 mlx5_esw_for_each_vport(esw, i, vport) { 1592 attr.vport = vport->vport; 1593 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1594 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1595 if (IS_ERR(fdb)) 1596 goto out; 1597 } 1598 return 0; 1599 1600 out: 1601 esw_vport_tbl_put(esw); 1602 return PTR_ERR(fdb); 1603 } 1604 1605 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1606 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1607 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1608 { 1609 struct mlx5_core_dev *dev = esw->dev; 1610 1611 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1612 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1613 1614 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1615 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1616 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1617 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1618 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1619 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1620 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1621 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1622 /* Disabled when ttl workaround is needed, e.g 1623 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1624 */ 1625 esw_warn(dev, 1626 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1627 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1628 } else { 1629 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1630 esw_info(dev, "Supported tc chains and prios offload\n"); 1631 } 1632 1633 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1634 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1635 } 1636 1637 static int 1638 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1639 { 1640 struct mlx5_core_dev *dev = esw->dev; 1641 struct mlx5_flow_table *nf_ft, *ft; 1642 struct mlx5_chains_attr attr = {}; 1643 struct mlx5_fs_chains *chains; 1644 int err; 1645 1646 esw_init_chains_offload_flags(esw, &attr.flags); 1647 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1648 attr.max_grp_num = esw->params.large_group_num; 1649 attr.default_ft = miss_fdb; 1650 attr.mapping = esw->offloads.reg_c0_obj_pool; 1651 attr.fs_base_prio = FDB_BYPASS_PATH; 1652 1653 chains = mlx5_chains_create(dev, &attr); 1654 if (IS_ERR(chains)) { 1655 err = PTR_ERR(chains); 1656 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1657 return err; 1658 } 1659 mlx5_chains_print_info(chains); 1660 1661 esw->fdb_table.offloads.esw_chains_priv = chains; 1662 1663 /* Create tc_end_ft which is the always created ft chain */ 1664 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1665 1, 0); 1666 if (IS_ERR(nf_ft)) { 1667 err = PTR_ERR(nf_ft); 1668 goto nf_ft_err; 1669 } 1670 1671 /* Always open the root for fast path */ 1672 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1673 if (IS_ERR(ft)) { 1674 err = PTR_ERR(ft); 1675 goto level_0_err; 1676 } 1677 1678 /* Open level 1 for split fdb rules now if prios isn't supported */ 1679 if (!mlx5_chains_prios_supported(chains)) { 1680 err = esw_vport_tbl_get(esw); 1681 if (err) 1682 goto level_1_err; 1683 } 1684 1685 mlx5_chains_set_end_ft(chains, nf_ft); 1686 1687 return 0; 1688 1689 level_1_err: 1690 mlx5_chains_put_table(chains, 0, 1, 0); 1691 level_0_err: 1692 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1693 nf_ft_err: 1694 mlx5_chains_destroy(chains); 1695 esw->fdb_table.offloads.esw_chains_priv = NULL; 1696 1697 return err; 1698 } 1699 1700 static void 1701 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1702 { 1703 if (!mlx5_chains_prios_supported(chains)) 1704 esw_vport_tbl_put(esw); 1705 mlx5_chains_put_table(chains, 0, 1, 0); 1706 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1707 mlx5_chains_destroy(chains); 1708 } 1709 1710 #else /* CONFIG_MLX5_CLS_ACT */ 1711 1712 static int 1713 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1714 { return 0; } 1715 1716 static void 1717 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1718 {} 1719 1720 #endif 1721 1722 static int 1723 esw_create_send_to_vport_group(struct mlx5_eswitch *esw, 1724 struct mlx5_flow_table *fdb, 1725 u32 *flow_group_in, 1726 int *ix) 1727 { 1728 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1729 struct mlx5_flow_group *g; 1730 void *match_criteria; 1731 int count, err = 0; 1732 1733 memset(flow_group_in, 0, inlen); 1734 1735 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS); 1736 1737 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1738 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1739 1740 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) && 1741 MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1742 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1743 misc_parameters.source_eswitch_owner_vhca_id); 1744 MLX5_SET(create_flow_group_in, flow_group_in, 1745 source_eswitch_owner_vhca_id_valid, 1); 1746 } 1747 1748 /* See comment at table_size calculation */ 1749 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1750 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1751 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); 1752 *ix += count; 1753 1754 g = mlx5_create_flow_group(fdb, flow_group_in); 1755 if (IS_ERR(g)) { 1756 err = PTR_ERR(g); 1757 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1758 goto out; 1759 } 1760 esw->fdb_table.offloads.send_to_vport_grp = g; 1761 1762 out: 1763 return err; 1764 } 1765 1766 static int 1767 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, 1768 struct mlx5_flow_table *fdb, 1769 u32 *flow_group_in, 1770 int *ix) 1771 { 1772 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1773 struct mlx5_flow_group *g; 1774 void *match_criteria; 1775 int err = 0; 1776 1777 if (!esw_src_port_rewrite_supported(esw)) 1778 return 0; 1779 1780 memset(flow_group_in, 0, inlen); 1781 1782 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1783 MLX5_MATCH_MISC_PARAMETERS_2); 1784 1785 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1786 1787 MLX5_SET(fte_match_param, match_criteria, 1788 misc_parameters_2.metadata_reg_c_0, 1789 mlx5_eswitch_get_vport_metadata_mask()); 1790 MLX5_SET(fte_match_param, match_criteria, 1791 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1792 1793 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1794 MLX5_SET(create_flow_group_in, flow_group_in, 1795 end_flow_index, *ix + esw->total_vports - 1); 1796 *ix += esw->total_vports; 1797 1798 g = mlx5_create_flow_group(fdb, flow_group_in); 1799 if (IS_ERR(g)) { 1800 err = PTR_ERR(g); 1801 esw_warn(esw->dev, 1802 "Failed to create send-to-vport meta flow group err(%d)\n", err); 1803 goto send_vport_meta_err; 1804 } 1805 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1806 1807 return 0; 1808 1809 send_vport_meta_err: 1810 return err; 1811 } 1812 1813 static int 1814 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, 1815 struct mlx5_flow_table *fdb, 1816 u32 *flow_group_in, 1817 int *ix) 1818 { 1819 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1); 1820 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1821 struct mlx5_flow_group *g; 1822 void *match_criteria; 1823 int err = 0; 1824 1825 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1826 return 0; 1827 1828 memset(flow_group_in, 0, inlen); 1829 1830 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 1831 1832 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1833 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1834 flow_group_in, 1835 match_criteria); 1836 1837 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1838 misc_parameters.source_eswitch_owner_vhca_id); 1839 1840 MLX5_SET(create_flow_group_in, flow_group_in, 1841 source_eswitch_owner_vhca_id_valid, 1); 1842 } 1843 1844 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1845 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1846 *ix + max_peer_ports); 1847 *ix += max_peer_ports + 1; 1848 1849 g = mlx5_create_flow_group(fdb, flow_group_in); 1850 if (IS_ERR(g)) { 1851 err = PTR_ERR(g); 1852 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); 1853 goto out; 1854 } 1855 esw->fdb_table.offloads.peer_miss_grp = g; 1856 1857 out: 1858 return err; 1859 } 1860 1861 static int 1862 esw_create_miss_group(struct mlx5_eswitch *esw, 1863 struct mlx5_flow_table *fdb, 1864 u32 *flow_group_in, 1865 int *ix) 1866 { 1867 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1868 struct mlx5_flow_group *g; 1869 void *match_criteria; 1870 int err = 0; 1871 u8 *dmac; 1872 1873 memset(flow_group_in, 0, inlen); 1874 1875 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1876 MLX5_MATCH_OUTER_HEADERS); 1877 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1878 match_criteria); 1879 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1880 outer_headers.dmac_47_16); 1881 dmac[0] = 0x01; 1882 1883 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1884 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1885 *ix + MLX5_ESW_MISS_FLOWS); 1886 1887 g = mlx5_create_flow_group(fdb, flow_group_in); 1888 if (IS_ERR(g)) { 1889 err = PTR_ERR(g); 1890 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); 1891 goto miss_err; 1892 } 1893 esw->fdb_table.offloads.miss_grp = g; 1894 1895 err = esw_add_fdb_miss_rule(esw); 1896 if (err) 1897 goto miss_rule_err; 1898 1899 return 0; 1900 1901 miss_rule_err: 1902 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1903 miss_err: 1904 return err; 1905 } 1906 1907 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1908 { 1909 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1910 struct mlx5_flow_table_attr ft_attr = {}; 1911 struct mlx5_core_dev *dev = esw->dev; 1912 struct mlx5_flow_namespace *root_ns; 1913 struct mlx5_flow_table *fdb = NULL; 1914 int table_size, ix = 0, err = 0; 1915 u32 flags = 0, *flow_group_in; 1916 1917 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1918 1919 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1920 if (!flow_group_in) 1921 return -ENOMEM; 1922 1923 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1924 if (!root_ns) { 1925 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1926 err = -EOPNOTSUPP; 1927 goto ns_err; 1928 } 1929 esw->fdb_table.offloads.ns = root_ns; 1930 err = mlx5_flow_namespace_set_mode(root_ns, 1931 esw->dev->priv.steering->mode); 1932 if (err) { 1933 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1934 goto ns_err; 1935 } 1936 1937 /* To be strictly correct: 1938 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1939 * should be: 1940 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1941 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1942 * but as the peer device might not be in switchdev mode it's not 1943 * possible. We use the fact that by default FW sets max vfs and max sfs 1944 * to the same value on both devices. If it needs to be changed in the future note 1945 * the peer miss group should also be created based on the number of 1946 * total vports of the peer (currently is also uses esw->total_vports). 1947 */ 1948 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1949 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS; 1950 1951 /* create the slow path fdb with encap set, so further table instances 1952 * can be created at run time while VFs are probed if the FW allows that. 1953 */ 1954 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1955 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1956 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1957 1958 ft_attr.flags = flags; 1959 ft_attr.max_fte = table_size; 1960 ft_attr.prio = FDB_SLOW_PATH; 1961 1962 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1963 if (IS_ERR(fdb)) { 1964 err = PTR_ERR(fdb); 1965 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1966 goto slow_fdb_err; 1967 } 1968 esw->fdb_table.offloads.slow_fdb = fdb; 1969 1970 /* Create empty TC-miss managed table. This allows plugging in following 1971 * priorities without directly exposing their level 0 table to 1972 * eswitch_offloads and passing it as miss_fdb to following call to 1973 * esw_chains_create(). 1974 */ 1975 memset(&ft_attr, 0, sizeof(ft_attr)); 1976 ft_attr.prio = FDB_TC_MISS; 1977 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 1978 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 1979 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 1980 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 1981 goto tc_miss_table_err; 1982 } 1983 1984 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 1985 if (err) { 1986 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1987 goto fdb_chains_err; 1988 } 1989 1990 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1991 if (err) 1992 goto send_vport_err; 1993 1994 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1995 if (err) 1996 goto send_vport_meta_err; 1997 1998 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); 1999 if (err) 2000 goto peer_miss_err; 2001 2002 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); 2003 if (err) 2004 goto miss_err; 2005 2006 kvfree(flow_group_in); 2007 return 0; 2008 2009 miss_err: 2010 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2011 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 2012 peer_miss_err: 2013 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 2014 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 2015 send_vport_meta_err: 2016 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 2017 send_vport_err: 2018 esw_chains_destroy(esw, esw_chains(esw)); 2019 fdb_chains_err: 2020 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 2021 tc_miss_table_err: 2022 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 2023 slow_fdb_err: 2024 /* Holds true only as long as DMFS is the default */ 2025 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 2026 ns_err: 2027 kvfree(flow_group_in); 2028 return err; 2029 } 2030 2031 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 2032 { 2033 if (!mlx5_eswitch_get_slow_fdb(esw)) 2034 return; 2035 2036 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 2037 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 2038 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 2039 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 2040 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 2041 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 2042 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2043 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 2044 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 2045 2046 esw_chains_destroy(esw, esw_chains(esw)); 2047 2048 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 2049 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 2050 /* Holds true only as long as DMFS is the default */ 2051 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 2052 MLX5_FLOW_STEERING_MODE_DMFS); 2053 } 2054 2055 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) 2056 { 2057 int nvports; 2058 2059 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 2060 if (mlx5e_tc_int_port_supported(esw)) 2061 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 2062 2063 return nvports; 2064 } 2065 2066 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 2067 { 2068 struct mlx5_flow_table_attr ft_attr = {}; 2069 struct mlx5_core_dev *dev = esw->dev; 2070 struct mlx5_flow_table *ft_offloads; 2071 struct mlx5_flow_namespace *ns; 2072 int err = 0; 2073 2074 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2075 if (!ns) { 2076 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2077 return -EOPNOTSUPP; 2078 } 2079 2080 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + 2081 MLX5_ESW_FT_OFFLOADS_DROP_RULE; 2082 ft_attr.prio = 1; 2083 2084 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 2085 if (IS_ERR(ft_offloads)) { 2086 err = PTR_ERR(ft_offloads); 2087 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 2088 return err; 2089 } 2090 2091 esw->offloads.ft_offloads = ft_offloads; 2092 return 0; 2093 } 2094 2095 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 2096 { 2097 struct mlx5_esw_offload *offloads = &esw->offloads; 2098 2099 mlx5_destroy_flow_table(offloads->ft_offloads); 2100 } 2101 2102 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 2103 { 2104 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2105 struct mlx5_flow_group *g; 2106 u32 *flow_group_in; 2107 int nvports; 2108 int err = 0; 2109 2110 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); 2111 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2112 if (!flow_group_in) 2113 return -ENOMEM; 2114 2115 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 2116 2117 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2118 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 2119 2120 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2121 2122 if (IS_ERR(g)) { 2123 err = PTR_ERR(g); 2124 esw_warn(esw->dev, "Failed to create vport rx group err %d\n", 2125 err); 2126 goto out; 2127 } 2128 2129 esw->offloads.vport_rx_group = g; 2130 out: 2131 kvfree(flow_group_in); 2132 return err; 2133 } 2134 2135 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 2136 { 2137 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 2138 } 2139 2140 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) 2141 { 2142 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 2143 * for the drop rule, which is placed at the end of the table. 2144 * So return the total of vport and int_port as rule index. 2145 */ 2146 return esw_get_nr_ft_offloads_steering_src_ports(esw); 2147 } 2148 2149 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) 2150 { 2151 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2152 struct mlx5_flow_group *g; 2153 u32 *flow_group_in; 2154 int flow_index; 2155 int err = 0; 2156 2157 flow_index = esw_create_vport_rx_drop_rule_index(esw); 2158 2159 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2160 if (!flow_group_in) 2161 return -ENOMEM; 2162 2163 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); 2164 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); 2165 2166 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2167 2168 if (IS_ERR(g)) { 2169 err = PTR_ERR(g); 2170 esw_warn(esw->dev, 2171 "Failed to create vport rx drop group err %d\n", err); 2172 goto out; 2173 } 2174 2175 esw->offloads.vport_rx_drop_group = g; 2176 out: 2177 kvfree(flow_group_in); 2178 return err; 2179 } 2180 2181 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) 2182 { 2183 if (esw->offloads.vport_rx_drop_group) 2184 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); 2185 } 2186 2187 void 2188 mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw, 2189 u16 vport, 2190 struct mlx5_flow_spec *spec) 2191 { 2192 void *misc; 2193 2194 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 2195 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 2196 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2197 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 2198 2199 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 2200 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2201 mlx5_eswitch_get_vport_metadata_mask()); 2202 2203 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 2204 } else { 2205 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 2206 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 2207 2208 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2209 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2210 2211 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2212 } 2213 } 2214 2215 struct mlx5_flow_handle * 2216 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 2217 struct mlx5_flow_destination *dest) 2218 { 2219 struct mlx5_flow_act flow_act = {0}; 2220 struct mlx5_flow_handle *flow_rule; 2221 struct mlx5_flow_spec *spec; 2222 2223 spec = kvzalloc_obj(*spec); 2224 if (!spec) { 2225 flow_rule = ERR_PTR(-ENOMEM); 2226 goto out; 2227 } 2228 2229 mlx5_esw_set_spec_source_port(esw, vport, spec); 2230 2231 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2232 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 2233 &flow_act, dest, 1); 2234 if (IS_ERR(flow_rule)) { 2235 esw_warn(esw->dev, 2236 "fs offloads: Failed to add vport rx rule err %pe\n", 2237 flow_rule); 2238 goto out; 2239 } 2240 2241 out: 2242 kvfree(spec); 2243 return flow_rule; 2244 } 2245 2246 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2247 { 2248 struct mlx5_flow_act flow_act = {}; 2249 struct mlx5_flow_handle *flow_rule; 2250 2251 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2252 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, 2253 &flow_act, NULL, 0); 2254 if (IS_ERR(flow_rule)) { 2255 esw_warn(esw->dev, 2256 "fs offloads: Failed to add vport rx drop rule err %pe\n", 2257 flow_rule); 2258 return PTR_ERR(flow_rule); 2259 } 2260 2261 esw->offloads.vport_rx_drop_rule = flow_rule; 2262 2263 return 0; 2264 } 2265 2266 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2267 { 2268 if (esw->offloads.vport_rx_drop_rule) 2269 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); 2270 } 2271 2272 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 2273 { 2274 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 2275 struct mlx5_core_dev *dev = esw->dev; 2276 struct mlx5_vport *vport; 2277 unsigned long i; 2278 2279 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2280 return -EOPNOTSUPP; 2281 2282 if (!mlx5_esw_is_fdb_created(esw)) 2283 return -EOPNOTSUPP; 2284 2285 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2286 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2287 mlx5_mode = MLX5_INLINE_MODE_NONE; 2288 goto out; 2289 case MLX5_CAP_INLINE_MODE_L2: 2290 mlx5_mode = MLX5_INLINE_MODE_L2; 2291 goto out; 2292 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2293 goto query_vports; 2294 } 2295 2296 query_vports: 2297 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2298 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2299 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2300 if (prev_mlx5_mode != mlx5_mode) 2301 return -EINVAL; 2302 prev_mlx5_mode = mlx5_mode; 2303 } 2304 2305 out: 2306 *mode = mlx5_mode; 2307 return 0; 2308 } 2309 2310 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2311 { 2312 struct mlx5_esw_offload *offloads = &esw->offloads; 2313 2314 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2315 return; 2316 2317 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2318 mlx5_destroy_flow_group(offloads->restore_group); 2319 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2320 } 2321 2322 static int esw_create_restore_table(struct mlx5_eswitch *esw) 2323 { 2324 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2325 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2326 struct mlx5_flow_table_attr ft_attr = {}; 2327 struct mlx5_core_dev *dev = esw->dev; 2328 struct mlx5_flow_namespace *ns; 2329 struct mlx5_modify_hdr *mod_hdr; 2330 void *match_criteria, *misc; 2331 struct mlx5_flow_table *ft; 2332 struct mlx5_flow_group *g; 2333 u32 *flow_group_in; 2334 int err = 0; 2335 2336 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2337 return 0; 2338 2339 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2340 if (!ns) { 2341 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2342 return -EOPNOTSUPP; 2343 } 2344 2345 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2346 if (!flow_group_in) { 2347 err = -ENOMEM; 2348 goto out_free; 2349 } 2350 2351 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2352 ft = mlx5_create_flow_table(ns, &ft_attr); 2353 if (IS_ERR(ft)) { 2354 err = PTR_ERR(ft); 2355 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2356 err); 2357 goto out_free; 2358 } 2359 2360 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2361 match_criteria); 2362 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2363 misc_parameters_2); 2364 2365 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2366 ESW_REG_C0_USER_DATA_METADATA_MASK); 2367 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2368 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2369 ft_attr.max_fte - 1); 2370 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2371 MLX5_MATCH_MISC_PARAMETERS_2); 2372 g = mlx5_create_flow_group(ft, flow_group_in); 2373 if (IS_ERR(g)) { 2374 err = PTR_ERR(g); 2375 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2376 err); 2377 goto err_group; 2378 } 2379 2380 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2381 MLX5_SET(copy_action_in, modact, src_field, 2382 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2383 MLX5_SET(copy_action_in, modact, dst_field, 2384 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2385 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2386 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2387 modact); 2388 if (IS_ERR(mod_hdr)) { 2389 err = PTR_ERR(mod_hdr); 2390 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2391 err); 2392 goto err_mod_hdr; 2393 } 2394 2395 esw->offloads.ft_offloads_restore = ft; 2396 esw->offloads.restore_group = g; 2397 esw->offloads.restore_copy_hdr_id = mod_hdr; 2398 2399 kvfree(flow_group_in); 2400 2401 return 0; 2402 2403 err_mod_hdr: 2404 mlx5_destroy_flow_group(g); 2405 err_group: 2406 mlx5_destroy_flow_table(ft); 2407 out_free: 2408 kvfree(flow_group_in); 2409 2410 return err; 2411 } 2412 2413 static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode) 2414 { 2415 mlx5_devcom_comp_lock(esw->dev->priv.hca_devcom_comp); 2416 if (esw->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV || 2417 mlx5_core_mp_enabled(esw->dev)) { 2418 esw->mode = mode; 2419 mlx5_rescan_drivers_locked(esw->dev); 2420 mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp); 2421 return; 2422 } 2423 2424 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; 2425 mlx5_rescan_drivers_locked(esw->dev); 2426 esw->mode = mode; 2427 esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; 2428 mlx5_rescan_drivers_locked(esw->dev); 2429 mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp); 2430 } 2431 2432 static void mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch *esw) 2433 { 2434 if (!esw->fdb_table.offloads.drop_root) 2435 return; 2436 2437 esw_debug(esw->dev, "Destroying FDB drop root table %#x fc %#x\n", 2438 esw->fdb_table.offloads.drop_root->id, 2439 esw->fdb_table.offloads.drop_root_fc->id); 2440 mlx5_del_flow_rules(esw->fdb_table.offloads.drop_root_rule); 2441 /* Don't free flow counter here, can be reused on a later activation */ 2442 mlx5_destroy_flow_table(esw->fdb_table.offloads.drop_root); 2443 esw->fdb_table.offloads.drop_root_rule = NULL; 2444 esw->fdb_table.offloads.drop_root = NULL; 2445 } 2446 2447 static int mlx5_esw_fdb_drop_create(struct mlx5_eswitch *esw) 2448 { 2449 struct mlx5_flow_destination drop_fc_dst = {}; 2450 struct mlx5_flow_table_attr ft_attr = {}; 2451 struct mlx5_flow_destination *dst = NULL; 2452 struct mlx5_core_dev *dev = esw->dev; 2453 struct mlx5_flow_namespace *root_ns; 2454 struct mlx5_flow_act flow_act = {}; 2455 struct mlx5_flow_handle *flow_rule; 2456 struct mlx5_flow_table *table; 2457 int err = 0, dst_num = 0; 2458 2459 if (esw->fdb_table.offloads.drop_root) 2460 return 0; 2461 2462 root_ns = esw->fdb_table.offloads.ns; 2463 2464 ft_attr.prio = FDB_DROP_ROOT; 2465 ft_attr.max_fte = 1; 2466 ft_attr.autogroup.max_num_groups = 1; 2467 table = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr); 2468 if (IS_ERR(table)) { 2469 esw_warn(dev, "Failed to create fdb drop root table, err %pe\n", 2470 table); 2471 return PTR_ERR(table); 2472 } 2473 2474 /* Drop FC reusable, create once on first deactivation of FDB */ 2475 if (!esw->fdb_table.offloads.drop_root_fc) { 2476 struct mlx5_fc *counter = mlx5_fc_create(dev, 0); 2477 2478 err = PTR_ERR_OR_ZERO(counter); 2479 if (err) 2480 esw_warn(esw->dev, "create fdb drop fc err %d\n", err); 2481 else 2482 esw->fdb_table.offloads.drop_root_fc = counter; 2483 } 2484 2485 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2486 2487 if (esw->fdb_table.offloads.drop_root_fc) { 2488 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 2489 drop_fc_dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2490 drop_fc_dst.counter = esw->fdb_table.offloads.drop_root_fc; 2491 dst = &drop_fc_dst; 2492 dst_num++; 2493 } 2494 2495 flow_rule = mlx5_add_flow_rules(table, NULL, &flow_act, dst, dst_num); 2496 err = PTR_ERR_OR_ZERO(flow_rule); 2497 if (err) { 2498 esw_warn(esw->dev, 2499 "fs offloads: Failed to add vport rx drop rule err %d\n", 2500 err); 2501 goto err_flow_rule; 2502 } 2503 2504 esw->fdb_table.offloads.drop_root = table; 2505 esw->fdb_table.offloads.drop_root_rule = flow_rule; 2506 esw_debug(esw->dev, "Created FDB drop root table %#x fc %#x\n", 2507 table->id, dst ? dst->counter->id : 0); 2508 return 0; 2509 2510 err_flow_rule: 2511 /* no need to free drop fc, esw_offloads_steering_cleanup will do it */ 2512 mlx5_destroy_flow_table(table); 2513 return err; 2514 } 2515 2516 static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw) 2517 { 2518 struct mlx5_vport *vport; 2519 unsigned long i; 2520 2521 mlx5_esw_fdb_drop_destroy(esw); 2522 mlx5_mpfs_enable(esw->dev); 2523 2524 mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { 2525 if (!vport->adjacent) 2526 continue; 2527 esw_debug(esw->dev, "Connecting vport %d to eswitch\n", 2528 vport->vport); 2529 mlx5_esw_adj_vport_modify(esw->dev, vport->vport, true); 2530 } 2531 2532 esw->offloads_inactive = false; 2533 esw_warn(esw->dev, "MPFS/FDB active\n"); 2534 } 2535 2536 static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw) 2537 { 2538 struct mlx5_vport *vport; 2539 unsigned long i; 2540 2541 mlx5_mpfs_disable(esw->dev); 2542 mlx5_esw_fdb_drop_create(esw); 2543 2544 mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { 2545 if (!vport->adjacent) 2546 continue; 2547 esw_debug(esw->dev, "Disconnecting vport %u from eswitch\n", 2548 vport->vport); 2549 2550 mlx5_esw_adj_vport_modify(esw->dev, vport->vport, false); 2551 } 2552 2553 esw->offloads_inactive = true; 2554 esw_warn(esw->dev, "MPFS/FDB inactive\n"); 2555 } 2556 2557 static int esw_offloads_start(struct mlx5_eswitch *esw, 2558 struct netlink_ext_ack *extack) 2559 { 2560 int err; 2561 2562 esw_mode_change(esw, MLX5_ESWITCH_OFFLOADS); 2563 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2564 if (err) { 2565 NL_SET_ERR_MSG_MOD(extack, 2566 "Failed setting eswitch to offloads"); 2567 esw_mode_change(esw, MLX5_ESWITCH_LEGACY); 2568 return err; 2569 } 2570 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2571 if (mlx5_eswitch_inline_mode_get(esw, 2572 &esw->offloads.inline_mode)) { 2573 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2574 NL_SET_ERR_MSG_MOD(extack, 2575 "Inline mode is different between vports"); 2576 } 2577 } 2578 return 0; 2579 } 2580 2581 void mlx5_esw_offloads_rep_remove(struct mlx5_eswitch *esw, 2582 const struct mlx5_vport *vport) 2583 { 2584 struct mlx5_eswitch_rep *rep = xa_load(&esw->offloads.vport_reps, 2585 vport->vport); 2586 2587 if (!rep) 2588 return; 2589 xa_erase(&esw->offloads.vport_reps, vport->vport); 2590 kfree(rep); 2591 } 2592 2593 int mlx5_esw_offloads_rep_add(struct mlx5_eswitch *esw, 2594 const struct mlx5_vport *vport) 2595 { 2596 struct mlx5_eswitch_rep *rep; 2597 int rep_type; 2598 int err; 2599 2600 rep = kzalloc_obj(*rep); 2601 if (!rep) 2602 return -ENOMEM; 2603 2604 rep->vport = vport->vport; 2605 rep->vport_index = vport->index; 2606 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2607 if (!esw->offloads.rep_ops[rep_type]) { 2608 atomic_set(&rep->rep_data[rep_type].state, 2609 REP_UNREGISTERED); 2610 continue; 2611 } 2612 /* Dynamic/delegated vports add their representors after 2613 * mlx5_eswitch_register_vport_reps, so mark them as registered 2614 * for them to be loaded later with the others. 2615 */ 2616 rep->esw = esw; 2617 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2618 } 2619 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2620 if (err) 2621 goto insert_err; 2622 2623 return 0; 2624 2625 insert_err: 2626 kfree(rep); 2627 return err; 2628 } 2629 2630 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2631 struct mlx5_eswitch_rep *rep) 2632 { 2633 xa_erase(&esw->offloads.vport_reps, rep->vport); 2634 kfree(rep); 2635 } 2636 2637 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2638 { 2639 struct mlx5_eswitch_rep *rep; 2640 unsigned long i; 2641 2642 mlx5_esw_for_each_rep(esw, i, rep) 2643 mlx5_esw_offloads_rep_cleanup(esw, rep); 2644 xa_destroy(&esw->offloads.vport_reps); 2645 } 2646 2647 static int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2648 { 2649 struct mlx5_vport *vport; 2650 unsigned long i; 2651 int err; 2652 2653 xa_init(&esw->offloads.vport_reps); 2654 2655 mlx5_esw_for_each_vport(esw, i, vport) { 2656 err = mlx5_esw_offloads_rep_add(esw, vport); 2657 if (err) 2658 goto err; 2659 } 2660 return 0; 2661 2662 err: 2663 esw_offloads_cleanup_reps(esw); 2664 return err; 2665 } 2666 2667 static int esw_port_metadata_set(struct devlink *devlink, u32 id, 2668 struct devlink_param_gset_ctx *ctx, 2669 struct netlink_ext_ack *extack) 2670 { 2671 struct mlx5_core_dev *dev = devlink_priv(devlink); 2672 struct mlx5_eswitch *esw = dev->priv.eswitch; 2673 int err = 0; 2674 2675 down_write(&esw->mode_lock); 2676 if (mlx5_esw_is_fdb_created(esw)) { 2677 err = -EBUSY; 2678 goto done; 2679 } 2680 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 2681 err = -EOPNOTSUPP; 2682 goto done; 2683 } 2684 if (ctx->val.vbool) 2685 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2686 else 2687 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2688 done: 2689 up_write(&esw->mode_lock); 2690 return err; 2691 } 2692 2693 static int esw_port_metadata_get(struct devlink *devlink, u32 id, 2694 struct devlink_param_gset_ctx *ctx, 2695 struct netlink_ext_ack *extack) 2696 { 2697 struct mlx5_core_dev *dev = devlink_priv(devlink); 2698 2699 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch); 2700 return 0; 2701 } 2702 2703 static int esw_port_metadata_validate(struct devlink *devlink, u32 id, 2704 union devlink_param_value val, 2705 struct netlink_ext_ack *extack) 2706 { 2707 struct mlx5_core_dev *dev = devlink_priv(devlink); 2708 u8 esw_mode; 2709 2710 esw_mode = mlx5_eswitch_mode(dev); 2711 if (esw_mode == MLX5_ESWITCH_OFFLOADS) { 2712 NL_SET_ERR_MSG_MOD(extack, 2713 "E-Switch must either disabled or non switchdev mode"); 2714 return -EBUSY; 2715 } 2716 return 0; 2717 } 2718 2719 static const struct devlink_param esw_devlink_params[] = { 2720 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA, 2721 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL, 2722 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 2723 esw_port_metadata_get, 2724 esw_port_metadata_set, 2725 esw_port_metadata_validate), 2726 }; 2727 2728 int esw_offloads_init(struct mlx5_eswitch *esw) 2729 { 2730 int err; 2731 2732 err = esw_offloads_init_reps(esw); 2733 if (err) 2734 return err; 2735 2736 if (MLX5_ESWITCH_MANAGER(esw->dev) && 2737 mlx5_esw_vport_match_metadata_supported(esw)) 2738 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2739 2740 err = devl_params_register(priv_to_devlink(esw->dev), 2741 esw_devlink_params, 2742 ARRAY_SIZE(esw_devlink_params)); 2743 if (err) 2744 goto err_params; 2745 2746 return 0; 2747 2748 err_params: 2749 esw_offloads_cleanup_reps(esw); 2750 return err; 2751 } 2752 2753 void esw_offloads_cleanup(struct mlx5_eswitch *esw) 2754 { 2755 devl_params_unregister(priv_to_devlink(esw->dev), 2756 esw_devlink_params, 2757 ARRAY_SIZE(esw_devlink_params)); 2758 esw_offloads_cleanup_reps(esw); 2759 } 2760 2761 static int __esw_offloads_load_rep(struct mlx5_eswitch *esw, 2762 struct mlx5_eswitch_rep *rep, u8 rep_type) 2763 { 2764 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2765 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) 2766 return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2767 2768 return 0; 2769 } 2770 2771 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2772 struct mlx5_eswitch_rep *rep, u8 rep_type) 2773 { 2774 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2775 REP_LOADED, REP_REGISTERED) == REP_LOADED) { 2776 if (rep_type == REP_ETH) 2777 __esw_offloads_unload_rep(esw, rep, REP_IB); 2778 esw->offloads.rep_ops[rep_type]->unload(rep); 2779 } 2780 } 2781 2782 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2783 { 2784 struct mlx5_eswitch_rep *rep; 2785 unsigned long i; 2786 2787 mlx5_esw_for_each_rep(esw, i, rep) 2788 __esw_offloads_unload_rep(esw, rep, rep_type); 2789 } 2790 2791 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2792 { 2793 struct mlx5_eswitch_rep *rep; 2794 int rep_type; 2795 int err; 2796 2797 rep = mlx5_eswitch_get_rep(esw, vport_num); 2798 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2799 err = __esw_offloads_load_rep(esw, rep, rep_type); 2800 if (err) 2801 goto err_reps; 2802 } 2803 2804 return 0; 2805 2806 err_reps: 2807 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2808 for (--rep_type; rep_type >= 0; rep_type--) 2809 __esw_offloads_unload_rep(esw, rep, rep_type); 2810 return err; 2811 } 2812 2813 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2814 { 2815 struct mlx5_eswitch_rep *rep; 2816 int rep_type; 2817 2818 rep = mlx5_eswitch_get_rep(esw, vport_num); 2819 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2820 __esw_offloads_unload_rep(esw, rep, rep_type); 2821 } 2822 2823 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2824 { 2825 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2826 return 0; 2827 2828 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport); 2829 } 2830 2831 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2832 { 2833 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2834 return; 2835 2836 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport); 2837 } 2838 2839 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport, 2840 struct mlx5_devlink_port *dl_port, 2841 u32 controller, u32 sfnum) 2842 { 2843 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum); 2844 } 2845 2846 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2847 { 2848 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport); 2849 } 2850 2851 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2852 { 2853 int err; 2854 2855 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2856 return 0; 2857 2858 err = mlx5_esw_offloads_devlink_port_register(esw, vport); 2859 if (err) 2860 return err; 2861 2862 err = mlx5_esw_offloads_rep_load(esw, vport->vport); 2863 if (err) 2864 goto load_err; 2865 return err; 2866 2867 load_err: 2868 mlx5_esw_offloads_devlink_port_unregister(vport); 2869 return err; 2870 } 2871 2872 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2873 { 2874 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2875 return; 2876 2877 mlx5_esw_offloads_rep_unload(esw, vport->vport); 2878 2879 mlx5_esw_offloads_devlink_port_unregister(vport); 2880 } 2881 2882 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2883 struct mlx5_core_dev *slave) 2884 { 2885 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2886 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2887 struct mlx5_flow_root_namespace *root; 2888 struct mlx5_flow_namespace *ns; 2889 int err; 2890 2891 MLX5_SET(set_flow_table_root_in, in, opcode, 2892 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2893 MLX5_SET(set_flow_table_root_in, in, table_type, 2894 FS_FT_FDB); 2895 2896 if (master) { 2897 ns = mlx5_get_flow_namespace(master, 2898 MLX5_FLOW_NAMESPACE_FDB); 2899 root = find_root(&ns->node); 2900 mutex_lock(&root->chain_lock); 2901 MLX5_SET(set_flow_table_root_in, in, 2902 table_eswitch_owner_vhca_id_valid, 1); 2903 MLX5_SET(set_flow_table_root_in, in, 2904 table_eswitch_owner_vhca_id, 2905 MLX5_CAP_GEN(master, vhca_id)); 2906 MLX5_SET(set_flow_table_root_in, in, table_id, 2907 root->root_ft->id); 2908 } else { 2909 ns = mlx5_get_flow_namespace(slave, 2910 MLX5_FLOW_NAMESPACE_FDB); 2911 root = find_root(&ns->node); 2912 mutex_lock(&root->chain_lock); 2913 MLX5_SET(set_flow_table_root_in, in, table_id, 2914 root->root_ft->id); 2915 } 2916 2917 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2918 mutex_unlock(&root->chain_lock); 2919 2920 return err; 2921 } 2922 2923 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 2924 struct mlx5_core_dev *slave, 2925 struct mlx5_vport *vport, 2926 struct mlx5_flow_table *acl) 2927 { 2928 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 2929 struct mlx5_flow_handle *flow_rule = NULL; 2930 struct mlx5_flow_destination dest = {}; 2931 struct mlx5_flow_act flow_act = {}; 2932 struct mlx5_flow_spec *spec; 2933 int err = 0; 2934 void *misc; 2935 2936 spec = kvzalloc_obj(*spec); 2937 if (!spec) 2938 return -ENOMEM; 2939 2940 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2941 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 2942 misc_parameters); 2943 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 2944 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index); 2945 2946 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2947 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2948 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 2949 source_eswitch_owner_vhca_id); 2950 2951 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2952 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 2953 dest.vport.num = slave->priv.eswitch->manager_vport; 2954 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 2955 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 2956 2957 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 2958 &dest, 1); 2959 if (IS_ERR(flow_rule)) { 2960 err = PTR_ERR(flow_rule); 2961 } else { 2962 err = xa_insert(&vport->egress.offloads.bounce_rules, 2963 slave_index, flow_rule, GFP_KERNEL); 2964 if (err) 2965 mlx5_del_flow_rules(flow_rule); 2966 } 2967 2968 kvfree(spec); 2969 return err; 2970 } 2971 2972 static int esw_master_egress_create_resources(struct mlx5_eswitch *esw, 2973 struct mlx5_flow_namespace *egress_ns, 2974 struct mlx5_vport *vport, size_t count) 2975 { 2976 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2977 struct mlx5_flow_table_attr ft_attr = { 2978 .max_fte = count, .prio = 0, .level = 0, 2979 }; 2980 struct mlx5_flow_table *acl; 2981 struct mlx5_flow_group *g; 2982 void *match_criteria; 2983 u32 *flow_group_in; 2984 int err; 2985 2986 if (vport->egress.acl) 2987 return 0; 2988 2989 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2990 if (!flow_group_in) 2991 return -ENOMEM; 2992 2993 if (vport->vport || mlx5_core_is_ecpf(esw->dev)) 2994 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; 2995 2996 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 2997 if (IS_ERR(acl)) { 2998 err = PTR_ERR(acl); 2999 goto out; 3000 } 3001 3002 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 3003 match_criteria); 3004 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 3005 misc_parameters.source_port); 3006 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 3007 misc_parameters.source_eswitch_owner_vhca_id); 3008 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 3009 MLX5_MATCH_MISC_PARAMETERS); 3010 3011 MLX5_SET(create_flow_group_in, flow_group_in, 3012 source_eswitch_owner_vhca_id_valid, 1); 3013 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 3014 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count); 3015 3016 g = mlx5_create_flow_group(acl, flow_group_in); 3017 if (IS_ERR(g)) { 3018 err = PTR_ERR(g); 3019 goto err_group; 3020 } 3021 3022 vport->egress.acl = acl; 3023 vport->egress.offloads.bounce_grp = g; 3024 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB; 3025 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC); 3026 3027 kvfree(flow_group_in); 3028 3029 return 0; 3030 3031 err_group: 3032 mlx5_destroy_flow_table(acl); 3033 out: 3034 kvfree(flow_group_in); 3035 return err; 3036 } 3037 3038 static void esw_master_egress_destroy_resources(struct mlx5_vport *vport) 3039 { 3040 if (!xa_empty(&vport->egress.offloads.bounce_rules)) 3041 return; 3042 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp); 3043 vport->egress.offloads.bounce_grp = NULL; 3044 mlx5_destroy_flow_table(vport->egress.acl); 3045 vport->egress.acl = NULL; 3046 } 3047 3048 static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 3049 struct mlx5_core_dev *slave, size_t count) 3050 { 3051 struct mlx5_eswitch *esw = master->priv.eswitch; 3052 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 3053 struct mlx5_flow_namespace *egress_ns; 3054 struct mlx5_vport *vport; 3055 int err; 3056 3057 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3058 if (IS_ERR(vport)) 3059 return PTR_ERR(vport); 3060 3061 egress_ns = mlx5_get_flow_vport_namespace(master, 3062 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 3063 vport->index); 3064 if (!egress_ns) 3065 return -EINVAL; 3066 3067 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB) 3068 return 0; 3069 3070 err = esw_master_egress_create_resources(esw, egress_ns, vport, count); 3071 if (err) 3072 return err; 3073 3074 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index)) 3075 return -EINVAL; 3076 3077 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl); 3078 if (err) 3079 goto err_rule; 3080 3081 return 0; 3082 3083 err_rule: 3084 esw_master_egress_destroy_resources(vport); 3085 return err; 3086 } 3087 3088 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev, 3089 struct mlx5_core_dev *slave_dev) 3090 { 3091 struct mlx5_vport *vport; 3092 3093 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 3094 dev->priv.eswitch->manager_vport); 3095 3096 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id)); 3097 3098 if (xa_empty(&vport->egress.offloads.bounce_rules)) { 3099 esw_acl_egress_ofld_cleanup(vport); 3100 xa_destroy(&vport->egress.offloads.bounce_rules); 3101 } 3102 } 3103 3104 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw, 3105 struct mlx5_eswitch *slave_esw, int max_slaves) 3106 { 3107 int err; 3108 3109 err = esw_set_slave_root_fdb(master_esw->dev, 3110 slave_esw->dev); 3111 if (err) 3112 return err; 3113 3114 err = esw_set_master_egress_rule(master_esw->dev, 3115 slave_esw->dev, max_slaves); 3116 if (err) 3117 goto err_acl; 3118 3119 return err; 3120 3121 err_acl: 3122 esw_set_slave_root_fdb(NULL, slave_esw->dev); 3123 return err; 3124 } 3125 3126 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw, 3127 struct mlx5_eswitch *slave_esw) 3128 { 3129 esw_set_slave_root_fdb(NULL, slave_esw->dev); 3130 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev); 3131 } 3132 3133 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 3134 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 3135 3136 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw, 3137 struct mlx5_eswitch *peer_esw) 3138 { 3139 const struct mlx5_eswitch_rep_ops *ops; 3140 struct mlx5_eswitch_rep *rep; 3141 unsigned long i; 3142 u8 rep_type; 3143 3144 mlx5_esw_for_each_rep(esw, i, rep) { 3145 rep_type = NUM_REP_TYPES; 3146 while (rep_type--) { 3147 ops = esw->offloads.rep_ops[rep_type]; 3148 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3149 ops->event) 3150 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw); 3151 } 3152 } 3153 } 3154 3155 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw, 3156 struct mlx5_eswitch *peer_esw) 3157 { 3158 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 3159 mlx5e_tc_clean_fdb_peer_flows(esw); 3160 #endif 3161 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw); 3162 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev); 3163 } 3164 3165 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 3166 struct mlx5_eswitch *peer_esw) 3167 { 3168 const struct mlx5_eswitch_rep_ops *ops; 3169 struct mlx5_eswitch_rep *rep; 3170 unsigned long i; 3171 u8 rep_type; 3172 int err; 3173 3174 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 3175 if (err) 3176 return err; 3177 3178 mlx5_esw_for_each_rep(esw, i, rep) { 3179 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 3180 ops = esw->offloads.rep_ops[rep_type]; 3181 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3182 ops->event) { 3183 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 3184 if (err) 3185 goto err_out; 3186 } 3187 } 3188 } 3189 3190 return 0; 3191 3192 err_out: 3193 mlx5_esw_offloads_unpair(esw, peer_esw); 3194 return err; 3195 } 3196 3197 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 3198 struct mlx5_eswitch *peer_esw, 3199 bool pair) 3200 { 3201 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 3202 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 3203 struct mlx5_flow_root_namespace *peer_ns; 3204 struct mlx5_flow_root_namespace *ns; 3205 int err; 3206 3207 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 3208 ns = esw->dev->priv.steering->fdb_root_ns; 3209 3210 if (pair) { 3211 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id); 3212 if (err) 3213 return err; 3214 3215 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id); 3216 if (err) { 3217 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 3218 return err; 3219 } 3220 } else { 3221 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 3222 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id); 3223 } 3224 3225 return 0; 3226 } 3227 3228 static int mlx5_esw_offloads_devcom_event(int event, 3229 void *my_data, 3230 void *event_data) 3231 { 3232 struct mlx5_eswitch *esw = my_data; 3233 struct mlx5_eswitch *peer_esw = event_data; 3234 u16 esw_i, peer_esw_i; 3235 bool esw_paired; 3236 int err; 3237 3238 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 3239 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id); 3240 esw_paired = !!xa_load(&esw->paired, peer_esw_i); 3241 3242 switch (event) { 3243 case ESW_OFFLOADS_DEVCOM_PAIR: 3244 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 3245 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 3246 break; 3247 3248 if (esw_paired) 3249 break; 3250 3251 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 3252 if (err) 3253 goto err_out; 3254 3255 err = mlx5_esw_offloads_pair(esw, peer_esw); 3256 if (err) 3257 goto err_peer; 3258 3259 err = mlx5_esw_offloads_pair(peer_esw, esw); 3260 if (err) 3261 goto err_pair; 3262 3263 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL); 3264 if (err) 3265 goto err_xa; 3266 3267 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL); 3268 if (err) 3269 goto err_peer_xa; 3270 3271 esw->num_peers++; 3272 peer_esw->num_peers++; 3273 mlx5_devcom_comp_set_ready(esw->devcom, true); 3274 break; 3275 3276 case ESW_OFFLOADS_DEVCOM_UNPAIR: 3277 if (!esw_paired) 3278 break; 3279 3280 peer_esw->num_peers--; 3281 esw->num_peers--; 3282 if (!esw->num_peers && !peer_esw->num_peers) 3283 mlx5_devcom_comp_set_ready(esw->devcom, false); 3284 xa_erase(&peer_esw->paired, esw_i); 3285 xa_erase(&esw->paired, peer_esw_i); 3286 mlx5_esw_offloads_unpair(peer_esw, esw); 3287 mlx5_esw_offloads_unpair(esw, peer_esw); 3288 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3289 break; 3290 } 3291 3292 return 0; 3293 3294 err_peer_xa: 3295 xa_erase(&esw->paired, peer_esw_i); 3296 err_xa: 3297 mlx5_esw_offloads_unpair(peer_esw, esw); 3298 err_pair: 3299 mlx5_esw_offloads_unpair(esw, peer_esw); 3300 err_peer: 3301 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3302 err_out: 3303 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 3304 event, err); 3305 return err; 3306 } 3307 3308 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, 3309 const struct mlx5_devcom_match_attr *attr) 3310 { 3311 int i; 3312 3313 for (i = 0; i < MLX5_MAX_PORTS; i++) 3314 INIT_LIST_HEAD(&esw->offloads.peer_flows[i]); 3315 mutex_init(&esw->offloads.peer_mutex); 3316 3317 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 3318 return; 3319 3320 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) && 3321 !mlx5_lag_is_supported(esw->dev)) 3322 return; 3323 3324 xa_init(&esw->paired); 3325 xa_init(&esw->fdb_table.offloads.peer_miss_rules); 3326 esw->num_peers = 0; 3327 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, 3328 MLX5_DEVCOM_ESW_OFFLOADS, 3329 attr, 3330 mlx5_esw_offloads_devcom_event, 3331 esw); 3332 if (!esw->devcom) 3333 return; 3334 3335 mlx5_devcom_send_event(esw->devcom, 3336 ESW_OFFLOADS_DEVCOM_PAIR, 3337 ESW_OFFLOADS_DEVCOM_UNPAIR, 3338 esw); 3339 } 3340 3341 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 3342 { 3343 if (!esw->devcom) 3344 return; 3345 3346 mlx5_devcom_send_event(esw->devcom, 3347 ESW_OFFLOADS_DEVCOM_UNPAIR, 3348 ESW_OFFLOADS_DEVCOM_UNPAIR, 3349 esw); 3350 3351 mlx5_devcom_unregister_component(esw->devcom); 3352 xa_destroy(&esw->paired); 3353 xa_destroy(&esw->fdb_table.offloads.peer_miss_rules); 3354 esw->devcom = NULL; 3355 } 3356 3357 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) 3358 { 3359 return mlx5_devcom_comp_is_ready(esw->devcom); 3360 } 3361 3362 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 3363 { 3364 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 3365 return false; 3366 3367 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 3368 MLX5_FDB_TO_VPORT_REG_C_0)) 3369 return false; 3370 3371 return true; 3372 } 3373 3374 #define MLX5_ESW_METADATA_RSVD_UPLINK 1 3375 3376 /* Share the same metadata for uplink's. This is fine because: 3377 * (a) In shared FDB mode (LAG) both uplink's are treated the 3378 * same and tagged with the same metadata. 3379 * (b) In non shared FDB mode, packets from physical port0 3380 * cannot hit eswitch of PF1 and vice versa. 3381 */ 3382 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 3383 { 3384 return MLX5_ESW_METADATA_RSVD_UPLINK; 3385 } 3386 3387 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 3388 { 3389 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 3390 /* Reserve 0xf for internal port offload */ 3391 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 3392 u32 pf_num; 3393 int id; 3394 3395 /* Only 4 bits of pf_num */ 3396 pf_num = mlx5_get_dev_index(esw->dev); 3397 if (pf_num > max_pf_num) 3398 return 0; 3399 3400 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 3401 /* Use only non-zero vport_id (2-4095) for all PF's */ 3402 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 3403 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 3404 vport_end_ida, GFP_KERNEL); 3405 if (id < 0) 3406 return 0; 3407 id = (pf_num << ESW_VPORT_BITS) | id; 3408 return id; 3409 } 3410 3411 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 3412 { 3413 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 3414 3415 /* Metadata contains only 12 bits of actual ida id */ 3416 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 3417 } 3418 3419 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 3420 struct mlx5_vport *vport) 3421 { 3422 if (vport->vport == MLX5_VPORT_UPLINK) 3423 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 3424 else 3425 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 3426 3427 vport->metadata = vport->default_metadata; 3428 return vport->metadata ? 0 : -ENOSPC; 3429 } 3430 3431 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 3432 struct mlx5_vport *vport) 3433 { 3434 if (!vport->default_metadata) 3435 return; 3436 3437 if (vport->vport == MLX5_VPORT_UPLINK) 3438 return; 3439 3440 WARN_ON(vport->metadata != vport->default_metadata); 3441 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 3442 } 3443 3444 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 3445 { 3446 struct mlx5_vport *vport; 3447 unsigned long i; 3448 3449 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3450 return; 3451 3452 mlx5_esw_for_each_vport(esw, i, vport) 3453 esw_offloads_vport_metadata_cleanup(esw, vport); 3454 } 3455 3456 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 3457 { 3458 struct mlx5_vport *vport; 3459 unsigned long i; 3460 int err; 3461 3462 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3463 return 0; 3464 3465 mlx5_esw_for_each_vport(esw, i, vport) { 3466 err = esw_offloads_vport_metadata_setup(esw, vport); 3467 if (err) 3468 goto metadata_err; 3469 } 3470 3471 return 0; 3472 3473 metadata_err: 3474 esw_offloads_metadata_uninit(esw); 3475 return err; 3476 } 3477 3478 int 3479 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 3480 struct mlx5_vport *vport) 3481 { 3482 int err; 3483 3484 err = esw_acl_ingress_ofld_setup(esw, vport); 3485 if (err) 3486 return err; 3487 3488 err = esw_acl_egress_ofld_setup(esw, vport); 3489 if (err) 3490 goto egress_err; 3491 3492 return 0; 3493 3494 egress_err: 3495 esw_acl_ingress_ofld_cleanup(esw, vport); 3496 return err; 3497 } 3498 3499 void 3500 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 3501 struct mlx5_vport *vport) 3502 { 3503 esw_acl_egress_ofld_cleanup(vport); 3504 esw_acl_ingress_ofld_cleanup(esw, vport); 3505 } 3506 3507 static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw) 3508 { 3509 struct mlx5_vport *uplink, *manager; 3510 int ret; 3511 3512 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3513 if (IS_ERR(uplink)) 3514 return PTR_ERR(uplink); 3515 3516 ret = esw_vport_create_offloads_acl_tables(esw, uplink); 3517 if (ret) 3518 return ret; 3519 3520 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3521 if (IS_ERR(manager)) { 3522 ret = PTR_ERR(manager); 3523 goto err_manager; 3524 } 3525 3526 ret = esw_vport_create_offloads_acl_tables(esw, manager); 3527 if (ret) 3528 goto err_manager; 3529 3530 return 0; 3531 3532 err_manager: 3533 esw_vport_destroy_offloads_acl_tables(esw, uplink); 3534 return ret; 3535 } 3536 3537 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw) 3538 { 3539 struct mlx5_vport *vport; 3540 3541 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3542 if (!IS_ERR(vport)) 3543 esw_vport_destroy_offloads_acl_tables(esw, vport); 3544 3545 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3546 if (!IS_ERR(vport)) 3547 esw_vport_destroy_offloads_acl_tables(esw, vport); 3548 } 3549 3550 int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw) 3551 { 3552 struct mlx5_eswitch_rep *rep; 3553 unsigned long i; 3554 int ret; 3555 3556 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3557 return 0; 3558 3559 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3560 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3561 return 0; 3562 3563 ret = __esw_offloads_load_rep(esw, rep, REP_IB); 3564 if (ret) 3565 return ret; 3566 3567 mlx5_esw_for_each_rep(esw, i, rep) { 3568 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3569 __esw_offloads_load_rep(esw, rep, REP_IB); 3570 } 3571 3572 return 0; 3573 } 3574 3575 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3576 { 3577 struct mlx5_esw_indir_table *indir; 3578 int err; 3579 3580 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3581 mutex_init(&esw->fdb_table.offloads.vports.lock); 3582 hash_init(esw->fdb_table.offloads.vports.table); 3583 atomic64_set(&esw->user_count, 0); 3584 3585 indir = mlx5_esw_indir_table_init(); 3586 if (IS_ERR(indir)) { 3587 err = PTR_ERR(indir); 3588 goto create_indir_err; 3589 } 3590 esw->fdb_table.offloads.indir = indir; 3591 3592 err = esw_create_offloads_acl_tables(esw); 3593 if (err) 3594 goto create_acl_err; 3595 3596 err = esw_create_offloads_table(esw); 3597 if (err) 3598 goto create_offloads_err; 3599 3600 err = esw_create_restore_table(esw); 3601 if (err) 3602 goto create_restore_err; 3603 3604 err = esw_create_offloads_fdb_tables(esw); 3605 if (err) 3606 goto create_fdb_err; 3607 3608 err = esw_create_vport_rx_group(esw); 3609 if (err) 3610 goto create_fg_err; 3611 3612 err = esw_create_vport_rx_drop_group(esw); 3613 if (err) 3614 goto create_rx_drop_fg_err; 3615 3616 err = esw_create_vport_rx_drop_rule(esw); 3617 if (err) 3618 goto create_rx_drop_rule_err; 3619 3620 return 0; 3621 3622 create_rx_drop_rule_err: 3623 esw_destroy_vport_rx_drop_group(esw); 3624 create_rx_drop_fg_err: 3625 esw_destroy_vport_rx_group(esw); 3626 create_fg_err: 3627 esw_destroy_offloads_fdb_tables(esw); 3628 create_fdb_err: 3629 esw_destroy_restore_table(esw); 3630 create_restore_err: 3631 esw_destroy_offloads_table(esw); 3632 create_offloads_err: 3633 esw_destroy_offloads_acl_tables(esw); 3634 create_acl_err: 3635 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3636 create_indir_err: 3637 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3638 return err; 3639 } 3640 3641 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 3642 { 3643 mlx5_esw_fdb_drop_destroy(esw); 3644 if (esw->fdb_table.offloads.drop_root_fc) 3645 mlx5_fc_destroy(esw->dev, esw->fdb_table.offloads.drop_root_fc); 3646 esw->fdb_table.offloads.drop_root_fc = NULL; 3647 esw_destroy_vport_rx_drop_rule(esw); 3648 esw_destroy_vport_rx_drop_group(esw); 3649 esw_destroy_vport_rx_group(esw); 3650 esw_destroy_offloads_fdb_tables(esw); 3651 esw_destroy_restore_table(esw); 3652 esw_destroy_offloads_table(esw); 3653 esw_destroy_offloads_acl_tables(esw); 3654 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3655 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3656 } 3657 3658 static void 3659 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, int work_gen, 3660 const u32 *out) 3661 { 3662 struct devlink *devlink; 3663 bool host_pf_disabled; 3664 u16 new_num_vfs; 3665 3666 devlink = priv_to_devlink(esw->dev); 3667 devl_lock(devlink); 3668 3669 /* Stale work from one or more mode changes ago. Bail out. */ 3670 if (work_gen != atomic_read(&esw->esw_funcs.generation)) 3671 goto unlock; 3672 3673 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 3674 host_params_context.host_num_of_vfs); 3675 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 3676 host_params_context.host_pf_disabled); 3677 3678 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 3679 goto unlock; 3680 3681 /* Number of VFs can only change from "0 to x" or "x to 0". */ 3682 if (esw->esw_funcs.num_vfs > 0) { 3683 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 3684 } else { 3685 int err; 3686 3687 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 3688 MLX5_VPORT_UC_ADDR_CHANGE); 3689 if (err) { 3690 devl_unlock(devlink); 3691 return; 3692 } 3693 } 3694 esw->esw_funcs.num_vfs = new_num_vfs; 3695 unlock: 3696 devl_unlock(devlink); 3697 } 3698 3699 static void esw_functions_changed_event_handler(struct work_struct *work) 3700 { 3701 struct mlx5_host_work *host_work; 3702 struct mlx5_eswitch *esw; 3703 const u32 *out; 3704 3705 host_work = container_of(work, struct mlx5_host_work, work); 3706 esw = host_work->esw; 3707 3708 out = mlx5_esw_query_functions(esw->dev); 3709 if (IS_ERR(out)) 3710 goto out; 3711 3712 esw_vfs_changed_event_handler(esw, host_work->work_gen, out); 3713 kvfree(out); 3714 out: 3715 kfree(host_work); 3716 } 3717 3718 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 3719 { 3720 struct mlx5_esw_functions *esw_funcs; 3721 struct mlx5_host_work *host_work; 3722 struct mlx5_eswitch *esw; 3723 3724 host_work = kzalloc_obj(*host_work, GFP_ATOMIC); 3725 if (!host_work) 3726 return NOTIFY_DONE; 3727 3728 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 3729 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 3730 3731 host_work->esw = esw; 3732 host_work->work_gen = atomic_read(&esw_funcs->generation); 3733 3734 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 3735 queue_work(esw->work_queue, &host_work->work); 3736 3737 return NOTIFY_OK; 3738 } 3739 3740 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 3741 { 3742 const u32 *query_host_out; 3743 3744 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3745 return 0; 3746 3747 query_host_out = mlx5_esw_query_functions(esw->dev); 3748 if (IS_ERR(query_host_out)) 3749 return PTR_ERR(query_host_out); 3750 3751 /* Mark non local controller with non zero controller number. */ 3752 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 3753 host_params_context.host_number); 3754 kvfree(query_host_out); 3755 return 0; 3756 } 3757 3758 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 3759 { 3760 /* Local controller is always valid */ 3761 if (controller == 0) 3762 return true; 3763 3764 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3765 return false; 3766 3767 /* External host number starts with zero in device */ 3768 return (controller == esw->offloads.host_number + 1); 3769 } 3770 3771 int esw_offloads_enable(struct mlx5_eswitch *esw) 3772 { 3773 u8 mapping_id[MLX5_SW_IMAGE_GUID_MAX_BYTES]; 3774 struct mapping_ctx *reg_c0_obj_pool; 3775 struct mlx5_vport *vport; 3776 unsigned long i; 3777 u8 id_len; 3778 int err; 3779 3780 mutex_init(&esw->offloads.termtbl_mutex); 3781 mlx5_esw_adjacent_vhcas_setup(esw); 3782 3783 err = mlx5_rdma_enable_roce(esw->dev); 3784 if (err) 3785 goto err_roce; 3786 3787 err = mlx5_esw_host_number_init(esw); 3788 if (err) 3789 goto err_metadata; 3790 3791 err = esw_offloads_metadata_init(esw); 3792 if (err) 3793 goto err_metadata; 3794 3795 err = esw_set_passing_vport_metadata(esw, true); 3796 if (err) 3797 goto err_vport_metadata; 3798 3799 mlx5_query_nic_sw_system_image_guid(esw->dev, mapping_id, &id_len); 3800 3801 reg_c0_obj_pool = mapping_create_for_id(mapping_id, id_len, 3802 MAPPING_TYPE_CHAIN, 3803 sizeof(struct mlx5_mapped_obj), 3804 ESW_REG_C0_USER_DATA_METADATA_MASK, 3805 true); 3806 3807 if (IS_ERR(reg_c0_obj_pool)) { 3808 err = PTR_ERR(reg_c0_obj_pool); 3809 goto err_pool; 3810 } 3811 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 3812 3813 err = esw_offloads_steering_init(esw); 3814 if (err) 3815 goto err_steering_init; 3816 3817 if (esw->offloads_inactive) 3818 mlx5_esw_fdb_inactive(esw); 3819 else 3820 mlx5_esw_fdb_active(esw); 3821 3822 /* Representor will control the vport link state */ 3823 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 3824 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3825 if (mlx5_core_ec_sriov_enabled(esw->dev)) 3826 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) 3827 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3828 3829 /* Uplink vport rep must load first. */ 3830 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3831 if (err) 3832 goto err_uplink; 3833 3834 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 3835 if (err) 3836 goto err_vports; 3837 3838 return 0; 3839 3840 err_vports: 3841 /* rollback to legacy, indicates don't unregister the uplink netdev */ 3842 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; 3843 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3844 err_uplink: 3845 esw_offloads_steering_cleanup(esw); 3846 err_steering_init: 3847 mapping_destroy(reg_c0_obj_pool); 3848 err_pool: 3849 esw_set_passing_vport_metadata(esw, false); 3850 err_vport_metadata: 3851 esw_offloads_metadata_uninit(esw); 3852 err_metadata: 3853 mlx5_rdma_disable_roce(esw->dev); 3854 err_roce: 3855 mlx5_esw_adjacent_vhcas_cleanup(esw); 3856 mutex_destroy(&esw->offloads.termtbl_mutex); 3857 return err; 3858 } 3859 3860 static int esw_offloads_stop(struct mlx5_eswitch *esw, 3861 struct netlink_ext_ack *extack) 3862 { 3863 int err; 3864 3865 esw_mode_change(esw, MLX5_ESWITCH_LEGACY); 3866 3867 /* If changing from switchdev to legacy mode without sriov enabled, 3868 * no need to create legacy fdb. 3869 */ 3870 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev)) 3871 return 0; 3872 3873 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3874 if (err) 3875 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 3876 3877 return err; 3878 } 3879 3880 void esw_offloads_disable(struct mlx5_eswitch *esw) 3881 { 3882 mlx5_eswitch_disable_pf_vf_vports(esw); 3883 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3884 esw_set_passing_vport_metadata(esw, false); 3885 esw_offloads_steering_cleanup(esw); 3886 mapping_destroy(esw->offloads.reg_c0_obj_pool); 3887 esw_offloads_metadata_uninit(esw); 3888 mlx5_rdma_disable_roce(esw->dev); 3889 mlx5_esw_adjacent_vhcas_cleanup(esw); 3890 /* must be done after vhcas cleanup to avoid adjacent vports connect */ 3891 if (esw->offloads_inactive) 3892 mlx5_esw_fdb_active(esw); /* legacy mode always active */ 3893 mutex_destroy(&esw->offloads.termtbl_mutex); 3894 } 3895 3896 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 3897 { 3898 switch (mode) { 3899 case DEVLINK_ESWITCH_MODE_LEGACY: 3900 *mlx5_mode = MLX5_ESWITCH_LEGACY; 3901 break; 3902 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 3903 case DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE: 3904 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 3905 break; 3906 default: 3907 return -EINVAL; 3908 } 3909 3910 return 0; 3911 } 3912 3913 static int esw_mode_to_devlink(struct mlx5_eswitch *esw, u16 *mode) 3914 { 3915 switch (esw->mode) { 3916 case MLX5_ESWITCH_LEGACY: 3917 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 3918 break; 3919 case MLX5_ESWITCH_OFFLOADS: 3920 if (esw->offloads_inactive) 3921 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE; 3922 else 3923 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 3924 break; 3925 default: 3926 return -EINVAL; 3927 } 3928 3929 return 0; 3930 } 3931 3932 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 3933 { 3934 switch (mode) { 3935 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 3936 *mlx5_mode = MLX5_INLINE_MODE_NONE; 3937 break; 3938 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 3939 *mlx5_mode = MLX5_INLINE_MODE_L2; 3940 break; 3941 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 3942 *mlx5_mode = MLX5_INLINE_MODE_IP; 3943 break; 3944 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 3945 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 3946 break; 3947 default: 3948 return -EINVAL; 3949 } 3950 3951 return 0; 3952 } 3953 3954 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 3955 { 3956 switch (mlx5_mode) { 3957 case MLX5_INLINE_MODE_NONE: 3958 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 3959 break; 3960 case MLX5_INLINE_MODE_L2: 3961 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 3962 break; 3963 case MLX5_INLINE_MODE_IP: 3964 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 3965 break; 3966 case MLX5_INLINE_MODE_TCP_UDP: 3967 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 3968 break; 3969 default: 3970 return -EINVAL; 3971 } 3972 3973 return 0; 3974 } 3975 3976 int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev) 3977 { 3978 struct mlx5_eswitch *esw = dev->priv.eswitch; 3979 int err; 3980 3981 if (!mlx5_esw_allowed(esw)) 3982 return 0; 3983 3984 /* Take TC into account */ 3985 err = mlx5_esw_try_lock(esw); 3986 if (err < 0) 3987 return err; 3988 3989 esw->offloads.num_block_mode++; 3990 mlx5_esw_unlock(esw); 3991 return 0; 3992 } 3993 3994 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev) 3995 { 3996 struct mlx5_eswitch *esw = dev->priv.eswitch; 3997 3998 if (!mlx5_esw_allowed(esw)) 3999 return; 4000 4001 down_write(&esw->mode_lock); 4002 esw->offloads.num_block_mode--; 4003 up_write(&esw->mode_lock); 4004 } 4005 4006 /* Returns false only when uplink netdev exists and its netns is different from 4007 * devlink's netns. True for all others so entering switchdev mode is allowed. 4008 */ 4009 static bool mlx5_devlink_netdev_netns_immutable_set(struct devlink *devlink, 4010 bool immutable) 4011 { 4012 struct mlx5_core_dev *mdev = devlink_priv(devlink); 4013 struct net_device *netdev; 4014 bool ret; 4015 4016 netdev = mlx5_uplink_netdev_get(mdev); 4017 if (!netdev) 4018 return true; 4019 4020 rtnl_lock(); 4021 netdev->netns_immutable = immutable; 4022 ret = net_eq(dev_net(netdev), devlink_net(devlink)); 4023 rtnl_unlock(); 4024 4025 mlx5_uplink_netdev_put(mdev, netdev); 4026 return ret; 4027 } 4028 4029 /* Returns true when only changing between active and inactive switchdev mode */ 4030 static bool mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch *esw, 4031 u16 devlink_mode) 4032 { 4033 /* current mode is not switchdev */ 4034 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 4035 return false; 4036 4037 /* new mode is not switchdev */ 4038 if (devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV && 4039 devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE) 4040 return false; 4041 4042 /* already inactive: no change in current state */ 4043 if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE && 4044 esw->offloads_inactive) 4045 return false; 4046 4047 /* already active: no change in current state */ 4048 if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && 4049 !esw->offloads_inactive) 4050 return false; 4051 4052 down_write(&esw->mode_lock); 4053 esw->offloads_inactive = !esw->offloads_inactive; 4054 esw->eswitch_operation_in_progress = true; 4055 up_write(&esw->mode_lock); 4056 4057 if (esw->offloads_inactive) 4058 mlx5_esw_fdb_inactive(esw); 4059 else 4060 mlx5_esw_fdb_active(esw); 4061 4062 down_write(&esw->mode_lock); 4063 esw->eswitch_operation_in_progress = false; 4064 up_write(&esw->mode_lock); 4065 return true; 4066 } 4067 4068 #define MLX5_ESW_HOLD_TIMEOUT_MS 7000 4069 #define MLX5_ESW_HOLD_RETRY_DELAY_MS 500 4070 4071 void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev) 4072 { 4073 unsigned long timeout; 4074 bool hold_esw = true; 4075 4076 /* Wait for any concurrent eswitch mode transition to complete. */ 4077 if (!mlx5_esw_hold(dev)) { 4078 timeout = jiffies + msecs_to_jiffies(MLX5_ESW_HOLD_TIMEOUT_MS); 4079 while (!mlx5_esw_hold(dev)) { 4080 if (!time_before(jiffies, timeout)) { 4081 hold_esw = false; 4082 break; 4083 } 4084 msleep(MLX5_ESW_HOLD_RETRY_DELAY_MS); 4085 } 4086 } 4087 if (hold_esw) { 4088 if (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS) 4089 mlx5_core_reps_aux_devs_remove(dev); 4090 mlx5_esw_release(dev); 4091 } 4092 } 4093 4094 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 4095 struct netlink_ext_ack *extack) 4096 { 4097 u16 cur_mlx5_mode, mlx5_mode = 0; 4098 struct mlx5_eswitch *esw; 4099 int err = 0; 4100 4101 esw = mlx5_devlink_eswitch_get(devlink); 4102 if (IS_ERR(esw)) 4103 return PTR_ERR(esw); 4104 4105 if (mlx5_fw_reset_in_progress(esw->dev)) { 4106 NL_SET_ERR_MSG_MOD(extack, "Can't change eswitch mode during firmware reset"); 4107 return -EBUSY; 4108 } 4109 4110 if (esw_mode_from_devlink(mode, &mlx5_mode)) 4111 return -EINVAL; 4112 4113 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && mlx5_get_sd(esw->dev)) { 4114 NL_SET_ERR_MSG_MOD(extack, 4115 "Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured."); 4116 return -EPERM; 4117 } 4118 4119 /* Avoid try_lock, active/inactive mode change is not restricted */ 4120 if (mlx5_devlink_switchdev_active_mode_change(esw, mode)) 4121 return 0; 4122 4123 mlx5_lag_disable_change(esw->dev); 4124 err = mlx5_esw_try_lock(esw); 4125 if (err < 0) { 4126 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 4127 goto enable_lag; 4128 } 4129 cur_mlx5_mode = err; 4130 err = 0; 4131 4132 if (cur_mlx5_mode == mlx5_mode) 4133 goto unlock; 4134 4135 if (esw->offloads.num_block_mode) { 4136 NL_SET_ERR_MSG_MOD(extack, 4137 "Can't change eswitch mode when IPsec SA and/or policies are configured"); 4138 err = -EOPNOTSUPP; 4139 goto unlock; 4140 } 4141 4142 esw->eswitch_operation_in_progress = true; 4143 up_write(&esw->mode_lock); 4144 4145 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && 4146 !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) { 4147 NL_SET_ERR_MSG_MOD(extack, 4148 "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's."); 4149 err = -EINVAL; 4150 goto skip; 4151 } 4152 4153 if (mlx5_mode == MLX5_ESWITCH_LEGACY) 4154 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; 4155 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) 4156 esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_SWITCH_LEGACY; 4157 mlx5_eswitch_disable_locked(esw); 4158 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) { 4159 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 4160 NL_SET_ERR_MSG_MOD(extack, 4161 "Can't change mode while devlink traps are active"); 4162 err = -EOPNOTSUPP; 4163 goto skip; 4164 } 4165 esw->offloads_inactive = 4166 (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE); 4167 err = esw_offloads_start(esw, extack); 4168 } else if (mlx5_mode == MLX5_ESWITCH_LEGACY) { 4169 err = esw_offloads_stop(esw, extack); 4170 } else { 4171 err = -EINVAL; 4172 } 4173 4174 skip: 4175 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && err) 4176 mlx5_devlink_netdev_netns_immutable_set(devlink, false); 4177 down_write(&esw->mode_lock); 4178 esw->eswitch_operation_in_progress = false; 4179 unlock: 4180 mlx5_esw_unlock(esw); 4181 enable_lag: 4182 mlx5_lag_enable_change(esw->dev); 4183 return err; 4184 } 4185 4186 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 4187 { 4188 struct mlx5_eswitch *esw; 4189 4190 esw = mlx5_devlink_eswitch_get(devlink); 4191 if (IS_ERR(esw)) 4192 return PTR_ERR(esw); 4193 4194 return esw_mode_to_devlink(esw, mode); 4195 } 4196 4197 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 4198 struct netlink_ext_ack *extack) 4199 { 4200 struct mlx5_core_dev *dev = esw->dev; 4201 struct mlx5_vport *vport; 4202 u16 err_vport_num = 0; 4203 unsigned long i; 4204 int err = 0; 4205 4206 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 4207 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 4208 if (err) { 4209 err_vport_num = vport->vport; 4210 NL_SET_ERR_MSG_MOD(extack, 4211 "Failed to set min inline on vport"); 4212 goto revert_inline_mode; 4213 } 4214 } 4215 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 4216 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 4217 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 4218 if (err) { 4219 err_vport_num = vport->vport; 4220 NL_SET_ERR_MSG_MOD(extack, 4221 "Failed to set min inline on vport"); 4222 goto revert_ec_vf_inline_mode; 4223 } 4224 } 4225 } 4226 return 0; 4227 4228 revert_ec_vf_inline_mode: 4229 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 4230 if (vport->vport == err_vport_num) 4231 break; 4232 mlx5_modify_nic_vport_min_inline(dev, 4233 vport->vport, 4234 esw->offloads.inline_mode); 4235 } 4236 revert_inline_mode: 4237 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 4238 if (vport->vport == err_vport_num) 4239 break; 4240 mlx5_modify_nic_vport_min_inline(dev, 4241 vport->vport, 4242 esw->offloads.inline_mode); 4243 } 4244 return err; 4245 } 4246 4247 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 4248 struct netlink_ext_ack *extack) 4249 { 4250 struct mlx5_core_dev *dev = devlink_priv(devlink); 4251 struct mlx5_eswitch *esw; 4252 u8 mlx5_mode; 4253 int err; 4254 4255 esw = mlx5_devlink_eswitch_get(devlink); 4256 if (IS_ERR(esw)) 4257 return PTR_ERR(esw); 4258 4259 down_write(&esw->mode_lock); 4260 4261 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 4262 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 4263 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 4264 err = 0; 4265 goto out; 4266 } 4267 4268 fallthrough; 4269 case MLX5_CAP_INLINE_MODE_L2: 4270 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 4271 err = -EOPNOTSUPP; 4272 goto out; 4273 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 4274 break; 4275 } 4276 4277 if (atomic64_read(&esw->offloads.num_flows) > 0) { 4278 NL_SET_ERR_MSG_MOD(extack, 4279 "Can't set inline mode when flows are configured"); 4280 err = -EOPNOTSUPP; 4281 goto out; 4282 } 4283 4284 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 4285 if (err) 4286 goto out; 4287 4288 esw->eswitch_operation_in_progress = true; 4289 up_write(&esw->mode_lock); 4290 4291 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 4292 if (!err) 4293 esw->offloads.inline_mode = mlx5_mode; 4294 4295 down_write(&esw->mode_lock); 4296 esw->eswitch_operation_in_progress = false; 4297 up_write(&esw->mode_lock); 4298 return 0; 4299 4300 out: 4301 up_write(&esw->mode_lock); 4302 return err; 4303 } 4304 4305 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 4306 { 4307 struct mlx5_eswitch *esw; 4308 4309 esw = mlx5_devlink_eswitch_get(devlink); 4310 if (IS_ERR(esw)) 4311 return PTR_ERR(esw); 4312 4313 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 4314 } 4315 4316 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev, bool from_fdb) 4317 { 4318 struct mlx5_eswitch *esw = dev->priv.eswitch; 4319 enum devlink_eswitch_encap_mode encap; 4320 bool allow_tunnel = false; 4321 4322 if (!mlx5_esw_allowed(esw)) 4323 return true; 4324 4325 down_write(&esw->mode_lock); 4326 encap = esw->offloads.encap; 4327 if (esw->mode == MLX5_ESWITCH_LEGACY || 4328 (encap == DEVLINK_ESWITCH_ENCAP_MODE_NONE && !from_fdb)) { 4329 allow_tunnel = true; 4330 esw->offloads.num_block_encap++; 4331 } 4332 up_write(&esw->mode_lock); 4333 4334 return allow_tunnel; 4335 } 4336 4337 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev) 4338 { 4339 struct mlx5_eswitch *esw = dev->priv.eswitch; 4340 4341 if (!mlx5_esw_allowed(esw)) 4342 return; 4343 4344 down_write(&esw->mode_lock); 4345 esw->offloads.num_block_encap--; 4346 up_write(&esw->mode_lock); 4347 } 4348 4349 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 4350 enum devlink_eswitch_encap_mode encap, 4351 struct netlink_ext_ack *extack) 4352 { 4353 struct mlx5_core_dev *dev = devlink_priv(devlink); 4354 struct mlx5_eswitch *esw; 4355 int err = 0; 4356 4357 esw = mlx5_devlink_eswitch_get(devlink); 4358 if (IS_ERR(esw)) 4359 return PTR_ERR(esw); 4360 4361 down_write(&esw->mode_lock); 4362 4363 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 4364 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 4365 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 4366 err = -EOPNOTSUPP; 4367 goto unlock; 4368 } 4369 4370 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 4371 err = -EOPNOTSUPP; 4372 goto unlock; 4373 } 4374 4375 if (esw->mode == MLX5_ESWITCH_LEGACY) { 4376 esw->offloads.encap = encap; 4377 goto unlock; 4378 } 4379 4380 if (esw->offloads.encap == encap) 4381 goto unlock; 4382 4383 if (atomic64_read(&esw->offloads.num_flows) > 0) { 4384 NL_SET_ERR_MSG_MOD(extack, 4385 "Can't set encapsulation when flows are configured"); 4386 err = -EOPNOTSUPP; 4387 goto unlock; 4388 } 4389 4390 if (esw->offloads.num_block_encap) { 4391 NL_SET_ERR_MSG_MOD(extack, 4392 "Can't set encapsulation when IPsec SA and/or policies are configured"); 4393 err = -EOPNOTSUPP; 4394 goto unlock; 4395 } 4396 4397 esw->eswitch_operation_in_progress = true; 4398 up_write(&esw->mode_lock); 4399 4400 esw_destroy_offloads_fdb_tables(esw); 4401 4402 esw->offloads.encap = encap; 4403 4404 err = esw_create_offloads_fdb_tables(esw); 4405 4406 if (err) { 4407 NL_SET_ERR_MSG_MOD(extack, 4408 "Failed re-creating fast FDB table"); 4409 esw->offloads.encap = !encap; 4410 (void)esw_create_offloads_fdb_tables(esw); 4411 } 4412 4413 down_write(&esw->mode_lock); 4414 esw->eswitch_operation_in_progress = false; 4415 4416 unlock: 4417 up_write(&esw->mode_lock); 4418 return err; 4419 } 4420 4421 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 4422 enum devlink_eswitch_encap_mode *encap) 4423 { 4424 struct mlx5_eswitch *esw; 4425 4426 esw = mlx5_devlink_eswitch_get(devlink); 4427 if (IS_ERR(esw)) 4428 return PTR_ERR(esw); 4429 4430 *encap = esw->offloads.encap; 4431 return 0; 4432 } 4433 4434 static bool 4435 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 4436 { 4437 /* Currently, only ECPF based device has representor for host PF. */ 4438 if (vport_num == MLX5_VPORT_PF && 4439 (!mlx5_core_is_ecpf_esw_manager(esw->dev) || 4440 !mlx5_esw_host_functions_enabled(esw->dev))) 4441 return false; 4442 4443 if (vport_num == MLX5_VPORT_ECPF && 4444 !mlx5_ecpf_vport_exists(esw->dev)) 4445 return false; 4446 4447 return true; 4448 } 4449 4450 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 4451 const struct mlx5_eswitch_rep_ops *ops, 4452 u8 rep_type) 4453 { 4454 struct mlx5_eswitch_rep_data *rep_data; 4455 struct mlx5_eswitch_rep *rep; 4456 unsigned long i; 4457 4458 esw->offloads.rep_ops[rep_type] = ops; 4459 mlx5_esw_for_each_rep(esw, i, rep) { 4460 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 4461 rep->esw = esw; 4462 rep_data = &rep->rep_data[rep_type]; 4463 atomic_set(&rep_data->state, REP_REGISTERED); 4464 } 4465 } 4466 } 4467 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 4468 4469 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 4470 { 4471 struct mlx5_eswitch_rep *rep; 4472 unsigned long i; 4473 4474 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 4475 __unload_reps_all_vport(esw, rep_type); 4476 4477 mlx5_esw_for_each_rep(esw, i, rep) 4478 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 4479 } 4480 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 4481 4482 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 4483 { 4484 struct mlx5_eswitch_rep *rep; 4485 4486 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 4487 return rep->rep_data[rep_type].priv; 4488 } 4489 4490 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 4491 u16 vport, 4492 u8 rep_type) 4493 { 4494 struct mlx5_eswitch_rep *rep; 4495 4496 rep = mlx5_eswitch_get_rep(esw, vport); 4497 4498 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 4499 esw->offloads.rep_ops[rep_type]->get_proto_dev) 4500 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 4501 return NULL; 4502 } 4503 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 4504 4505 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 4506 { 4507 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 4508 } 4509 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 4510 4511 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 4512 u16 vport) 4513 { 4514 return mlx5_eswitch_get_rep(esw, vport); 4515 } 4516 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 4517 4518 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 4519 { 4520 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 4521 } 4522 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 4523 4524 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 4525 { 4526 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 4527 } 4528 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 4529 4530 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 4531 u16 vport_num) 4532 { 4533 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4534 4535 if (WARN_ON_ONCE(IS_ERR(vport))) 4536 return 0; 4537 4538 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 4539 } 4540 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 4541 4542 int mlx5_esw_vport_vhca_id_map(struct mlx5_eswitch *esw, 4543 struct mlx5_vport *vport) 4544 { 4545 u16 *old_entry, *vhca_map_entry, vhca_id; 4546 4547 if (WARN_ONCE(MLX5_VPORT_INVAL_VHCA_ID(vport), 4548 "vport %d vhca_id is not set", vport->vport)) { 4549 int err; 4550 4551 err = mlx5_vport_get_vhca_id(vport->dev, vport->vport, 4552 &vhca_id); 4553 if (err) 4554 return err; 4555 vport->vhca_id = vhca_id; 4556 } 4557 4558 vhca_id = vport->vhca_id; 4559 vhca_map_entry = kmalloc_obj(*vhca_map_entry); 4560 if (!vhca_map_entry) 4561 return -ENOMEM; 4562 4563 *vhca_map_entry = vport->vport; 4564 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 4565 if (xa_is_err(old_entry)) { 4566 kfree(vhca_map_entry); 4567 return xa_err(old_entry); 4568 } 4569 kfree(old_entry); 4570 return 0; 4571 } 4572 4573 void mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch *esw, 4574 struct mlx5_vport *vport) 4575 { 4576 u16 *vhca_map_entry; 4577 4578 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vport->vhca_id); 4579 kfree(vhca_map_entry); 4580 } 4581 4582 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 4583 { 4584 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 4585 4586 if (!res) 4587 return -ENOENT; 4588 4589 *vport_num = *res; 4590 return 0; 4591 } 4592 4593 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 4594 u16 vport_num) 4595 { 4596 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4597 4598 if (WARN_ON_ONCE(IS_ERR(vport))) 4599 return 0; 4600 4601 return vport->metadata; 4602 } 4603 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 4604 4605 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port, 4606 u8 *hw_addr, int *hw_addr_len, 4607 struct netlink_ext_ack *extack) 4608 { 4609 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4610 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4611 4612 mutex_lock(&esw->state_lock); 4613 4614 mlx5_query_nic_vport_mac_address(esw->dev, vport->vport, true, 4615 vport->info.mac); 4616 ether_addr_copy(hw_addr, vport->info.mac); 4617 *hw_addr_len = ETH_ALEN; 4618 mutex_unlock(&esw->state_lock); 4619 return 0; 4620 } 4621 4622 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port, 4623 const u8 *hw_addr, int hw_addr_len, 4624 struct netlink_ext_ack *extack) 4625 { 4626 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4627 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4628 4629 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr); 4630 } 4631 4632 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, 4633 struct netlink_ext_ack *extack) 4634 { 4635 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4636 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4637 4638 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4639 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4640 return -EOPNOTSUPP; 4641 } 4642 4643 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4644 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4645 return -EOPNOTSUPP; 4646 } 4647 4648 mutex_lock(&esw->state_lock); 4649 *is_enabled = vport->info.mig_enabled; 4650 mutex_unlock(&esw->state_lock); 4651 return 0; 4652 } 4653 4654 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, 4655 struct netlink_ext_ack *extack) 4656 { 4657 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4658 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4659 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4660 void *query_ctx; 4661 void *hca_caps; 4662 int err; 4663 4664 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4665 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4666 return -EOPNOTSUPP; 4667 } 4668 4669 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4670 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4671 return -EOPNOTSUPP; 4672 } 4673 4674 mutex_lock(&esw->state_lock); 4675 4676 if (vport->info.mig_enabled == enable) { 4677 err = 0; 4678 goto out; 4679 } 4680 4681 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4682 if (!query_ctx) { 4683 err = -ENOMEM; 4684 goto out; 4685 } 4686 4687 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, 4688 MLX5_CAP_GENERAL_2); 4689 if (err) { 4690 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4691 goto out_free; 4692 } 4693 4694 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4695 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable); 4696 4697 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, 4698 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 4699 if (err) { 4700 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); 4701 goto out_free; 4702 } 4703 4704 vport->info.mig_enabled = enable; 4705 4706 out_free: 4707 kfree(query_ctx); 4708 out: 4709 mutex_unlock(&esw->state_lock); 4710 return err; 4711 } 4712 4713 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, 4714 struct netlink_ext_ack *extack) 4715 { 4716 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4717 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4718 4719 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4720 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4721 return -EOPNOTSUPP; 4722 } 4723 4724 mutex_lock(&esw->state_lock); 4725 *is_enabled = vport->info.roce_enabled; 4726 mutex_unlock(&esw->state_lock); 4727 return 0; 4728 } 4729 4730 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, 4731 struct netlink_ext_ack *extack) 4732 { 4733 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4734 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4735 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4736 u16 vport_num = vport->vport; 4737 void *query_ctx; 4738 void *hca_caps; 4739 int err; 4740 4741 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4742 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4743 return -EOPNOTSUPP; 4744 } 4745 4746 mutex_lock(&esw->state_lock); 4747 4748 if (vport->info.roce_enabled == enable) { 4749 err = 0; 4750 goto out; 4751 } 4752 4753 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4754 if (!query_ctx) { 4755 err = -ENOMEM; 4756 goto out; 4757 } 4758 4759 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4760 MLX5_CAP_GENERAL); 4761 if (err) { 4762 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4763 goto out_free; 4764 } 4765 4766 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4767 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); 4768 4769 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 4770 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 4771 if (err) { 4772 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); 4773 goto out_free; 4774 } 4775 4776 vport->info.roce_enabled = enable; 4777 4778 out_free: 4779 kfree(query_ctx); 4780 out: 4781 mutex_unlock(&esw->state_lock); 4782 return err; 4783 } 4784 4785 int mlx5_devlink_pf_port_fn_state_get(struct devlink_port *port, 4786 enum devlink_port_fn_state *state, 4787 enum devlink_port_fn_opstate *opstate, 4788 struct netlink_ext_ack *extack) 4789 { 4790 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4791 const u32 *query_out; 4792 bool pf_disabled; 4793 4794 if (vport->vport != MLX5_VPORT_PF) { 4795 NL_SET_ERR_MSG_MOD(extack, "State get is not supported for VF"); 4796 return -EOPNOTSUPP; 4797 } 4798 4799 *state = vport->pf_activated ? 4800 DEVLINK_PORT_FN_STATE_ACTIVE : DEVLINK_PORT_FN_STATE_INACTIVE; 4801 4802 query_out = mlx5_esw_query_functions(vport->dev); 4803 if (IS_ERR(query_out)) 4804 return PTR_ERR(query_out); 4805 4806 pf_disabled = MLX5_GET(query_esw_functions_out, query_out, 4807 host_params_context.host_pf_disabled); 4808 4809 *opstate = pf_disabled ? DEVLINK_PORT_FN_OPSTATE_DETACHED : 4810 DEVLINK_PORT_FN_OPSTATE_ATTACHED; 4811 4812 kvfree(query_out); 4813 return 0; 4814 } 4815 4816 int mlx5_devlink_pf_port_fn_state_set(struct devlink_port *port, 4817 enum devlink_port_fn_state state, 4818 struct netlink_ext_ack *extack) 4819 { 4820 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4821 struct mlx5_core_dev *dev; 4822 4823 if (vport->vport != MLX5_VPORT_PF) { 4824 NL_SET_ERR_MSG_MOD(extack, "State set is not supported for VF"); 4825 return -EOPNOTSUPP; 4826 } 4827 4828 dev = vport->dev; 4829 4830 switch (state) { 4831 case DEVLINK_PORT_FN_STATE_ACTIVE: 4832 return mlx5_esw_host_pf_enable_hca(dev); 4833 case DEVLINK_PORT_FN_STATE_INACTIVE: 4834 return mlx5_esw_host_pf_disable_hca(dev); 4835 default: 4836 return -EOPNOTSUPP; 4837 } 4838 } 4839 4840 int 4841 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule, 4842 struct mlx5_esw_flow_attr *esw_attr, int attr_idx) 4843 { 4844 struct mlx5_flow_destination new_dest = {}; 4845 struct mlx5_flow_destination old_dest = {}; 4846 4847 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 4848 return 0; 4849 4850 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4851 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4852 4853 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest); 4854 } 4855 4856 #ifdef CONFIG_XFRM_OFFLOAD 4857 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled, 4858 struct netlink_ext_ack *extack) 4859 { 4860 struct mlx5_eswitch *esw; 4861 struct mlx5_vport *vport; 4862 int err = 0; 4863 4864 esw = mlx5_devlink_eswitch_get(port->devlink); 4865 if (IS_ERR(esw)) 4866 return PTR_ERR(esw); 4867 4868 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4869 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto"); 4870 return -EOPNOTSUPP; 4871 } 4872 4873 vport = mlx5_devlink_port_vport_get(port); 4874 4875 mutex_lock(&esw->state_lock); 4876 if (!vport->enabled) { 4877 err = -EOPNOTSUPP; 4878 goto unlock; 4879 } 4880 4881 *is_enabled = vport->info.ipsec_crypto_enabled; 4882 unlock: 4883 mutex_unlock(&esw->state_lock); 4884 return err; 4885 } 4886 4887 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable, 4888 struct netlink_ext_ack *extack) 4889 { 4890 struct mlx5_eswitch *esw; 4891 struct mlx5_vport *vport; 4892 u16 vport_num; 4893 int err; 4894 4895 esw = mlx5_devlink_eswitch_get(port->devlink); 4896 if (IS_ERR(esw)) 4897 return PTR_ERR(esw); 4898 4899 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4900 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num); 4901 if (err) { 4902 NL_SET_ERR_MSG_MOD(extack, 4903 "Device doesn't support IPsec crypto"); 4904 return err; 4905 } 4906 4907 vport = mlx5_devlink_port_vport_get(port); 4908 4909 mutex_lock(&esw->state_lock); 4910 if (!vport->enabled) { 4911 err = -EOPNOTSUPP; 4912 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4913 goto unlock; 4914 } 4915 4916 if (vport->info.ipsec_crypto_enabled == enable) 4917 goto unlock; 4918 4919 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 4920 err = -EBUSY; 4921 goto unlock; 4922 } 4923 4924 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable); 4925 if (err) { 4926 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto"); 4927 goto unlock; 4928 } 4929 4930 vport->info.ipsec_crypto_enabled = enable; 4931 if (enable) 4932 esw->enabled_ipsec_vf_count++; 4933 else 4934 esw->enabled_ipsec_vf_count--; 4935 unlock: 4936 mutex_unlock(&esw->state_lock); 4937 return err; 4938 } 4939 4940 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled, 4941 struct netlink_ext_ack *extack) 4942 { 4943 struct mlx5_eswitch *esw; 4944 struct mlx5_vport *vport; 4945 int err = 0; 4946 4947 esw = mlx5_devlink_eswitch_get(port->devlink); 4948 if (IS_ERR(esw)) 4949 return PTR_ERR(esw); 4950 4951 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4952 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet"); 4953 return -EOPNOTSUPP; 4954 } 4955 4956 vport = mlx5_devlink_port_vport_get(port); 4957 4958 mutex_lock(&esw->state_lock); 4959 if (!vport->enabled) { 4960 err = -EOPNOTSUPP; 4961 goto unlock; 4962 } 4963 4964 *is_enabled = vport->info.ipsec_packet_enabled; 4965 unlock: 4966 mutex_unlock(&esw->state_lock); 4967 return err; 4968 } 4969 4970 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, 4971 bool enable, 4972 struct netlink_ext_ack *extack) 4973 { 4974 struct mlx5_eswitch *esw; 4975 struct mlx5_vport *vport; 4976 u16 vport_num; 4977 int err; 4978 4979 esw = mlx5_devlink_eswitch_get(port->devlink); 4980 if (IS_ERR(esw)) 4981 return PTR_ERR(esw); 4982 4983 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4984 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num); 4985 if (err) { 4986 NL_SET_ERR_MSG_MOD(extack, 4987 "Device doesn't support IPsec packet mode"); 4988 return err; 4989 } 4990 4991 vport = mlx5_devlink_port_vport_get(port); 4992 mutex_lock(&esw->state_lock); 4993 if (!vport->enabled) { 4994 err = -EOPNOTSUPP; 4995 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4996 goto unlock; 4997 } 4998 4999 if (vport->info.ipsec_packet_enabled == enable) 5000 goto unlock; 5001 5002 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 5003 err = -EBUSY; 5004 goto unlock; 5005 } 5006 5007 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable); 5008 if (err) { 5009 NL_SET_ERR_MSG_MOD(extack, 5010 "Failed to set IPsec packet mode"); 5011 goto unlock; 5012 } 5013 5014 vport->info.ipsec_packet_enabled = enable; 5015 if (enable) 5016 esw->enabled_ipsec_vf_count++; 5017 else 5018 esw->enabled_ipsec_vf_count--; 5019 unlock: 5020 mutex_unlock(&esw->state_lock); 5021 return err; 5022 } 5023 #endif /* CONFIG_XFRM_OFFLOAD */ 5024 5025 int 5026 mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs, 5027 struct netlink_ext_ack *extack) 5028 { 5029 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5030 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 5031 u16 vport_num = vport->vport; 5032 struct mlx5_eswitch *esw; 5033 void *query_ctx; 5034 void *hca_caps; 5035 u32 max_eqs; 5036 int err; 5037 5038 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5039 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 5040 NL_SET_ERR_MSG_MOD(extack, 5041 "Device doesn't support VHCA management"); 5042 return -EOPNOTSUPP; 5043 } 5044 5045 if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) { 5046 NL_SET_ERR_MSG_MOD(extack, 5047 "Device doesn't support getting the max number of EQs"); 5048 return -EOPNOTSUPP; 5049 } 5050 5051 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 5052 if (!query_ctx) 5053 return -ENOMEM; 5054 5055 mutex_lock(&esw->state_lock); 5056 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 5057 MLX5_CAP_GENERAL_2); 5058 if (err) { 5059 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 5060 goto out; 5061 } 5062 5063 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 5064 max_eqs = MLX5_GET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b); 5065 if (max_eqs < MLX5_ESW_MAX_CTRL_EQS) 5066 *max_io_eqs = 0; 5067 else 5068 *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS; 5069 out: 5070 mutex_unlock(&esw->state_lock); 5071 kfree(query_ctx); 5072 return err; 5073 } 5074 5075 int 5076 mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs, 5077 struct netlink_ext_ack *extack) 5078 { 5079 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5080 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 5081 u16 vport_num = vport->vport; 5082 struct mlx5_eswitch *esw; 5083 void *query_ctx; 5084 void *hca_caps; 5085 u16 max_eqs; 5086 int err; 5087 5088 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5089 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 5090 NL_SET_ERR_MSG_MOD(extack, 5091 "Device doesn't support VHCA management"); 5092 return -EOPNOTSUPP; 5093 } 5094 5095 if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) { 5096 NL_SET_ERR_MSG_MOD(extack, 5097 "Device doesn't support changing the max number of EQs"); 5098 return -EOPNOTSUPP; 5099 } 5100 5101 if (check_add_overflow(max_io_eqs, MLX5_ESW_MAX_CTRL_EQS, &max_eqs)) { 5102 NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range"); 5103 return -EINVAL; 5104 } 5105 5106 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 5107 if (!query_ctx) 5108 return -ENOMEM; 5109 5110 mutex_lock(&esw->state_lock); 5111 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 5112 MLX5_CAP_GENERAL_2); 5113 if (err) { 5114 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 5115 goto out; 5116 } 5117 5118 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 5119 MLX5_SET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b, max_eqs); 5120 5121 if (mlx5_esw_is_sf_vport(esw, vport_num)) 5122 MLX5_SET(cmd_hca_cap_2, hca_caps, sf_eq_usage, 1); 5123 5124 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 5125 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 5126 if (err) 5127 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA caps"); 5128 vport->max_eqs_set = true; 5129 out: 5130 mutex_unlock(&esw->state_lock); 5131 kfree(query_ctx); 5132 return err; 5133 } 5134 5135 int 5136 mlx5_devlink_port_fn_max_io_eqs_set_sf_default(struct devlink_port *port, 5137 struct netlink_ext_ack *extack) 5138 { 5139 return mlx5_devlink_port_fn_max_io_eqs_set(port, 5140 MLX5_ESW_DEFAULT_SF_COMP_EQS, 5141 extack); 5142 } 5143