1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include <linux/lockdep.h> 40 #include "mlx5_core.h" 41 #include "eswitch.h" 42 #include "esw/indir_table.h" 43 #include "esw/acl/ofld.h" 44 #include "rdma.h" 45 #include "en.h" 46 #include "fs_core.h" 47 #include "lib/mlx5.h" 48 #include "lib/devcom.h" 49 #include "lib/sd.h" 50 #include "lib/eq.h" 51 #include "lib/fs_chains.h" 52 #include "en_tc.h" 53 #include "en/mapping.h" 54 #include "devlink.h" 55 #include "lag/lag.h" 56 #include "en/tc/post_meter.h" 57 #include "fw_reset.h" 58 59 /* There are two match-all miss flows, one for unicast dst mac and 60 * one for multicast. 61 */ 62 #define MLX5_ESW_MISS_FLOWS (2) 63 #define UPLINK_REP_INDEX 0 64 65 #define MLX5_ESW_VPORT_TBL_SIZE 128 66 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 67 68 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 69 70 #define MLX5_ESW_MAX_CTRL_EQS 4 71 #define MLX5_ESW_DEFAULT_SF_COMP_EQS 8 72 73 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 74 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 75 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 76 .flags = 0, 77 }; 78 79 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 80 u16 vport_num) 81 { 82 return xa_load(&esw->offloads.vport_reps, vport_num); 83 } 84 85 static void 86 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 87 struct mlx5_flow_spec *spec, 88 struct mlx5_esw_flow_attr *attr) 89 { 90 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 91 return; 92 93 if (attr->int_port) { 94 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 95 96 return; 97 } 98 99 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 100 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 101 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 102 } 103 104 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 105 * are not needed as well in the following process. So clear them all for simplicity. 106 */ 107 void 108 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 109 { 110 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 111 void *misc2; 112 113 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 114 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 115 116 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 117 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 118 119 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 120 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 121 } 122 } 123 124 static void 125 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 126 struct mlx5_flow_spec *spec, 127 struct mlx5_flow_attr *attr, 128 struct mlx5_eswitch *src_esw, 129 u16 vport) 130 { 131 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 132 u32 metadata; 133 void *misc2; 134 void *misc; 135 136 /* Use metadata matching because vport is not represented by single 137 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 138 */ 139 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 140 if (mlx5_esw_indir_table_decap_vport(attr)) 141 vport = mlx5_esw_indir_table_decap_vport(attr); 142 143 if (!attr->chain && esw_attr && esw_attr->int_port) 144 metadata = 145 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 146 else 147 metadata = 148 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 149 150 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 151 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 152 153 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 154 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 155 mlx5_eswitch_get_vport_metadata_mask()); 156 157 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 158 } else { 159 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 160 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 161 162 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 163 MLX5_SET(fte_match_set_misc, misc, 164 source_eswitch_owner_vhca_id, 165 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 166 167 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 168 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 169 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 170 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 171 source_eswitch_owner_vhca_id); 172 173 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 174 } 175 } 176 177 static int 178 esw_setup_decap_indir(struct mlx5_eswitch *esw, 179 struct mlx5_flow_attr *attr) 180 { 181 struct mlx5_flow_table *ft; 182 183 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 184 return -EOPNOTSUPP; 185 186 ft = mlx5_esw_indir_table_get(esw, attr, 187 mlx5_esw_indir_table_decap_vport(attr), true); 188 return PTR_ERR_OR_ZERO(ft); 189 } 190 191 static void 192 esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 193 struct mlx5_flow_attr *attr) 194 { 195 if (mlx5_esw_indir_table_decap_vport(attr)) 196 mlx5_esw_indir_table_put(esw, 197 mlx5_esw_indir_table_decap_vport(attr), 198 true); 199 } 200 201 static int 202 esw_setup_mtu_dest(struct mlx5_flow_destination *dest, 203 struct mlx5e_meter_attr *meter, 204 int i) 205 { 206 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; 207 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; 208 dest[i].range.min = 0; 209 dest[i].range.max = meter->params.mtu; 210 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); 211 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); 212 213 return 0; 214 } 215 216 static int 217 esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 218 struct mlx5_flow_act *flow_act, 219 u32 sampler_id, 220 int i) 221 { 222 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 223 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 224 dest[i].sampler_id = sampler_id; 225 226 return 0; 227 } 228 229 static int 230 esw_setup_ft_dest(struct mlx5_flow_destination *dest, 231 struct mlx5_flow_act *flow_act, 232 struct mlx5_eswitch *esw, 233 struct mlx5_flow_attr *attr, 234 int i) 235 { 236 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 237 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 238 dest[i].ft = attr->dest_ft; 239 240 if (mlx5_esw_indir_table_decap_vport(attr)) 241 return esw_setup_decap_indir(esw, attr); 242 return 0; 243 } 244 245 static void 246 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 247 struct mlx5_fs_chains *chains, int i) 248 { 249 if (mlx5_chains_ignore_flow_level_supported(chains)) 250 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 251 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 252 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 253 } 254 255 static void 256 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 257 struct mlx5_eswitch *esw, int i) 258 { 259 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 260 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 261 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 262 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); 263 } 264 265 static int 266 esw_setup_chain_dest(struct mlx5_flow_destination *dest, 267 struct mlx5_flow_act *flow_act, 268 struct mlx5_fs_chains *chains, 269 u32 chain, u32 prio, u32 level, 270 int i) 271 { 272 struct mlx5_flow_table *ft; 273 274 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 275 ft = mlx5_chains_get_table(chains, chain, prio, level); 276 if (IS_ERR(ft)) 277 return PTR_ERR(ft); 278 279 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 280 dest[i].ft = ft; 281 return 0; 282 } 283 284 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 285 int from, int to) 286 { 287 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 288 struct mlx5_fs_chains *chains = esw_chains(esw); 289 int i; 290 291 for (i = from; i < to; i++) 292 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 293 mlx5_chains_put_table(chains, 0, 1, 0); 294 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 295 esw_attr->dests[i].mdev)) 296 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false); 297 } 298 299 static bool 300 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 301 { 302 int i; 303 304 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 305 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 306 return true; 307 return false; 308 } 309 310 static int 311 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 312 struct mlx5_flow_act *flow_act, 313 struct mlx5_eswitch *esw, 314 struct mlx5_fs_chains *chains, 315 struct mlx5_flow_attr *attr, 316 int *i) 317 { 318 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 319 int err; 320 321 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 322 return -EOPNOTSUPP; 323 324 /* flow steering cannot handle more than one dest with the same ft 325 * in a single flow 326 */ 327 if (esw_attr->out_count - esw_attr->split_count > 1) 328 return -EOPNOTSUPP; 329 330 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 331 if (err) 332 return err; 333 334 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 335 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 336 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 337 } 338 (*i)++; 339 340 return 0; 341 } 342 343 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 344 struct mlx5_flow_attr *attr) 345 { 346 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 347 348 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 349 } 350 351 static bool 352 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 353 { 354 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 355 bool result = false; 356 int i; 357 358 /* Indirect table is supported only for flows with in_port uplink 359 * and the destination is vport on the same eswitch as the uplink, 360 * return false in case at least one of destinations doesn't meet 361 * this criteria. 362 */ 363 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 364 if (esw_attr->dests[i].vport_valid && 365 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 366 esw_attr->dests[i].mdev)) { 367 result = true; 368 } else { 369 result = false; 370 break; 371 } 372 } 373 return result; 374 } 375 376 static int 377 esw_setup_indir_table(struct mlx5_flow_destination *dest, 378 struct mlx5_flow_act *flow_act, 379 struct mlx5_eswitch *esw, 380 struct mlx5_flow_attr *attr, 381 int *i) 382 { 383 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 384 int j, err; 385 386 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 387 return -EOPNOTSUPP; 388 389 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 390 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 391 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 392 393 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, 394 esw_attr->dests[j].vport, false); 395 if (IS_ERR(dest[*i].ft)) { 396 err = PTR_ERR(dest[*i].ft); 397 goto err_indir_tbl_get; 398 } 399 } 400 401 if (mlx5_esw_indir_table_decap_vport(attr)) { 402 err = esw_setup_decap_indir(esw, attr); 403 if (err) 404 goto err_indir_tbl_get; 405 } 406 407 return 0; 408 409 err_indir_tbl_get: 410 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 411 return err; 412 } 413 414 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 415 { 416 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 417 418 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 419 esw_cleanup_decap_indir(esw, attr); 420 } 421 422 static void 423 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 424 { 425 mlx5_chains_put_table(chains, chain, prio, level); 426 } 427 428 static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2) 429 { 430 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id); 431 } 432 433 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw, 434 struct mlx5_esw_flow_attr *esw_attr, 435 int attr_idx) 436 { 437 if (esw->offloads.ft_ipsec_tx_pol && 438 esw_attr->dests[attr_idx].vport_valid && 439 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK && 440 /* To be aligned with software, encryption is needed only for tunnel device */ 441 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) && 442 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport && 443 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev)) 444 return true; 445 446 return false; 447 } 448 449 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw, 450 struct mlx5_esw_flow_attr *esw_attr) 451 { 452 int i; 453 454 if (!esw->offloads.ft_ipsec_tx_pol) 455 return true; 456 457 for (i = 0; i < esw_attr->split_count; i++) 458 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i)) 459 return false; 460 461 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 462 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) && 463 (esw_attr->out_count - esw_attr->split_count > 1)) 464 return false; 465 466 return true; 467 } 468 469 static void 470 esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 471 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 472 int attr_idx, int dest_idx, bool pkt_reformat) 473 { 474 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 475 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport; 476 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 477 dest[dest_idx].vport.vhca_id = 478 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 479 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 480 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 481 mlx5_lag_is_mpesw(esw->dev)) 482 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 483 } 484 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 485 if (pkt_reformat) { 486 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 487 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 488 } 489 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 490 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 491 } 492 } 493 494 static void 495 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 496 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 497 int attr_idx, int dest_idx, bool pkt_reformat) 498 { 499 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol; 500 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 501 if (pkt_reformat && 502 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 503 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 504 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 505 } 506 } 507 508 static void 509 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 510 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 511 int attr_idx, int dest_idx, bool pkt_reformat) 512 { 513 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 514 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr, 515 attr_idx, dest_idx, pkt_reformat); 516 else 517 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr, 518 attr_idx, dest_idx, pkt_reformat); 519 } 520 521 static int 522 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 523 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 524 int i) 525 { 526 int j; 527 528 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 529 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 530 return i; 531 } 532 533 static bool 534 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 535 { 536 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 537 mlx5_eswitch_vport_match_metadata_enabled(esw) && 538 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 539 } 540 541 static bool 542 esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest) 543 { 544 bool internal_dest = false, external_dest = false; 545 int i; 546 547 for (i = 0; i < max_dest; i++) { 548 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT && 549 dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK) 550 continue; 551 552 /* Uplink dest is external, but considered as internal 553 * if there is reformat because firmware uses LB+hairpin to support it. 554 */ 555 if (dests[i].vport.num == MLX5_VPORT_UPLINK && 556 !(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)) 557 external_dest = true; 558 else 559 internal_dest = true; 560 561 if (internal_dest && external_dest) 562 return true; 563 } 564 565 return false; 566 } 567 568 static int 569 esw_setup_dests(struct mlx5_flow_destination *dest, 570 struct mlx5_flow_act *flow_act, 571 struct mlx5_eswitch *esw, 572 struct mlx5_flow_attr *attr, 573 struct mlx5_flow_spec *spec, 574 int *i) 575 { 576 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 577 struct mlx5_fs_chains *chains = esw_chains(esw); 578 int err = 0; 579 580 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 581 esw_src_port_rewrite_supported(esw)) 582 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 583 584 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 585 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 586 (*i)++; 587 goto out; 588 } 589 590 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { 591 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 592 (*i)++; 593 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 594 esw_setup_accept_dest(dest, flow_act, chains, *i); 595 (*i)++; 596 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { 597 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); 598 (*i)++; 599 } else if (esw_is_indir_table(esw, attr)) { 600 err = esw_setup_indir_table(dest, flow_act, esw, attr, i); 601 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 602 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 603 } else { 604 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 605 606 if (attr->dest_ft) { 607 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i); 608 (*i)++; 609 } else if (attr->dest_chain) { 610 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 611 1, 0, *i); 612 (*i)++; 613 } 614 } 615 616 if (attr->extra_split_ft) { 617 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 618 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 619 dest[*i].ft = attr->extra_split_ft; 620 (*i)++; 621 } 622 623 out: 624 return err; 625 } 626 627 static void 628 esw_cleanup_dests(struct mlx5_eswitch *esw, 629 struct mlx5_flow_attr *attr) 630 { 631 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 632 struct mlx5_fs_chains *chains = esw_chains(esw); 633 634 if (attr->dest_ft) { 635 esw_cleanup_decap_indir(esw, attr); 636 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 637 if (attr->dest_chain) 638 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 639 else if (esw_is_indir_table(esw, attr)) 640 esw_cleanup_indir_table(esw, attr); 641 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 642 esw_cleanup_chain_src_port_rewrite(esw, attr); 643 } 644 } 645 646 static void 647 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 648 { 649 struct mlx5e_flow_meter_handle *meter; 650 651 meter = attr->meter_attr.meter; 652 flow_act->exe_aso.type = attr->exe_aso_type; 653 flow_act->exe_aso.object_id = meter->obj_id; 654 flow_act->exe_aso.base_id = mlx5e_flow_meter_get_base_id(meter); 655 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 656 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 657 /* use metadata reg 5 for packet color */ 658 flow_act->exe_aso.return_reg_id = 5; 659 } 660 661 struct mlx5_flow_handle * 662 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 663 struct mlx5_flow_spec *spec, 664 struct mlx5_flow_attr *attr) 665 { 666 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 667 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 668 struct mlx5_fs_chains *chains = esw_chains(esw); 669 bool split = !!(esw_attr->split_count); 670 struct mlx5_vport_tbl_attr fwd_attr; 671 struct mlx5_flow_destination *dest; 672 struct mlx5_flow_handle *rule; 673 struct mlx5_flow_table *fdb; 674 int i = 0; 675 676 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 677 return ERR_PTR(-EOPNOTSUPP); 678 679 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 680 return ERR_PTR(-EOPNOTSUPP); 681 682 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr)) 683 return ERR_PTR(-EOPNOTSUPP); 684 685 dest = kzalloc_objs(*dest, MLX5_MAX_FLOW_FWD_VPORTS + 1); 686 if (!dest) 687 return ERR_PTR(-ENOMEM); 688 689 flow_act.action = attr->action; 690 691 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 692 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 693 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 694 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 695 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 696 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 697 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 698 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 699 } 700 } 701 702 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 703 704 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 705 int err; 706 707 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 708 if (err) { 709 rule = ERR_PTR(err); 710 goto err_create_goto_table; 711 } 712 713 /* Header rewrite with combined wire+loopback in FDB is not allowed */ 714 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) && 715 esw_dests_to_int_external(dest, i)) { 716 esw_warn(esw->dev, 717 "FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n"); 718 rule = ERR_PTR(-EINVAL); 719 goto err_esw_get; 720 } 721 } 722 723 if (esw_attr->decap_pkt_reformat) 724 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 725 726 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 727 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 728 dest[i].counter = attr->counter; 729 i++; 730 } 731 732 if (attr->outer_match_level != MLX5_MATCH_NONE) 733 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 734 if (attr->inner_match_level != MLX5_MATCH_NONE) 735 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 736 737 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 738 flow_act.modify_hdr = attr->modify_hdr; 739 740 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 741 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 742 esw_setup_meter(attr, &flow_act); 743 744 if (split) { 745 fwd_attr.chain = attr->chain; 746 fwd_attr.prio = attr->prio; 747 fwd_attr.vport = esw_attr->in_rep->vport; 748 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 749 750 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 751 } else { 752 if (attr->chain || attr->prio) 753 fdb = mlx5_chains_get_table(chains, attr->chain, 754 attr->prio, 0); 755 else 756 fdb = attr->ft; 757 758 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 759 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 760 esw_attr->in_mdev->priv.eswitch, 761 esw_attr->in_rep->vport); 762 } 763 if (IS_ERR(fdb)) { 764 rule = ERR_CAST(fdb); 765 goto err_esw_get; 766 } 767 768 if (!i) { 769 kfree(dest); 770 dest = NULL; 771 } 772 773 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 774 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 775 &flow_act, dest, i); 776 else 777 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 778 if (IS_ERR(rule)) 779 goto err_add_rule; 780 else 781 atomic64_inc(&esw->offloads.num_flows); 782 783 kfree(dest); 784 return rule; 785 786 err_add_rule: 787 if (split) 788 mlx5_esw_vporttbl_put(esw, &fwd_attr); 789 else if (attr->chain || attr->prio) 790 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 791 err_esw_get: 792 esw_cleanup_dests(esw, attr); 793 err_create_goto_table: 794 kfree(dest); 795 return rule; 796 } 797 798 struct mlx5_flow_handle * 799 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 800 struct mlx5_flow_spec *spec, 801 struct mlx5_flow_attr *attr) 802 { 803 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 804 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 805 struct mlx5_fs_chains *chains = esw_chains(esw); 806 struct mlx5_vport_tbl_attr fwd_attr; 807 struct mlx5_flow_destination *dest; 808 struct mlx5_flow_table *fast_fdb; 809 struct mlx5_flow_table *fwd_fdb; 810 struct mlx5_flow_handle *rule; 811 int i, err = 0; 812 813 dest = kzalloc_objs(*dest, MLX5_MAX_FLOW_FWD_VPORTS + 1); 814 if (!dest) 815 return ERR_PTR(-ENOMEM); 816 817 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 818 if (IS_ERR(fast_fdb)) { 819 rule = ERR_CAST(fast_fdb); 820 goto err_get_fast; 821 } 822 823 fwd_attr.chain = attr->chain; 824 fwd_attr.prio = attr->prio; 825 fwd_attr.vport = esw_attr->in_rep->vport; 826 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 827 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 828 if (IS_ERR(fwd_fdb)) { 829 rule = ERR_CAST(fwd_fdb); 830 goto err_get_fwd; 831 } 832 833 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 834 for (i = 0; i < esw_attr->split_count; i++) { 835 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 836 /* Source port rewrite (forward to ovs internal port or statck device) isn't 837 * supported in the rule of split action. 838 */ 839 err = -EOPNOTSUPP; 840 else 841 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 842 843 if (err) { 844 rule = ERR_PTR(err); 845 goto err_chain_src_rewrite; 846 } 847 } 848 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 849 dest[i].ft = fwd_fdb; 850 i++; 851 852 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 853 esw_attr->in_mdev->priv.eswitch, 854 esw_attr->in_rep->vport); 855 856 if (attr->outer_match_level != MLX5_MATCH_NONE) 857 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 858 859 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 860 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 861 862 if (IS_ERR(rule)) { 863 i = esw_attr->split_count; 864 goto err_chain_src_rewrite; 865 } 866 867 atomic64_inc(&esw->offloads.num_flows); 868 869 kfree(dest); 870 return rule; 871 err_chain_src_rewrite: 872 mlx5_esw_vporttbl_put(esw, &fwd_attr); 873 err_get_fwd: 874 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 875 err_get_fast: 876 kfree(dest); 877 return rule; 878 } 879 880 static void 881 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 882 struct mlx5_flow_handle *rule, 883 struct mlx5_flow_attr *attr, 884 bool fwd_rule) 885 { 886 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 887 struct mlx5_fs_chains *chains = esw_chains(esw); 888 bool split = (esw_attr->split_count > 0); 889 struct mlx5_vport_tbl_attr fwd_attr; 890 int i; 891 892 mlx5_del_flow_rules(rule); 893 894 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 895 /* unref the term table */ 896 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 897 if (esw_attr->dests[i].termtbl) 898 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 899 } 900 } 901 902 atomic64_dec(&esw->offloads.num_flows); 903 904 if (fwd_rule || split) { 905 fwd_attr.chain = attr->chain; 906 fwd_attr.prio = attr->prio; 907 fwd_attr.vport = esw_attr->in_rep->vport; 908 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 909 } 910 911 if (fwd_rule) { 912 mlx5_esw_vporttbl_put(esw, &fwd_attr); 913 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 914 } else { 915 if (split) 916 mlx5_esw_vporttbl_put(esw, &fwd_attr); 917 else if (attr->chain || attr->prio) 918 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 919 esw_cleanup_dests(esw, attr); 920 } 921 } 922 923 void 924 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 925 struct mlx5_flow_handle *rule, 926 struct mlx5_flow_attr *attr) 927 { 928 __mlx5_eswitch_del_rule(esw, rule, attr, false); 929 } 930 931 void 932 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 933 struct mlx5_flow_handle *rule, 934 struct mlx5_flow_attr *attr) 935 { 936 __mlx5_eswitch_del_rule(esw, rule, attr, true); 937 } 938 939 struct mlx5_flow_handle * 940 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 941 struct mlx5_eswitch *from_esw, 942 struct mlx5_eswitch_rep *rep, 943 u32 sqn) 944 { 945 struct mlx5_flow_act flow_act = {0}; 946 struct mlx5_flow_destination dest = {}; 947 struct mlx5_flow_handle *flow_rule; 948 struct mlx5_flow_spec *spec; 949 void *misc; 950 u16 vport; 951 952 spec = kvzalloc_obj(*spec); 953 if (!spec) { 954 flow_rule = ERR_PTR(-ENOMEM); 955 goto out; 956 } 957 958 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 959 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 960 961 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 962 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 963 964 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 965 966 /* source vport is the esw manager */ 967 vport = from_esw->manager_vport; 968 969 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) { 970 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 971 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 972 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport)); 973 974 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 975 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 976 mlx5_eswitch_get_vport_metadata_mask()); 977 978 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 979 } else { 980 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 981 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 982 983 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 984 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 985 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 986 987 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 988 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 989 990 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 991 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 992 source_eswitch_owner_vhca_id); 993 994 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 995 } 996 997 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 998 dest.vport.num = rep->vport; 999 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 1000 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1001 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1002 1003 if (rep->vport == MLX5_VPORT_UPLINK && 1004 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) { 1005 dest.ft = on_esw->offloads.ft_ipsec_tx_pol; 1006 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; 1007 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1008 } else { 1009 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1010 dest.vport.num = rep->vport; 1011 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 1012 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1013 } 1014 1015 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) && 1016 rep->vport == MLX5_VPORT_UPLINK) 1017 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 1018 1019 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), 1020 spec, &flow_act, &dest, 1); 1021 if (IS_ERR(flow_rule)) 1022 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %pe\n", 1023 flow_rule); 1024 out: 1025 kvfree(spec); 1026 return flow_rule; 1027 } 1028 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1029 1030 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1031 { 1032 mlx5_del_flow_rules(rule); 1033 } 1034 1035 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) 1036 { 1037 if (rule) 1038 mlx5_del_flow_rules(rule); 1039 } 1040 1041 struct mlx5_flow_handle * 1042 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) 1043 { 1044 struct mlx5_flow_destination dest = {}; 1045 struct mlx5_flow_act flow_act = {0}; 1046 struct mlx5_flow_handle *flow_rule; 1047 struct mlx5_flow_spec *spec; 1048 1049 spec = kvzalloc_obj(*spec); 1050 if (!spec) 1051 return ERR_PTR(-ENOMEM); 1052 1053 MLX5_SET(fte_match_param, spec->match_criteria, 1054 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1055 MLX5_SET(fte_match_param, spec->match_criteria, 1056 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1057 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1058 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1059 1060 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1061 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1062 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1063 1064 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1065 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1066 dest.vport.num = vport_num; 1067 1068 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1069 spec, &flow_act, &dest, 1); 1070 if (IS_ERR(flow_rule)) 1071 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %pe\n", 1072 vport_num, flow_rule); 1073 1074 kvfree(spec); 1075 return flow_rule; 1076 } 1077 1078 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1079 { 1080 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1081 MLX5_FDB_TO_VPORT_REG_C_1; 1082 } 1083 1084 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1085 { 1086 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1087 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1088 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1089 u8 curr, wanted; 1090 int err; 1091 1092 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1093 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1094 return 0; 1095 1096 MLX5_SET(query_esw_vport_context_in, in, opcode, 1097 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1098 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1099 if (err) 1100 return err; 1101 1102 curr = MLX5_GET(query_esw_vport_context_out, out, 1103 esw_vport_context.fdb_to_vport_reg_c_id); 1104 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1105 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1106 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1107 1108 if (enable) 1109 curr |= wanted; 1110 else 1111 curr &= ~wanted; 1112 1113 MLX5_SET(modify_esw_vport_context_in, min, 1114 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1115 MLX5_SET(modify_esw_vport_context_in, min, 1116 field_select.fdb_to_vport_reg_c_id, 1); 1117 1118 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1119 if (!err) { 1120 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1121 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1122 else 1123 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1124 } 1125 1126 return err; 1127 } 1128 1129 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1130 struct mlx5_core_dev *peer_dev, 1131 struct mlx5_flow_spec *spec, 1132 struct mlx5_flow_destination *dest) 1133 { 1134 void *misc; 1135 1136 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1137 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1138 misc_parameters_2); 1139 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1140 mlx5_eswitch_get_vport_metadata_mask()); 1141 1142 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1143 } else { 1144 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1145 misc_parameters); 1146 1147 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1148 MLX5_CAP_GEN(peer_dev, vhca_id)); 1149 1150 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1151 1152 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1153 misc_parameters); 1154 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1155 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1156 source_eswitch_owner_vhca_id); 1157 } 1158 1159 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1160 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1161 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1162 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1163 } 1164 1165 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1166 struct mlx5_eswitch *peer_esw, 1167 struct mlx5_flow_spec *spec, 1168 u16 vport) 1169 { 1170 void *misc; 1171 1172 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1173 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1174 misc_parameters_2); 1175 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1176 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1177 vport)); 1178 } else { 1179 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1180 misc_parameters); 1181 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1182 } 1183 } 1184 1185 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1186 struct mlx5_core_dev *peer_dev) 1187 { 1188 struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch; 1189 struct mlx5_flow_destination dest = {}; 1190 struct mlx5_flow_act flow_act = {0}; 1191 struct mlx5_flow_handle **flows; 1192 struct mlx5_flow_handle *flow; 1193 struct mlx5_vport *peer_vport; 1194 struct mlx5_flow_spec *spec; 1195 int err; 1196 unsigned long i; 1197 void *misc; 1198 1199 if (!MLX5_VPORT_MANAGER(peer_dev) && 1200 !mlx5_core_is_ecpf_esw_manager(peer_dev)) 1201 return 0; 1202 1203 spec = kvzalloc_obj(*spec); 1204 if (!spec) 1205 return -ENOMEM; 1206 1207 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1208 1209 flows = kvzalloc_objs(*flows, peer_esw->total_vports); 1210 if (!flows) { 1211 err = -ENOMEM; 1212 goto alloc_flows_err; 1213 } 1214 1215 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1216 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1217 misc_parameters); 1218 1219 if (mlx5_core_is_ecpf_esw_manager(peer_dev) && 1220 mlx5_esw_host_functions_enabled(peer_dev)) { 1221 peer_vport = mlx5_eswitch_get_vport(peer_esw, 1222 MLX5_VPORT_HOST_PF); 1223 esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, 1224 MLX5_VPORT_HOST_PF); 1225 1226 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1227 spec, &flow_act, &dest, 1); 1228 if (IS_ERR(flow)) { 1229 err = PTR_ERR(flow); 1230 goto add_pf_flow_err; 1231 } 1232 flows[peer_vport->index] = flow; 1233 } 1234 1235 mlx5_esw_for_each_spf_vport(peer_esw, i, peer_vport) { 1236 esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, 1237 peer_vport->vport); 1238 1239 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1240 spec, &flow_act, &dest, 1); 1241 if (IS_ERR(flow)) { 1242 err = PTR_ERR(flow); 1243 goto add_ecpf_flow_err; 1244 } 1245 flows[peer_vport->index] = flow; 1246 } 1247 1248 if (mlx5_ecpf_vport_exists(peer_dev)) { 1249 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); 1250 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1251 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1252 spec, &flow_act, &dest, 1); 1253 if (IS_ERR(flow)) { 1254 err = PTR_ERR(flow); 1255 goto add_ecpf_flow_err; 1256 } 1257 flows[peer_vport->index] = flow; 1258 } 1259 1260 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, 1261 mlx5_core_max_vfs(peer_dev)) { 1262 esw_set_peer_miss_rule_source_port(esw, peer_esw, spec, 1263 peer_vport->vport); 1264 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1265 spec, &flow_act, &dest, 1); 1266 if (IS_ERR(flow)) { 1267 err = PTR_ERR(flow); 1268 goto add_vf_flow_err; 1269 } 1270 flows[peer_vport->index] = flow; 1271 } 1272 1273 if (mlx5_core_ec_sriov_enabled(peer_dev)) { 1274 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, 1275 mlx5_core_max_ec_vfs(peer_dev)) { 1276 esw_set_peer_miss_rule_source_port(esw, peer_esw, 1277 spec, 1278 peer_vport->vport); 1279 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1280 spec, &flow_act, &dest, 1); 1281 if (IS_ERR(flow)) { 1282 err = PTR_ERR(flow); 1283 goto add_ec_vf_flow_err; 1284 } 1285 flows[peer_vport->index] = flow; 1286 } 1287 } 1288 1289 err = xa_insert(&esw->fdb_table.offloads.peer_miss_rules, 1290 MLX5_CAP_GEN(peer_dev, vhca_id), flows, GFP_KERNEL); 1291 if (err) 1292 goto add_ec_vf_flow_err; 1293 1294 kvfree(spec); 1295 return 0; 1296 1297 add_ec_vf_flow_err: 1298 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, 1299 mlx5_core_max_ec_vfs(peer_dev)) { 1300 if (!flows[peer_vport->index]) 1301 continue; 1302 mlx5_del_flow_rules(flows[peer_vport->index]); 1303 } 1304 add_vf_flow_err: 1305 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, 1306 mlx5_core_max_vfs(peer_dev)) { 1307 if (!flows[peer_vport->index]) 1308 continue; 1309 mlx5_del_flow_rules(flows[peer_vport->index]); 1310 } 1311 if (mlx5_ecpf_vport_exists(peer_dev)) { 1312 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); 1313 mlx5_del_flow_rules(flows[peer_vport->index]); 1314 } 1315 add_ecpf_flow_err: 1316 mlx5_esw_for_each_spf_vport(peer_esw, i, peer_vport) { 1317 if (!flows[peer_vport->index]) 1318 continue; 1319 mlx5_del_flow_rules(flows[peer_vport->index]); 1320 } 1321 if (mlx5_core_is_ecpf_esw_manager(peer_dev) && 1322 mlx5_esw_host_functions_enabled(peer_dev)) { 1323 peer_vport = mlx5_eswitch_get_vport(peer_esw, 1324 MLX5_VPORT_HOST_PF); 1325 mlx5_del_flow_rules(flows[peer_vport->index]); 1326 } 1327 add_pf_flow_err: 1328 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1329 kvfree(flows); 1330 alloc_flows_err: 1331 kvfree(spec); 1332 return err; 1333 } 1334 1335 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1336 struct mlx5_core_dev *peer_dev) 1337 { 1338 struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch; 1339 u16 peer_vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1340 struct mlx5_flow_handle **flows; 1341 struct mlx5_vport *peer_vport; 1342 unsigned long i; 1343 1344 flows = xa_erase(&esw->fdb_table.offloads.peer_miss_rules, 1345 peer_vhca_id); 1346 if (!flows) 1347 return; 1348 1349 if (mlx5_core_ec_sriov_enabled(peer_dev)) { 1350 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport, 1351 mlx5_core_max_ec_vfs(peer_dev)) 1352 mlx5_del_flow_rules(flows[peer_vport->index]); 1353 } 1354 1355 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport, 1356 mlx5_core_max_vfs(peer_dev)) 1357 mlx5_del_flow_rules(flows[peer_vport->index]); 1358 1359 if (mlx5_ecpf_vport_exists(peer_dev)) { 1360 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF); 1361 mlx5_del_flow_rules(flows[peer_vport->index]); 1362 } 1363 1364 mlx5_esw_for_each_spf_vport(peer_esw, i, peer_vport) 1365 mlx5_del_flow_rules(flows[peer_vport->index]); 1366 1367 if (mlx5_core_is_ecpf_esw_manager(peer_dev) && 1368 mlx5_esw_host_functions_enabled(peer_dev)) { 1369 peer_vport = mlx5_eswitch_get_vport(peer_esw, 1370 MLX5_VPORT_HOST_PF); 1371 mlx5_del_flow_rules(flows[peer_vport->index]); 1372 } 1373 1374 kvfree(flows); 1375 } 1376 1377 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1378 { 1379 struct mlx5_flow_act flow_act = {0}; 1380 struct mlx5_flow_destination dest = {}; 1381 struct mlx5_flow_handle *flow_rule = NULL; 1382 struct mlx5_flow_spec *spec; 1383 void *headers_c; 1384 void *headers_v; 1385 int err = 0; 1386 u8 *dmac_c; 1387 u8 *dmac_v; 1388 1389 spec = kvzalloc_obj(*spec); 1390 if (!spec) { 1391 err = -ENOMEM; 1392 goto out; 1393 } 1394 1395 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1396 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1397 outer_headers); 1398 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1399 outer_headers.dmac_47_16); 1400 dmac_c[0] = 0x01; 1401 1402 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1403 dest.vport.num = esw->manager_vport; 1404 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1405 1406 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1407 spec, &flow_act, &dest, 1); 1408 if (IS_ERR(flow_rule)) { 1409 err = PTR_ERR(flow_rule); 1410 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1411 goto out; 1412 } 1413 1414 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1415 1416 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1417 outer_headers); 1418 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1419 outer_headers.dmac_47_16); 1420 dmac_v[0] = 0x01; 1421 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1422 spec, &flow_act, &dest, 1); 1423 if (IS_ERR(flow_rule)) { 1424 err = PTR_ERR(flow_rule); 1425 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1426 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1427 goto out; 1428 } 1429 1430 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1431 1432 out: 1433 kvfree(spec); 1434 return err; 1435 } 1436 1437 struct mlx5_flow_handle * 1438 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1439 { 1440 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1441 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1442 struct mlx5_flow_context *flow_context; 1443 struct mlx5_flow_handle *flow_rule; 1444 struct mlx5_flow_destination dest; 1445 struct mlx5_flow_spec *spec; 1446 void *misc; 1447 1448 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1449 return ERR_PTR(-EOPNOTSUPP); 1450 1451 spec = kvzalloc_obj(*spec); 1452 if (!spec) 1453 return ERR_PTR(-ENOMEM); 1454 1455 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1456 misc_parameters_2); 1457 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1458 ESW_REG_C0_USER_DATA_METADATA_MASK); 1459 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1460 misc_parameters_2); 1461 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1462 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1463 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1464 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1465 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1466 1467 flow_context = &spec->flow_context; 1468 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1469 flow_context->flow_tag = tag; 1470 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1471 dest.ft = esw->offloads.ft_offloads; 1472 1473 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1474 kvfree(spec); 1475 1476 if (IS_ERR(flow_rule)) 1477 esw_warn(esw->dev, 1478 "Failed to create restore rule for tag: %d, err(%d)\n", 1479 tag, (int)PTR_ERR(flow_rule)); 1480 1481 return flow_rule; 1482 } 1483 1484 struct mlx5_flow_group * 1485 mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw, 1486 struct mlx5_flow_table *ft) 1487 { 1488 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1489 struct mlx5_flow_group *fg; 1490 void *match_criteria; 1491 void *flow_group_in; 1492 1493 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 1494 return ERR_PTR(-EOPNOTSUPP); 1495 1496 if (IS_ERR(ft)) 1497 return ERR_CAST(ft); 1498 1499 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1500 if (!flow_group_in) 1501 return ERR_PTR(-ENOMEM); 1502 1503 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1504 match_criteria); 1505 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1506 MLX5_MATCH_MISC_PARAMETERS_2); 1507 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1508 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1509 ft->max_fte - 1); 1510 1511 MLX5_SET(fte_match_param, match_criteria, 1512 misc_parameters_2.metadata_reg_c_0, 1513 mlx5_eswitch_get_vport_metadata_mask()); 1514 1515 fg = mlx5_create_flow_group(ft, flow_group_in); 1516 kvfree(flow_group_in); 1517 if (IS_ERR(fg)) 1518 esw_warn(esw->dev, "Can't create LAG demux flow group\n"); 1519 1520 return fg; 1521 } 1522 1523 struct mlx5_flow_handle * 1524 mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num, 1525 struct mlx5_flow_table *lag_ft) 1526 { 1527 struct mlx5_flow_spec *spec = kvzalloc_obj(*spec); 1528 struct mlx5_flow_destination dest = {}; 1529 struct mlx5_flow_act flow_act = {}; 1530 struct mlx5_flow_handle *ret; 1531 void *misc; 1532 1533 if (!spec) 1534 return ERR_PTR(-ENOMEM); 1535 1536 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1537 kvfree(spec); 1538 return ERR_PTR(-EOPNOTSUPP); 1539 } 1540 1541 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1542 misc_parameters_2); 1543 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1544 mlx5_eswitch_get_vport_metadata_mask()); 1545 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1546 1547 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1548 misc_parameters_2); 1549 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1550 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1551 1552 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1553 dest.type = MLX5_FLOW_DESTINATION_TYPE_VHCA_RX; 1554 dest.vhca.id = MLX5_CAP_GEN(esw->dev, vhca_id); 1555 1556 ret = mlx5_add_flow_rules(lag_ft, spec, &flow_act, &dest, 1); 1557 kvfree(spec); 1558 return ret; 1559 } 1560 1561 #define MAX_PF_SQ 256 1562 #define MAX_SQ_NVPORTS 32 1563 1564 void 1565 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1566 u32 *flow_group_in, 1567 int match_params) 1568 { 1569 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1570 flow_group_in, 1571 match_criteria); 1572 1573 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1574 MLX5_SET(create_flow_group_in, flow_group_in, 1575 match_criteria_enable, 1576 MLX5_MATCH_MISC_PARAMETERS_2 | match_params); 1577 1578 MLX5_SET(fte_match_param, match_criteria, 1579 misc_parameters_2.metadata_reg_c_0, 1580 mlx5_eswitch_get_vport_metadata_mask()); 1581 } else { 1582 MLX5_SET(create_flow_group_in, flow_group_in, 1583 match_criteria_enable, 1584 MLX5_MATCH_MISC_PARAMETERS | match_params); 1585 1586 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1587 misc_parameters.source_port); 1588 } 1589 } 1590 1591 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1592 static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1593 { 1594 struct mlx5_vport_tbl_attr attr; 1595 struct mlx5_vport *vport; 1596 unsigned long i; 1597 1598 attr.chain = 0; 1599 attr.prio = 1; 1600 mlx5_esw_for_each_vport(esw, i, vport) { 1601 attr.vport = vport->vport; 1602 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1603 mlx5_esw_vporttbl_put(esw, &attr); 1604 } 1605 } 1606 1607 static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1608 { 1609 struct mlx5_vport_tbl_attr attr; 1610 struct mlx5_flow_table *fdb; 1611 struct mlx5_vport *vport; 1612 unsigned long i; 1613 1614 attr.chain = 0; 1615 attr.prio = 1; 1616 mlx5_esw_for_each_vport(esw, i, vport) { 1617 attr.vport = vport->vport; 1618 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1619 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1620 if (IS_ERR(fdb)) 1621 goto out; 1622 } 1623 return 0; 1624 1625 out: 1626 esw_vport_tbl_put(esw); 1627 return PTR_ERR(fdb); 1628 } 1629 1630 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1631 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1632 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1633 { 1634 struct mlx5_core_dev *dev = esw->dev; 1635 1636 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1637 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1638 1639 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1640 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1641 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1642 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1643 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1644 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1645 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1646 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1647 /* Disabled when ttl workaround is needed, e.g 1648 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1649 */ 1650 esw_warn(dev, 1651 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1652 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1653 } else { 1654 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1655 esw_info(dev, "Supported tc chains and prios offload\n"); 1656 } 1657 1658 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1659 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1660 } 1661 1662 static int 1663 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1664 { 1665 struct mlx5_core_dev *dev = esw->dev; 1666 struct mlx5_flow_table *nf_ft, *ft; 1667 struct mlx5_chains_attr attr = {}; 1668 struct mlx5_fs_chains *chains; 1669 int err; 1670 1671 esw_init_chains_offload_flags(esw, &attr.flags); 1672 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1673 attr.max_grp_num = esw->params.large_group_num; 1674 attr.default_ft = miss_fdb; 1675 attr.mapping = esw->offloads.reg_c0_obj_pool; 1676 attr.fs_base_prio = FDB_BYPASS_PATH; 1677 1678 chains = mlx5_chains_create(dev, &attr); 1679 if (IS_ERR(chains)) { 1680 err = PTR_ERR(chains); 1681 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1682 return err; 1683 } 1684 mlx5_chains_print_info(chains); 1685 1686 esw->fdb_table.offloads.esw_chains_priv = chains; 1687 1688 /* Create tc_end_ft which is the always created ft chain */ 1689 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1690 1, 0); 1691 if (IS_ERR(nf_ft)) { 1692 err = PTR_ERR(nf_ft); 1693 goto nf_ft_err; 1694 } 1695 1696 /* Always open the root for fast path */ 1697 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1698 if (IS_ERR(ft)) { 1699 err = PTR_ERR(ft); 1700 goto level_0_err; 1701 } 1702 1703 /* Open level 1 for split fdb rules now if prios isn't supported */ 1704 if (!mlx5_chains_prios_supported(chains)) { 1705 err = esw_vport_tbl_get(esw); 1706 if (err) 1707 goto level_1_err; 1708 } 1709 1710 mlx5_chains_set_end_ft(chains, nf_ft); 1711 1712 return 0; 1713 1714 level_1_err: 1715 mlx5_chains_put_table(chains, 0, 1, 0); 1716 level_0_err: 1717 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1718 nf_ft_err: 1719 mlx5_chains_destroy(chains); 1720 esw->fdb_table.offloads.esw_chains_priv = NULL; 1721 1722 return err; 1723 } 1724 1725 static void 1726 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1727 { 1728 if (!mlx5_chains_prios_supported(chains)) 1729 esw_vport_tbl_put(esw); 1730 mlx5_chains_put_table(chains, 0, 1, 0); 1731 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1732 mlx5_chains_destroy(chains); 1733 } 1734 1735 #else /* CONFIG_MLX5_CLS_ACT */ 1736 1737 static int 1738 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1739 { return 0; } 1740 1741 static void 1742 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1743 {} 1744 1745 #endif 1746 1747 static int 1748 esw_create_send_to_vport_group(struct mlx5_eswitch *esw, 1749 struct mlx5_flow_table *fdb, 1750 u32 *flow_group_in, 1751 int *ix) 1752 { 1753 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1754 struct mlx5_flow_group *g; 1755 void *match_criteria; 1756 int count, err = 0; 1757 1758 memset(flow_group_in, 0, inlen); 1759 1760 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS); 1761 1762 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1763 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1764 1765 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) && 1766 MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1767 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1768 misc_parameters.source_eswitch_owner_vhca_id); 1769 MLX5_SET(create_flow_group_in, flow_group_in, 1770 source_eswitch_owner_vhca_id_valid, 1); 1771 } 1772 1773 /* See comment at table_size calculation */ 1774 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1775 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1776 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); 1777 *ix += count; 1778 1779 g = mlx5_create_flow_group(fdb, flow_group_in); 1780 if (IS_ERR(g)) { 1781 err = PTR_ERR(g); 1782 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1783 goto out; 1784 } 1785 esw->fdb_table.offloads.send_to_vport_grp = g; 1786 1787 out: 1788 return err; 1789 } 1790 1791 static int 1792 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, 1793 struct mlx5_flow_table *fdb, 1794 u32 *flow_group_in, 1795 int *ix) 1796 { 1797 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1798 struct mlx5_flow_group *g; 1799 void *match_criteria; 1800 int err = 0; 1801 1802 if (!esw_src_port_rewrite_supported(esw)) 1803 return 0; 1804 1805 memset(flow_group_in, 0, inlen); 1806 1807 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1808 MLX5_MATCH_MISC_PARAMETERS_2); 1809 1810 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1811 1812 MLX5_SET(fte_match_param, match_criteria, 1813 misc_parameters_2.metadata_reg_c_0, 1814 mlx5_eswitch_get_vport_metadata_mask()); 1815 MLX5_SET(fte_match_param, match_criteria, 1816 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1817 1818 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1819 MLX5_SET(create_flow_group_in, flow_group_in, 1820 end_flow_index, *ix + esw->total_vports - 1); 1821 *ix += esw->total_vports; 1822 1823 g = mlx5_create_flow_group(fdb, flow_group_in); 1824 if (IS_ERR(g)) { 1825 err = PTR_ERR(g); 1826 esw_warn(esw->dev, 1827 "Failed to create send-to-vport meta flow group err(%d)\n", err); 1828 goto send_vport_meta_err; 1829 } 1830 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1831 1832 return 0; 1833 1834 send_vport_meta_err: 1835 return err; 1836 } 1837 1838 static int 1839 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, 1840 struct mlx5_flow_table *fdb, 1841 u32 *flow_group_in, 1842 int *ix) 1843 { 1844 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1); 1845 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1846 struct mlx5_flow_group *g; 1847 void *match_criteria; 1848 int err = 0; 1849 1850 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1851 return 0; 1852 1853 memset(flow_group_in, 0, inlen); 1854 1855 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 1856 1857 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1858 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1859 flow_group_in, 1860 match_criteria); 1861 1862 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1863 misc_parameters.source_eswitch_owner_vhca_id); 1864 1865 MLX5_SET(create_flow_group_in, flow_group_in, 1866 source_eswitch_owner_vhca_id_valid, 1); 1867 } 1868 1869 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1870 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1871 *ix + max_peer_ports); 1872 *ix += max_peer_ports + 1; 1873 1874 g = mlx5_create_flow_group(fdb, flow_group_in); 1875 if (IS_ERR(g)) { 1876 err = PTR_ERR(g); 1877 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); 1878 goto out; 1879 } 1880 esw->fdb_table.offloads.peer_miss_grp = g; 1881 1882 out: 1883 return err; 1884 } 1885 1886 static int 1887 esw_create_miss_group(struct mlx5_eswitch *esw, 1888 struct mlx5_flow_table *fdb, 1889 u32 *flow_group_in, 1890 int *ix) 1891 { 1892 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1893 struct mlx5_flow_group *g; 1894 void *match_criteria; 1895 int err = 0; 1896 u8 *dmac; 1897 1898 memset(flow_group_in, 0, inlen); 1899 1900 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1901 MLX5_MATCH_OUTER_HEADERS); 1902 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1903 match_criteria); 1904 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1905 outer_headers.dmac_47_16); 1906 dmac[0] = 0x01; 1907 1908 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1909 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1910 *ix + MLX5_ESW_MISS_FLOWS); 1911 1912 g = mlx5_create_flow_group(fdb, flow_group_in); 1913 if (IS_ERR(g)) { 1914 err = PTR_ERR(g); 1915 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); 1916 goto miss_err; 1917 } 1918 esw->fdb_table.offloads.miss_grp = g; 1919 1920 err = esw_add_fdb_miss_rule(esw); 1921 if (err) 1922 goto miss_rule_err; 1923 1924 return 0; 1925 1926 miss_rule_err: 1927 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1928 miss_err: 1929 return err; 1930 } 1931 1932 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1933 { 1934 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1935 struct mlx5_flow_table_attr ft_attr = {}; 1936 struct mlx5_core_dev *dev = esw->dev; 1937 struct mlx5_flow_namespace *root_ns; 1938 struct mlx5_flow_table *fdb = NULL; 1939 int table_size, ix = 0, err = 0; 1940 u32 flags = 0, *flow_group_in; 1941 1942 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1943 1944 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1945 if (!flow_group_in) 1946 return -ENOMEM; 1947 1948 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1949 if (!root_ns) { 1950 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1951 err = -EOPNOTSUPP; 1952 goto ns_err; 1953 } 1954 esw->fdb_table.offloads.ns = root_ns; 1955 err = mlx5_flow_namespace_set_mode(root_ns, 1956 esw->dev->priv.steering->mode); 1957 if (err) { 1958 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1959 goto ns_err; 1960 } 1961 1962 /* To be strictly correct: 1963 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1964 * should be: 1965 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1966 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1967 * but as the peer device might not be in switchdev mode it's not 1968 * possible. We use the fact that by default FW sets max vfs and max sfs 1969 * to the same value on both devices. If it needs to be changed in the future note 1970 * the peer miss group should also be created based on the number of 1971 * total vports of the peer (currently is also uses esw->total_vports). 1972 */ 1973 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1974 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS; 1975 1976 /* create the slow path fdb with encap set, so further table instances 1977 * can be created at run time while VFs are probed if the FW allows that. 1978 */ 1979 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1980 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1981 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1982 1983 ft_attr.flags = flags; 1984 ft_attr.max_fte = table_size; 1985 ft_attr.prio = FDB_SLOW_PATH; 1986 1987 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1988 if (IS_ERR(fdb)) { 1989 err = PTR_ERR(fdb); 1990 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1991 goto slow_fdb_err; 1992 } 1993 esw->fdb_table.offloads.slow_fdb = fdb; 1994 1995 /* Create empty TC-miss managed table. This allows plugging in following 1996 * priorities without directly exposing their level 0 table to 1997 * eswitch_offloads and passing it as miss_fdb to following call to 1998 * esw_chains_create(). 1999 */ 2000 memset(&ft_attr, 0, sizeof(ft_attr)); 2001 ft_attr.prio = FDB_TC_MISS; 2002 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 2003 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 2004 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 2005 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 2006 goto tc_miss_table_err; 2007 } 2008 2009 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 2010 if (err) { 2011 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 2012 goto fdb_chains_err; 2013 } 2014 2015 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); 2016 if (err) 2017 goto send_vport_err; 2018 2019 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); 2020 if (err) 2021 goto send_vport_meta_err; 2022 2023 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); 2024 if (err) 2025 goto peer_miss_err; 2026 2027 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); 2028 if (err) 2029 goto miss_err; 2030 2031 kvfree(flow_group_in); 2032 return 0; 2033 2034 miss_err: 2035 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2036 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 2037 peer_miss_err: 2038 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 2039 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 2040 send_vport_meta_err: 2041 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 2042 send_vport_err: 2043 esw_chains_destroy(esw, esw_chains(esw)); 2044 fdb_chains_err: 2045 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 2046 tc_miss_table_err: 2047 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 2048 slow_fdb_err: 2049 /* Holds true only as long as DMFS is the default */ 2050 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 2051 ns_err: 2052 kvfree(flow_group_in); 2053 return err; 2054 } 2055 2056 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 2057 { 2058 if (!mlx5_eswitch_get_slow_fdb(esw)) 2059 return; 2060 2061 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 2062 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 2063 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 2064 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 2065 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 2066 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 2067 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2068 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 2069 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 2070 2071 esw_chains_destroy(esw, esw_chains(esw)); 2072 2073 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 2074 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 2075 /* Holds true only as long as DMFS is the default */ 2076 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 2077 MLX5_FLOW_STEERING_MODE_DMFS); 2078 } 2079 2080 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) 2081 { 2082 int nvports; 2083 2084 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 2085 if (mlx5e_tc_int_port_supported(esw)) 2086 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 2087 2088 return nvports; 2089 } 2090 2091 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 2092 { 2093 struct mlx5_flow_table_attr ft_attr = {}; 2094 struct mlx5_core_dev *dev = esw->dev; 2095 struct mlx5_flow_table *ft_offloads; 2096 struct mlx5_flow_namespace *ns; 2097 int err = 0; 2098 2099 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2100 if (!ns) { 2101 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2102 return -EOPNOTSUPP; 2103 } 2104 2105 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + 2106 MLX5_ESW_FT_OFFLOADS_DROP_RULE; 2107 ft_attr.prio = 1; 2108 2109 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 2110 if (IS_ERR(ft_offloads)) { 2111 err = PTR_ERR(ft_offloads); 2112 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 2113 return err; 2114 } 2115 2116 esw->offloads.ft_offloads = ft_offloads; 2117 return 0; 2118 } 2119 2120 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 2121 { 2122 struct mlx5_esw_offload *offloads = &esw->offloads; 2123 2124 mlx5_destroy_flow_table(offloads->ft_offloads); 2125 } 2126 2127 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 2128 { 2129 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2130 struct mlx5_flow_group *g; 2131 u32 *flow_group_in; 2132 int nvports; 2133 int err = 0; 2134 2135 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); 2136 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2137 if (!flow_group_in) 2138 return -ENOMEM; 2139 2140 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 2141 2142 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2143 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 2144 2145 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2146 2147 if (IS_ERR(g)) { 2148 err = PTR_ERR(g); 2149 esw_warn(esw->dev, "Failed to create vport rx group err %d\n", 2150 err); 2151 goto out; 2152 } 2153 2154 esw->offloads.vport_rx_group = g; 2155 out: 2156 kvfree(flow_group_in); 2157 return err; 2158 } 2159 2160 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 2161 { 2162 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 2163 } 2164 2165 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) 2166 { 2167 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 2168 * for the drop rule, which is placed at the end of the table. 2169 * So return the total of vport and int_port as rule index. 2170 */ 2171 return esw_get_nr_ft_offloads_steering_src_ports(esw); 2172 } 2173 2174 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) 2175 { 2176 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2177 struct mlx5_flow_group *g; 2178 u32 *flow_group_in; 2179 int flow_index; 2180 int err = 0; 2181 2182 flow_index = esw_create_vport_rx_drop_rule_index(esw); 2183 2184 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2185 if (!flow_group_in) 2186 return -ENOMEM; 2187 2188 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); 2189 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); 2190 2191 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2192 2193 if (IS_ERR(g)) { 2194 err = PTR_ERR(g); 2195 esw_warn(esw->dev, 2196 "Failed to create vport rx drop group err %d\n", err); 2197 goto out; 2198 } 2199 2200 esw->offloads.vport_rx_drop_group = g; 2201 out: 2202 kvfree(flow_group_in); 2203 return err; 2204 } 2205 2206 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) 2207 { 2208 if (esw->offloads.vport_rx_drop_group) 2209 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); 2210 } 2211 2212 void 2213 mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw, 2214 u16 vport, 2215 struct mlx5_flow_spec *spec) 2216 { 2217 void *misc; 2218 2219 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 2220 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 2221 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2222 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 2223 2224 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 2225 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2226 mlx5_eswitch_get_vport_metadata_mask()); 2227 2228 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 2229 } else { 2230 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 2231 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 2232 2233 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2234 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2235 2236 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2237 } 2238 } 2239 2240 struct mlx5_flow_handle * 2241 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 2242 struct mlx5_flow_destination *dest) 2243 { 2244 struct mlx5_flow_act flow_act = {0}; 2245 struct mlx5_flow_handle *flow_rule; 2246 struct mlx5_flow_spec *spec; 2247 2248 spec = kvzalloc_obj(*spec); 2249 if (!spec) { 2250 flow_rule = ERR_PTR(-ENOMEM); 2251 goto out; 2252 } 2253 2254 mlx5_esw_set_spec_source_port(esw, vport, spec); 2255 2256 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2257 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 2258 &flow_act, dest, 1); 2259 if (IS_ERR(flow_rule)) { 2260 esw_warn(esw->dev, 2261 "fs offloads: Failed to add vport rx rule err %pe\n", 2262 flow_rule); 2263 goto out; 2264 } 2265 2266 out: 2267 kvfree(spec); 2268 return flow_rule; 2269 } 2270 2271 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2272 { 2273 struct mlx5_flow_act flow_act = {}; 2274 struct mlx5_flow_handle *flow_rule; 2275 2276 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2277 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, 2278 &flow_act, NULL, 0); 2279 if (IS_ERR(flow_rule)) { 2280 esw_warn(esw->dev, 2281 "fs offloads: Failed to add vport rx drop rule err %pe\n", 2282 flow_rule); 2283 return PTR_ERR(flow_rule); 2284 } 2285 2286 esw->offloads.vport_rx_drop_rule = flow_rule; 2287 2288 return 0; 2289 } 2290 2291 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2292 { 2293 if (esw->offloads.vport_rx_drop_rule) 2294 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); 2295 } 2296 2297 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 2298 { 2299 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 2300 struct mlx5_core_dev *dev = esw->dev; 2301 struct mlx5_vport *vport; 2302 unsigned long i; 2303 2304 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2305 return -EOPNOTSUPP; 2306 2307 if (!mlx5_esw_is_fdb_created(esw)) 2308 return -EOPNOTSUPP; 2309 2310 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2311 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2312 mlx5_mode = MLX5_INLINE_MODE_NONE; 2313 goto out; 2314 case MLX5_CAP_INLINE_MODE_L2: 2315 mlx5_mode = MLX5_INLINE_MODE_L2; 2316 goto out; 2317 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2318 goto query_vports; 2319 } 2320 2321 query_vports: 2322 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2323 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2324 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2325 if (prev_mlx5_mode != mlx5_mode) 2326 return -EINVAL; 2327 prev_mlx5_mode = mlx5_mode; 2328 } 2329 2330 out: 2331 *mode = mlx5_mode; 2332 return 0; 2333 } 2334 2335 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2336 { 2337 struct mlx5_esw_offload *offloads = &esw->offloads; 2338 2339 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2340 return; 2341 2342 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2343 mlx5_destroy_flow_group(offloads->restore_group); 2344 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2345 } 2346 2347 static int esw_create_restore_table(struct mlx5_eswitch *esw) 2348 { 2349 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2350 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2351 struct mlx5_flow_table_attr ft_attr = {}; 2352 struct mlx5_core_dev *dev = esw->dev; 2353 struct mlx5_flow_namespace *ns; 2354 struct mlx5_modify_hdr *mod_hdr; 2355 void *match_criteria, *misc; 2356 struct mlx5_flow_table *ft; 2357 struct mlx5_flow_group *g; 2358 u32 *flow_group_in; 2359 int err = 0; 2360 2361 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2362 return 0; 2363 2364 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2365 if (!ns) { 2366 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2367 return -EOPNOTSUPP; 2368 } 2369 2370 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2371 if (!flow_group_in) { 2372 err = -ENOMEM; 2373 goto out_free; 2374 } 2375 2376 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2377 ft = mlx5_create_flow_table(ns, &ft_attr); 2378 if (IS_ERR(ft)) { 2379 err = PTR_ERR(ft); 2380 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2381 err); 2382 goto out_free; 2383 } 2384 2385 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2386 match_criteria); 2387 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2388 misc_parameters_2); 2389 2390 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2391 ESW_REG_C0_USER_DATA_METADATA_MASK); 2392 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2393 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2394 ft_attr.max_fte - 1); 2395 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2396 MLX5_MATCH_MISC_PARAMETERS_2); 2397 g = mlx5_create_flow_group(ft, flow_group_in); 2398 if (IS_ERR(g)) { 2399 err = PTR_ERR(g); 2400 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2401 err); 2402 goto err_group; 2403 } 2404 2405 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2406 MLX5_SET(copy_action_in, modact, src_field, 2407 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2408 MLX5_SET(copy_action_in, modact, dst_field, 2409 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2410 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2411 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2412 modact); 2413 if (IS_ERR(mod_hdr)) { 2414 err = PTR_ERR(mod_hdr); 2415 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2416 err); 2417 goto err_mod_hdr; 2418 } 2419 2420 esw->offloads.ft_offloads_restore = ft; 2421 esw->offloads.restore_group = g; 2422 esw->offloads.restore_copy_hdr_id = mod_hdr; 2423 2424 kvfree(flow_group_in); 2425 2426 return 0; 2427 2428 err_mod_hdr: 2429 mlx5_destroy_flow_group(g); 2430 err_group: 2431 mlx5_destroy_flow_table(ft); 2432 out_free: 2433 kvfree(flow_group_in); 2434 2435 return err; 2436 } 2437 2438 static void mlx5_esw_assert_reps_locked(struct mlx5_eswitch *esw) 2439 { 2440 lockdep_assert_held(&esw->offloads.reps_lock); 2441 } 2442 2443 void mlx5_esw_reps_block(struct mlx5_eswitch *esw) 2444 { 2445 mutex_lock(&esw->offloads.reps_lock); 2446 } 2447 2448 static void mlx5_esw_reps_block_nested(struct mlx5_eswitch *esw) 2449 { 2450 mutex_lock_nested(&esw->offloads.reps_lock, SINGLE_DEPTH_NESTING); 2451 } 2452 2453 void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw) 2454 { 2455 mutex_unlock(&esw->offloads.reps_lock); 2456 } 2457 2458 static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode) 2459 { 2460 mlx5_esw_reps_unblock(esw); 2461 mlx5_devcom_comp_lock(esw->dev->priv.hca_devcom_comp); 2462 if (esw->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV || 2463 mlx5_core_mp_enabled(esw->dev)) { 2464 esw->mode = mode; 2465 goto out; 2466 } 2467 2468 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; 2469 mlx5_rescan_drivers_locked(esw->dev); 2470 esw->mode = mode; 2471 esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; 2472 out: 2473 mlx5_rescan_drivers_locked(esw->dev); 2474 mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp); 2475 mlx5_esw_reps_block(esw); 2476 } 2477 2478 static void mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch *esw) 2479 { 2480 if (!esw->fdb_table.offloads.drop_root) 2481 return; 2482 2483 esw_debug(esw->dev, "Destroying FDB drop root table %#x fc %#x\n", 2484 esw->fdb_table.offloads.drop_root->id, 2485 esw->fdb_table.offloads.drop_root_fc->id); 2486 mlx5_del_flow_rules(esw->fdb_table.offloads.drop_root_rule); 2487 /* Don't free flow counter here, can be reused on a later activation */ 2488 mlx5_destroy_flow_table(esw->fdb_table.offloads.drop_root); 2489 esw->fdb_table.offloads.drop_root_rule = NULL; 2490 esw->fdb_table.offloads.drop_root = NULL; 2491 } 2492 2493 static int mlx5_esw_fdb_drop_create(struct mlx5_eswitch *esw) 2494 { 2495 struct mlx5_flow_destination drop_fc_dst = {}; 2496 struct mlx5_flow_table_attr ft_attr = {}; 2497 struct mlx5_flow_destination *dst = NULL; 2498 struct mlx5_core_dev *dev = esw->dev; 2499 struct mlx5_flow_namespace *root_ns; 2500 struct mlx5_flow_act flow_act = {}; 2501 struct mlx5_flow_handle *flow_rule; 2502 struct mlx5_flow_table *table; 2503 int err = 0, dst_num = 0; 2504 2505 if (esw->fdb_table.offloads.drop_root) 2506 return 0; 2507 2508 root_ns = esw->fdb_table.offloads.ns; 2509 2510 ft_attr.prio = FDB_DROP_ROOT; 2511 ft_attr.max_fte = 1; 2512 ft_attr.autogroup.max_num_groups = 1; 2513 table = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr); 2514 if (IS_ERR(table)) { 2515 esw_warn(dev, "Failed to create fdb drop root table, err %pe\n", 2516 table); 2517 return PTR_ERR(table); 2518 } 2519 2520 /* Drop FC reusable, create once on first deactivation of FDB */ 2521 if (!esw->fdb_table.offloads.drop_root_fc) { 2522 struct mlx5_fc *counter = mlx5_fc_create(dev, 0); 2523 2524 err = PTR_ERR_OR_ZERO(counter); 2525 if (err) 2526 esw_warn(esw->dev, "create fdb drop fc err %d\n", err); 2527 else 2528 esw->fdb_table.offloads.drop_root_fc = counter; 2529 } 2530 2531 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2532 2533 if (esw->fdb_table.offloads.drop_root_fc) { 2534 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 2535 drop_fc_dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2536 drop_fc_dst.counter = esw->fdb_table.offloads.drop_root_fc; 2537 dst = &drop_fc_dst; 2538 dst_num++; 2539 } 2540 2541 flow_rule = mlx5_add_flow_rules(table, NULL, &flow_act, dst, dst_num); 2542 err = PTR_ERR_OR_ZERO(flow_rule); 2543 if (err) { 2544 esw_warn(esw->dev, 2545 "fs offloads: Failed to add vport rx drop rule err %d\n", 2546 err); 2547 goto err_flow_rule; 2548 } 2549 2550 esw->fdb_table.offloads.drop_root = table; 2551 esw->fdb_table.offloads.drop_root_rule = flow_rule; 2552 esw_debug(esw->dev, "Created FDB drop root table %#x fc %#x\n", 2553 table->id, dst ? dst->counter->id : 0); 2554 return 0; 2555 2556 err_flow_rule: 2557 /* no need to free drop fc, esw_offloads_steering_cleanup will do it */ 2558 mlx5_destroy_flow_table(table); 2559 return err; 2560 } 2561 2562 static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw) 2563 { 2564 struct mlx5_vport *vport; 2565 unsigned long i; 2566 2567 mlx5_esw_fdb_drop_destroy(esw); 2568 mlx5_mpfs_enable(esw->dev); 2569 2570 mutex_lock(&esw->state_lock); 2571 mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { 2572 if (!vport->adjacent) 2573 continue; 2574 esw_debug(esw->dev, "Connecting vport %d to eswitch\n", 2575 vport->vport); 2576 mlx5_esw_adj_vport_modify(esw->dev, vport->vport, true); 2577 } 2578 mutex_unlock(&esw->state_lock); 2579 2580 esw->offloads_inactive = false; 2581 esw_warn(esw->dev, "MPFS/FDB active\n"); 2582 } 2583 2584 static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw) 2585 { 2586 struct mlx5_vport *vport; 2587 unsigned long i; 2588 2589 mlx5_mpfs_disable(esw->dev); 2590 mlx5_esw_fdb_drop_create(esw); 2591 2592 mutex_lock(&esw->state_lock); 2593 mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { 2594 if (!vport->adjacent) 2595 continue; 2596 esw_debug(esw->dev, "Disconnecting vport %u from eswitch\n", 2597 vport->vport); 2598 2599 mlx5_esw_adj_vport_modify(esw->dev, vport->vport, false); 2600 } 2601 mutex_unlock(&esw->state_lock); 2602 2603 esw->offloads_inactive = true; 2604 esw_warn(esw->dev, "MPFS/FDB inactive\n"); 2605 } 2606 2607 static int esw_offloads_start(struct mlx5_eswitch *esw, 2608 struct netlink_ext_ack *extack) 2609 { 2610 int err; 2611 2612 esw_mode_change(esw, MLX5_ESWITCH_OFFLOADS); 2613 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2614 if (err) { 2615 NL_SET_ERR_MSG_MOD(extack, 2616 "Failed setting eswitch to offloads"); 2617 esw_mode_change(esw, MLX5_ESWITCH_LEGACY); 2618 return err; 2619 } 2620 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2621 if (mlx5_eswitch_inline_mode_get(esw, 2622 &esw->offloads.inline_mode)) { 2623 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2624 NL_SET_ERR_MSG_MOD(extack, 2625 "Inline mode is different between vports"); 2626 } 2627 } 2628 return 0; 2629 } 2630 2631 void mlx5_esw_offloads_rep_remove(struct mlx5_eswitch *esw, 2632 const struct mlx5_vport *vport) 2633 { 2634 struct mlx5_eswitch_rep *rep = xa_load(&esw->offloads.vport_reps, 2635 vport->vport); 2636 2637 if (!rep) 2638 return; 2639 xa_erase(&esw->offloads.vport_reps, vport->vport); 2640 kfree(rep); 2641 } 2642 2643 int mlx5_esw_offloads_rep_add(struct mlx5_eswitch *esw, 2644 const struct mlx5_vport *vport) 2645 { 2646 struct mlx5_eswitch_rep *rep; 2647 int rep_type; 2648 int err; 2649 2650 rep = kzalloc_obj(*rep); 2651 if (!rep) 2652 return -ENOMEM; 2653 2654 rep->vport = vport->vport; 2655 rep->vport_index = vport->index; 2656 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2657 if (!esw->offloads.rep_ops[rep_type]) { 2658 atomic_set(&rep->rep_data[rep_type].state, 2659 REP_UNREGISTERED); 2660 continue; 2661 } 2662 /* Dynamic/delegated vports add their representors after 2663 * mlx5_eswitch_register_vport_reps, so mark them as registered 2664 * for them to be loaded later with the others. 2665 */ 2666 rep->esw = esw; 2667 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2668 } 2669 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2670 if (err) 2671 goto insert_err; 2672 2673 return 0; 2674 2675 insert_err: 2676 kfree(rep); 2677 return err; 2678 } 2679 2680 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2681 struct mlx5_eswitch_rep *rep) 2682 { 2683 xa_erase(&esw->offloads.vport_reps, rep->vport); 2684 kfree(rep); 2685 } 2686 2687 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2688 { 2689 struct mlx5_eswitch_rep *rep; 2690 unsigned long i; 2691 2692 mlx5_esw_for_each_rep(esw, i, rep) 2693 mlx5_esw_offloads_rep_cleanup(esw, rep); 2694 xa_destroy(&esw->offloads.vport_reps); 2695 mutex_destroy(&esw->offloads.reps_lock); 2696 } 2697 2698 static int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2699 { 2700 struct mlx5_vport *vport; 2701 unsigned long i; 2702 int err; 2703 2704 xa_init(&esw->offloads.vport_reps); 2705 mutex_init(&esw->offloads.reps_lock); 2706 2707 mlx5_esw_for_each_vport(esw, i, vport) { 2708 err = mlx5_esw_offloads_rep_add(esw, vport); 2709 if (err) 2710 goto err; 2711 } 2712 return 0; 2713 2714 err: 2715 esw_offloads_cleanup_reps(esw); 2716 return err; 2717 } 2718 2719 static int esw_port_metadata_set(struct devlink *devlink, u32 id, 2720 struct devlink_param_gset_ctx *ctx, 2721 struct netlink_ext_ack *extack) 2722 { 2723 struct mlx5_core_dev *dev = devlink_priv(devlink); 2724 struct mlx5_eswitch *esw = dev->priv.eswitch; 2725 int err = 0; 2726 2727 down_write(&esw->mode_lock); 2728 if (mlx5_esw_is_fdb_created(esw)) { 2729 err = -EBUSY; 2730 goto done; 2731 } 2732 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 2733 err = -EOPNOTSUPP; 2734 goto done; 2735 } 2736 if (ctx->val.vbool) 2737 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2738 else 2739 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2740 done: 2741 up_write(&esw->mode_lock); 2742 return err; 2743 } 2744 2745 static int esw_port_metadata_get(struct devlink *devlink, u32 id, 2746 struct devlink_param_gset_ctx *ctx, 2747 struct netlink_ext_ack *extack) 2748 { 2749 struct mlx5_core_dev *dev = devlink_priv(devlink); 2750 2751 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch); 2752 return 0; 2753 } 2754 2755 static int esw_port_metadata_validate(struct devlink *devlink, u32 id, 2756 union devlink_param_value *val, 2757 struct netlink_ext_ack *extack) 2758 { 2759 struct mlx5_core_dev *dev = devlink_priv(devlink); 2760 u8 esw_mode; 2761 2762 esw_mode = mlx5_eswitch_mode(dev); 2763 if (esw_mode == MLX5_ESWITCH_OFFLOADS) { 2764 NL_SET_ERR_MSG_MOD(extack, 2765 "E-Switch must either disabled or non switchdev mode"); 2766 return -EBUSY; 2767 } 2768 return 0; 2769 } 2770 2771 static const struct devlink_param esw_devlink_params[] = { 2772 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA, 2773 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL, 2774 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 2775 esw_port_metadata_get, 2776 esw_port_metadata_set, 2777 esw_port_metadata_validate), 2778 }; 2779 2780 int esw_offloads_init(struct mlx5_eswitch *esw) 2781 { 2782 int err; 2783 2784 err = esw_offloads_init_reps(esw); 2785 if (err) 2786 return err; 2787 2788 if (MLX5_ESWITCH_MANAGER(esw->dev) && 2789 mlx5_esw_vport_match_metadata_supported(esw)) 2790 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2791 2792 err = devl_params_register(priv_to_devlink(esw->dev), 2793 esw_devlink_params, 2794 ARRAY_SIZE(esw_devlink_params)); 2795 if (err) 2796 goto err_params; 2797 2798 return 0; 2799 2800 err_params: 2801 esw_offloads_cleanup_reps(esw); 2802 return err; 2803 } 2804 2805 void esw_offloads_cleanup(struct mlx5_eswitch *esw) 2806 { 2807 devl_params_unregister(priv_to_devlink(esw->dev), 2808 esw_devlink_params, 2809 ARRAY_SIZE(esw_devlink_params)); 2810 esw_offloads_cleanup_reps(esw); 2811 } 2812 2813 static int __esw_offloads_load_rep(struct mlx5_eswitch *esw, 2814 struct mlx5_eswitch_rep *rep, 2815 u8 rep_type, bool *newly_loaded) 2816 { 2817 int err; 2818 2819 mlx5_esw_assert_reps_locked(esw); 2820 2821 if (newly_loaded) 2822 *newly_loaded = false; 2823 2824 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2825 REP_REGISTERED, REP_LOADED) != REP_REGISTERED) 2826 return 0; 2827 2828 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2829 if (err) { 2830 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2831 return err; 2832 } 2833 2834 if (newly_loaded) 2835 *newly_loaded = true; 2836 2837 return 0; 2838 } 2839 2840 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2841 struct mlx5_eswitch_rep *rep, u8 rep_type) 2842 { 2843 mlx5_esw_assert_reps_locked(esw); 2844 2845 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2846 REP_LOADED, REP_REGISTERED) == REP_LOADED) { 2847 if (rep_type == REP_ETH) 2848 __esw_offloads_unload_rep(esw, rep, REP_IB); 2849 esw->offloads.rep_ops[rep_type]->unload(rep); 2850 } 2851 } 2852 2853 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2854 { 2855 struct mlx5_eswitch_rep *rep; 2856 unsigned long i; 2857 2858 mlx5_esw_for_each_rep(esw, i, rep) 2859 __esw_offloads_unload_rep(esw, rep, rep_type); 2860 } 2861 2862 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2863 { 2864 struct mlx5_eswitch_rep *rep; 2865 unsigned long loaded = 0; 2866 bool newly_loaded; 2867 int rep_type; 2868 int err; 2869 2870 if (vport_num != MLX5_VPORT_UPLINK && 2871 mlx5_get_sd(esw->dev) && !mlx5_lag_is_active(esw->dev)) 2872 return 0; 2873 2874 rep = mlx5_eswitch_get_rep(esw, vport_num); 2875 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2876 err = __esw_offloads_load_rep(esw, rep, rep_type, 2877 &newly_loaded); 2878 if (err) 2879 goto err_reps; 2880 if (newly_loaded) 2881 loaded |= BIT(rep_type); 2882 } 2883 2884 return 0; 2885 2886 err_reps: 2887 while (--rep_type >= 0) 2888 if (test_bit(rep_type, &loaded)) 2889 __esw_offloads_unload_rep(esw, rep, rep_type); 2890 return err; 2891 } 2892 2893 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2894 { 2895 struct mlx5_eswitch_rep *rep; 2896 int rep_type; 2897 2898 rep = mlx5_eswitch_get_rep(esw, vport_num); 2899 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2900 __esw_offloads_unload_rep(esw, rep, rep_type); 2901 } 2902 2903 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2904 { 2905 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2906 return 0; 2907 2908 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport); 2909 } 2910 2911 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2912 { 2913 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2914 return; 2915 2916 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport); 2917 } 2918 2919 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport, 2920 struct mlx5_devlink_port *dl_port, 2921 u32 controller, u32 sfnum) 2922 { 2923 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum); 2924 } 2925 2926 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2927 { 2928 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport); 2929 } 2930 2931 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2932 { 2933 int err; 2934 2935 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2936 return 0; 2937 2938 err = mlx5_esw_offloads_devlink_port_register(esw, vport); 2939 if (err) 2940 return err; 2941 2942 err = mlx5_esw_offloads_rep_load(esw, vport->vport); 2943 if (err) 2944 goto load_err; 2945 return err; 2946 2947 load_err: 2948 mlx5_esw_offloads_devlink_port_unregister(vport); 2949 return err; 2950 } 2951 2952 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2953 { 2954 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2955 return; 2956 2957 mlx5_esw_offloads_rep_unload(esw, vport->vport); 2958 2959 mlx5_esw_offloads_devlink_port_unregister(vport); 2960 } 2961 2962 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2963 struct mlx5_core_dev *slave) 2964 { 2965 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2966 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2967 struct mlx5_flow_root_namespace *root; 2968 struct mlx5_flow_namespace *ns; 2969 int err; 2970 2971 MLX5_SET(set_flow_table_root_in, in, opcode, 2972 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2973 MLX5_SET(set_flow_table_root_in, in, table_type, 2974 FS_FT_FDB); 2975 2976 if (master) { 2977 ns = mlx5_get_flow_namespace(master, 2978 MLX5_FLOW_NAMESPACE_FDB); 2979 root = find_root(&ns->node); 2980 mutex_lock(&root->chain_lock); 2981 MLX5_SET(set_flow_table_root_in, in, 2982 table_eswitch_owner_vhca_id_valid, 1); 2983 MLX5_SET(set_flow_table_root_in, in, 2984 table_eswitch_owner_vhca_id, 2985 MLX5_CAP_GEN(master, vhca_id)); 2986 MLX5_SET(set_flow_table_root_in, in, table_id, 2987 root->root_ft->id); 2988 } else { 2989 ns = mlx5_get_flow_namespace(slave, 2990 MLX5_FLOW_NAMESPACE_FDB); 2991 root = find_root(&ns->node); 2992 mutex_lock(&root->chain_lock); 2993 MLX5_SET(set_flow_table_root_in, in, table_id, 2994 root->root_ft->id); 2995 } 2996 2997 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2998 mutex_unlock(&root->chain_lock); 2999 3000 return err; 3001 } 3002 3003 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 3004 struct mlx5_core_dev *slave, 3005 struct mlx5_vport *vport, 3006 struct mlx5_flow_table *acl) 3007 { 3008 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 3009 struct mlx5_flow_handle *flow_rule = NULL; 3010 struct mlx5_flow_destination dest = {}; 3011 struct mlx5_flow_act flow_act = {}; 3012 struct mlx5_flow_spec *spec; 3013 int err = 0; 3014 void *misc; 3015 3016 spec = kvzalloc_obj(*spec); 3017 if (!spec) 3018 return -ENOMEM; 3019 3020 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 3021 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 3022 misc_parameters); 3023 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 3024 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index); 3025 3026 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 3027 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 3028 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 3029 source_eswitch_owner_vhca_id); 3030 3031 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 3032 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 3033 dest.vport.num = slave->priv.eswitch->manager_vport; 3034 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 3035 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 3036 3037 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 3038 &dest, 1); 3039 if (IS_ERR(flow_rule)) { 3040 err = PTR_ERR(flow_rule); 3041 } else { 3042 err = xa_insert(&vport->egress.offloads.bounce_rules, 3043 slave_index, flow_rule, GFP_KERNEL); 3044 if (err) 3045 mlx5_del_flow_rules(flow_rule); 3046 } 3047 3048 kvfree(spec); 3049 return err; 3050 } 3051 3052 static int esw_slave_egress_create_resources(struct mlx5_eswitch *esw, 3053 struct mlx5_vport *vport) 3054 { 3055 struct mlx5_flow_table_attr ft_attr = { 3056 .max_fte = 1, .prio = 0, .level = 0, 3057 }; 3058 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 3059 struct mlx5_flow_namespace *ns; 3060 struct mlx5_flow_table *acl; 3061 struct mlx5_flow_group *g; 3062 u32 *flow_group_in; 3063 int err = 0; 3064 3065 if (vport->egress.acl) 3066 return 0; 3067 3068 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC); 3069 ns = mlx5_get_flow_vport_namespace(esw->dev, 3070 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 3071 vport->index); 3072 if (!ns) 3073 return -EINVAL; 3074 3075 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 3076 if (!flow_group_in) 3077 return -ENOMEM; 3078 3079 if (vport->vport || mlx5_core_is_ecpf(esw->dev)) 3080 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; 3081 3082 acl = mlx5_create_vport_flow_table(ns, &ft_attr, vport->vport); 3083 if (IS_ERR(acl)) { 3084 err = PTR_ERR(acl); 3085 goto out; 3086 } 3087 3088 g = mlx5_create_flow_group(acl, flow_group_in); 3089 if (IS_ERR(g)) { 3090 err = PTR_ERR(g); 3091 goto err_table; 3092 } 3093 3094 vport->egress.acl = acl; 3095 vport->egress.offloads.bounce_grp = g; 3096 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB; 3097 err = 0; 3098 3099 err_table: 3100 if (err && !IS_ERR_OR_NULL(acl)) { 3101 mlx5_destroy_flow_table(acl); 3102 vport->egress.acl = NULL; 3103 } 3104 out: 3105 kvfree(flow_group_in); 3106 return err; 3107 } 3108 3109 static void esw_slave_egress_destroy_resources(struct mlx5_vport *vport) 3110 { 3111 if (!IS_ERR_OR_NULL(vport->egress.offloads.bounce_grp)) { 3112 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp); 3113 vport->egress.offloads.bounce_grp = NULL; 3114 } 3115 if (!IS_ERR_OR_NULL(vport->egress.acl)) { 3116 esw_acl_egress_ofld_cleanup(vport); 3117 xa_destroy(&vport->egress.offloads.bounce_rules); 3118 } 3119 } 3120 3121 static int esw_set_slave_egress_rule(struct mlx5_core_dev *master, 3122 struct mlx5_core_dev *slave) 3123 { 3124 struct mlx5_eswitch *slave_esw = slave->priv.eswitch; 3125 u16 master_vhca = MLX5_CAP_GEN(master, vhca_id); 3126 struct mlx5_flow_destination dest = {}; 3127 struct mlx5_flow_handle *bounce_rule; 3128 struct mlx5_flow_act flow_act = {}; 3129 struct mlx5_vport *slave_vport; 3130 int err; 3131 3132 slave_vport = mlx5_eswitch_get_vport(slave_esw, 3133 slave_esw->manager_vport); 3134 if (IS_ERR(slave_vport)) 3135 return PTR_ERR(slave_vport); 3136 3137 err = esw_slave_egress_create_resources(slave_esw, slave_vport); 3138 if (err) 3139 return err; 3140 3141 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 3142 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 3143 dest.vport.num = master->priv.eswitch->manager_vport; 3144 dest.vport.vhca_id = master_vhca; 3145 dest.vport.flags = MLX5_FLOW_DEST_VPORT_VHCA_ID; 3146 3147 bounce_rule = mlx5_add_flow_rules(slave_vport->egress.acl, NULL, 3148 &flow_act, &dest, 1); 3149 if (IS_ERR(bounce_rule)) { 3150 err = PTR_ERR(bounce_rule); 3151 goto err_rule; 3152 } 3153 err = xa_insert(&slave_vport->egress.offloads.bounce_rules, 3154 master_vhca, bounce_rule, GFP_KERNEL); 3155 if (err) 3156 goto err_insert; 3157 3158 return 0; 3159 err_insert: 3160 mlx5_del_flow_rules(bounce_rule); 3161 err_rule: 3162 esw_slave_egress_destroy_resources(slave_vport); 3163 return err; 3164 } 3165 3166 static void esw_unset_slave_egress_rule(struct mlx5_core_dev *master, 3167 struct mlx5_core_dev *slave) 3168 { 3169 struct mlx5_eswitch *slave_esw = slave->priv.eswitch; 3170 u16 master_vhca = MLX5_CAP_GEN(master, vhca_id); 3171 struct mlx5_vport *slave_vport; 3172 3173 slave_vport = mlx5_eswitch_get_vport(slave_esw, 3174 slave_esw->manager_vport); 3175 if (IS_ERR(slave_vport)) 3176 return; 3177 3178 esw_acl_egress_ofld_bounce_rule_destroy(slave_vport, master_vhca); 3179 esw_slave_egress_destroy_resources(slave_vport); 3180 } 3181 3182 static int esw_master_egress_create_resources(struct mlx5_eswitch *esw, 3183 struct mlx5_flow_namespace *egress_ns, 3184 struct mlx5_vport *vport, size_t count) 3185 { 3186 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 3187 struct mlx5_flow_table_attr ft_attr = { 3188 .max_fte = count, .prio = 0, .level = 0, 3189 }; 3190 struct mlx5_flow_table *acl; 3191 struct mlx5_flow_group *g; 3192 void *match_criteria; 3193 u32 *flow_group_in; 3194 int err; 3195 3196 if (vport->egress.acl) 3197 return 0; 3198 3199 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 3200 if (!flow_group_in) 3201 return -ENOMEM; 3202 3203 if (vport->vport || mlx5_core_is_ecpf(esw->dev)) 3204 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; 3205 3206 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 3207 if (IS_ERR(acl)) { 3208 err = PTR_ERR(acl); 3209 goto out; 3210 } 3211 3212 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 3213 match_criteria); 3214 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 3215 misc_parameters.source_port); 3216 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 3217 misc_parameters.source_eswitch_owner_vhca_id); 3218 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 3219 MLX5_MATCH_MISC_PARAMETERS); 3220 3221 MLX5_SET(create_flow_group_in, flow_group_in, 3222 source_eswitch_owner_vhca_id_valid, 1); 3223 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 3224 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count); 3225 3226 g = mlx5_create_flow_group(acl, flow_group_in); 3227 if (IS_ERR(g)) { 3228 err = PTR_ERR(g); 3229 goto err_group; 3230 } 3231 3232 vport->egress.acl = acl; 3233 vport->egress.offloads.bounce_grp = g; 3234 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB; 3235 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC); 3236 3237 kvfree(flow_group_in); 3238 3239 return 0; 3240 3241 err_group: 3242 mlx5_destroy_flow_table(acl); 3243 out: 3244 kvfree(flow_group_in); 3245 return err; 3246 } 3247 3248 static void esw_master_egress_destroy_resources(struct mlx5_vport *vport) 3249 { 3250 if (!xa_empty(&vport->egress.offloads.bounce_rules)) 3251 return; 3252 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp); 3253 vport->egress.offloads.bounce_grp = NULL; 3254 mlx5_destroy_flow_table(vport->egress.acl); 3255 vport->egress.acl = NULL; 3256 } 3257 3258 static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 3259 struct mlx5_core_dev *slave, size_t count) 3260 { 3261 struct mlx5_eswitch *esw = master->priv.eswitch; 3262 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 3263 struct mlx5_flow_namespace *egress_ns; 3264 struct mlx5_vport *vport; 3265 int err; 3266 3267 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3268 if (IS_ERR(vport)) 3269 return PTR_ERR(vport); 3270 3271 egress_ns = mlx5_get_flow_vport_namespace(master, 3272 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 3273 vport->index); 3274 if (!egress_ns) 3275 return -EINVAL; 3276 3277 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB) 3278 return 0; 3279 3280 err = esw_master_egress_create_resources(esw, egress_ns, vport, count); 3281 if (err) 3282 return err; 3283 3284 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index)) 3285 return -EINVAL; 3286 3287 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl); 3288 if (err) 3289 goto err_rule; 3290 3291 return 0; 3292 3293 err_rule: 3294 esw_master_egress_destroy_resources(vport); 3295 return err; 3296 } 3297 3298 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev, 3299 struct mlx5_core_dev *slave_dev) 3300 { 3301 struct mlx5_vport *vport; 3302 3303 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 3304 dev->priv.eswitch->manager_vport); 3305 3306 if (!vport->egress.acl) 3307 return; 3308 3309 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id)); 3310 3311 if (xa_empty(&vport->egress.offloads.bounce_rules)) { 3312 esw_acl_egress_ofld_cleanup(vport); 3313 xa_destroy(&vport->egress.offloads.bounce_rules); 3314 } 3315 } 3316 3317 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw, 3318 struct mlx5_eswitch *slave_esw, int max_slaves) 3319 { 3320 int err; 3321 3322 err = esw_set_slave_root_fdb(master_esw->dev, 3323 slave_esw->dev); 3324 if (err) 3325 return err; 3326 3327 if (!mlx5_sd_is_primary(slave_esw->dev)) 3328 return 0; 3329 3330 err = esw_set_master_egress_rule(master_esw->dev, 3331 slave_esw->dev, max_slaves); 3332 if (err) 3333 goto err_acl; 3334 3335 return err; 3336 3337 err_acl: 3338 esw_set_slave_root_fdb(NULL, slave_esw->dev); 3339 return err; 3340 } 3341 3342 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw, 3343 struct mlx5_eswitch *slave_esw) 3344 { 3345 esw_set_slave_root_fdb(NULL, slave_esw->dev); 3346 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev); 3347 } 3348 3349 int mlx5_eswitch_offloads_vport_lag_add_one(struct mlx5_eswitch *master_esw, 3350 struct mlx5_eswitch *slave_esw) 3351 { 3352 return esw_set_slave_egress_rule(master_esw->dev, slave_esw->dev); 3353 } 3354 3355 void mlx5_eswitch_offloads_vport_lag_del_one(struct mlx5_eswitch *master_esw, 3356 struct mlx5_eswitch *slave_esw) 3357 { 3358 esw_unset_slave_egress_rule(master_esw->dev, slave_esw->dev); 3359 } 3360 3361 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 3362 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 3363 3364 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw, 3365 struct mlx5_eswitch *peer_esw) 3366 { 3367 const struct mlx5_eswitch_rep_ops *ops; 3368 struct mlx5_eswitch_rep *rep; 3369 unsigned long i; 3370 u8 rep_type; 3371 3372 mlx5_esw_for_each_rep(esw, i, rep) { 3373 rep_type = NUM_REP_TYPES; 3374 while (rep_type--) { 3375 ops = esw->offloads.rep_ops[rep_type]; 3376 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3377 ops->event) 3378 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw); 3379 } 3380 } 3381 } 3382 3383 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw, 3384 struct mlx5_eswitch *peer_esw) 3385 { 3386 struct list_head *dup_peer_flows; 3387 3388 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 3389 mlx5e_tc_clean_fdb_peer_flows(esw); 3390 #endif 3391 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw); 3392 dup_peer_flows = xa_erase(&esw->offloads.peer_flows, 3393 MLX5_CAP_GEN(peer_esw->dev, vhca_id)); 3394 kfree(dup_peer_flows); 3395 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev); 3396 } 3397 3398 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 3399 struct mlx5_eswitch *peer_esw) 3400 { 3401 const struct mlx5_eswitch_rep_ops *ops; 3402 struct list_head *dup_peer_flows; 3403 struct mlx5_eswitch_rep *rep; 3404 unsigned long i; 3405 u8 rep_type; 3406 int err; 3407 3408 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 3409 if (err) 3410 return err; 3411 3412 dup_peer_flows = kzalloc_obj(*dup_peer_flows); 3413 if (!dup_peer_flows) { 3414 err = -ENOMEM; 3415 goto err_out; 3416 } 3417 INIT_LIST_HEAD(dup_peer_flows); 3418 err = xa_err(xa_store(&esw->offloads.peer_flows, 3419 MLX5_CAP_GEN(peer_esw->dev, vhca_id), 3420 dup_peer_flows, GFP_KERNEL)); 3421 if (err) { 3422 kfree(dup_peer_flows); 3423 goto err_out; 3424 } 3425 3426 mlx5_esw_for_each_rep(esw, i, rep) { 3427 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 3428 ops = esw->offloads.rep_ops[rep_type]; 3429 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3430 ops->event) { 3431 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 3432 if (err) 3433 goto err_out; 3434 } 3435 } 3436 } 3437 3438 return 0; 3439 3440 err_out: 3441 mlx5_esw_offloads_unpair(esw, peer_esw); 3442 return err; 3443 } 3444 3445 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 3446 struct mlx5_eswitch *peer_esw, 3447 bool pair) 3448 { 3449 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 3450 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 3451 struct mlx5_flow_root_namespace *peer_ns; 3452 struct mlx5_flow_root_namespace *ns; 3453 int err; 3454 3455 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 3456 ns = esw->dev->priv.steering->fdb_root_ns; 3457 3458 if (pair) { 3459 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id); 3460 if (err) 3461 return err; 3462 3463 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id); 3464 if (err) { 3465 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 3466 return err; 3467 } 3468 } else { 3469 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 3470 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id); 3471 } 3472 3473 return 0; 3474 } 3475 3476 bool mlx5_eswitch_is_peer(struct mlx5_eswitch *esw, 3477 struct mlx5_eswitch *peer_esw) 3478 { 3479 u16 peer_esw_i; 3480 3481 if (!mlx5_esw_allowed(esw) || !mlx5_esw_allowed(peer_esw)) 3482 return false; 3483 3484 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 3485 return !!xa_load(&esw->paired, peer_esw_i); 3486 } 3487 3488 static int mlx5_esw_offloads_devcom_event(int event, 3489 void *my_data, 3490 void *event_data) 3491 { 3492 struct mlx5_eswitch *esw = my_data; 3493 struct mlx5_eswitch *peer_esw = event_data; 3494 u16 esw_i, peer_esw_i; 3495 bool esw_paired; 3496 int err; 3497 3498 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 3499 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id); 3500 esw_paired = !!xa_load(&esw->paired, peer_esw_i); 3501 3502 switch (event) { 3503 case ESW_OFFLOADS_DEVCOM_PAIR: 3504 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 3505 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 3506 break; 3507 3508 if (esw_paired) 3509 break; 3510 3511 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 3512 if (err) 3513 goto err_out; 3514 3515 err = mlx5_esw_offloads_pair(esw, peer_esw); 3516 if (err) 3517 goto err_peer; 3518 3519 err = mlx5_esw_offloads_pair(peer_esw, esw); 3520 if (err) 3521 goto err_pair; 3522 3523 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL); 3524 if (err) 3525 goto err_xa; 3526 3527 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL); 3528 if (err) 3529 goto err_peer_xa; 3530 3531 esw->num_peers++; 3532 peer_esw->num_peers++; 3533 mlx5_devcom_comp_set_ready(esw->devcom, true); 3534 break; 3535 3536 case ESW_OFFLOADS_DEVCOM_UNPAIR: 3537 if (!esw_paired) 3538 break; 3539 3540 peer_esw->num_peers--; 3541 esw->num_peers--; 3542 if (!esw->num_peers && !peer_esw->num_peers) 3543 mlx5_devcom_comp_set_ready(esw->devcom, false); 3544 xa_erase(&peer_esw->paired, esw_i); 3545 xa_erase(&esw->paired, peer_esw_i); 3546 mlx5_esw_offloads_unpair(peer_esw, esw); 3547 mlx5_esw_offloads_unpair(esw, peer_esw); 3548 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3549 break; 3550 } 3551 3552 return 0; 3553 3554 err_peer_xa: 3555 xa_erase(&esw->paired, peer_esw_i); 3556 err_xa: 3557 mlx5_esw_offloads_unpair(peer_esw, esw); 3558 err_pair: 3559 mlx5_esw_offloads_unpair(esw, peer_esw); 3560 err_peer: 3561 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3562 err_out: 3563 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 3564 event, err); 3565 return err; 3566 } 3567 3568 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, 3569 const struct mlx5_devcom_match_attr *attr) 3570 { 3571 int err; 3572 3573 xa_init(&esw->offloads.peer_flows); 3574 mutex_init(&esw->offloads.peer_mutex); 3575 3576 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 3577 return; 3578 3579 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) && 3580 (!mlx5_lag_is_supported(esw->dev) && !mlx5_get_sd(esw->dev))) 3581 return; 3582 3583 xa_init(&esw->paired); 3584 xa_init(&esw->fdb_table.offloads.peer_miss_rules); 3585 esw->num_peers = 0; 3586 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, 3587 MLX5_DEVCOM_ESW_OFFLOADS, 3588 attr, 3589 mlx5_esw_offloads_devcom_event, 3590 esw); 3591 if (!esw->devcom) 3592 return; 3593 3594 err = mlx5_devcom_send_event(esw->devcom, 3595 ESW_OFFLOADS_DEVCOM_PAIR, 3596 ESW_OFFLOADS_DEVCOM_UNPAIR, 3597 esw); 3598 if (err) 3599 mlx5_esw_offloads_devcom_cleanup(esw); 3600 } 3601 3602 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 3603 { 3604 if (!esw->devcom) 3605 return; 3606 3607 mlx5_devcom_send_event(esw->devcom, 3608 ESW_OFFLOADS_DEVCOM_UNPAIR, 3609 ESW_OFFLOADS_DEVCOM_UNPAIR, 3610 esw); 3611 3612 mlx5_devcom_unregister_component(esw->devcom); 3613 xa_destroy(&esw->paired); 3614 xa_destroy(&esw->fdb_table.offloads.peer_miss_rules); 3615 esw->devcom = NULL; 3616 WARN_ON_ONCE(!xa_empty(&esw->offloads.peer_flows)); 3617 xa_destroy(&esw->offloads.peer_flows); 3618 } 3619 3620 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) 3621 { 3622 return mlx5_devcom_comp_is_ready(esw->devcom); 3623 } 3624 3625 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 3626 { 3627 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 3628 return false; 3629 3630 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 3631 MLX5_FDB_TO_VPORT_REG_C_0)) 3632 return false; 3633 3634 return true; 3635 } 3636 3637 #define MLX5_ESW_METADATA_RSVD_UPLINK 1 3638 3639 /* Share the same metadata for uplink's. This is fine because: 3640 * (a) In shared FDB mode (LAG) both uplink's are treated the 3641 * same and tagged with the same metadata. 3642 * (b) In non shared FDB mode, packets from physical port0 3643 * cannot hit eswitch of PF1 and vice versa. 3644 */ 3645 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 3646 { 3647 return MLX5_ESW_METADATA_RSVD_UPLINK; 3648 } 3649 3650 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 3651 { 3652 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 3653 /* Reserve 0xf for internal port offload */ 3654 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 3655 int pf_num; 3656 int id; 3657 3658 /* Only 4 bits of pf_num */ 3659 pf_num = mlx5_sd_pf_num_get(esw->dev); 3660 if (pf_num < 0 || pf_num > max_pf_num) 3661 return 0; 3662 3663 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 3664 /* Use only non-zero vport_id (2-4095) for all PF's */ 3665 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 3666 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 3667 vport_end_ida, GFP_KERNEL); 3668 if (id < 0) 3669 return 0; 3670 id = (pf_num << ESW_VPORT_BITS) | id; 3671 return id; 3672 } 3673 3674 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 3675 { 3676 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 3677 3678 /* Metadata contains only 12 bits of actual ida id */ 3679 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 3680 } 3681 3682 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 3683 struct mlx5_vport *vport) 3684 { 3685 if (vport->vport == MLX5_VPORT_UPLINK) 3686 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 3687 else 3688 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 3689 3690 vport->metadata = vport->default_metadata; 3691 return vport->metadata ? 0 : -ENOSPC; 3692 } 3693 3694 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 3695 struct mlx5_vport *vport) 3696 { 3697 if (!vport->default_metadata) 3698 return; 3699 3700 if (vport->vport == MLX5_VPORT_UPLINK) 3701 return; 3702 3703 WARN_ON(vport->metadata != vport->default_metadata); 3704 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 3705 vport->default_metadata = 0; 3706 } 3707 3708 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 3709 { 3710 struct mlx5_vport *vport; 3711 unsigned long i; 3712 3713 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3714 return; 3715 3716 mlx5_esw_for_each_vport(esw, i, vport) 3717 esw_offloads_vport_metadata_cleanup(esw, vport); 3718 } 3719 3720 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 3721 { 3722 struct mlx5_vport *vport; 3723 unsigned long i; 3724 int err; 3725 3726 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3727 return 0; 3728 3729 mlx5_esw_for_each_vport(esw, i, vport) { 3730 err = esw_offloads_vport_metadata_setup(esw, vport); 3731 if (err) 3732 goto metadata_err; 3733 } 3734 3735 return 0; 3736 3737 metadata_err: 3738 esw_offloads_metadata_uninit(esw); 3739 return err; 3740 } 3741 3742 /* Deferred metadata init for SD devices: allocate vport metadata and 3743 * refresh the ingress ACL for every vport whose ACL was created with 3744 * metadata=0 in esw_create_offloads_acl_tables() / esw_vport_setup(). 3745 * 3746 * No Rep is loaded at this point ==> no Rep net-dev exists, so no need 3747 * to take rtnl lock. 3748 * 3749 * Safe to call multiple times - subsequent calls are no-ops. 3750 */ 3751 int mlx5_esw_offloads_init_deferred_metadata(struct mlx5_eswitch *esw) 3752 { 3753 struct mlx5_vport *manager, *vport; 3754 unsigned long i; 3755 int err; 3756 3757 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3758 return 0; 3759 3760 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3761 if (IS_ERR(manager)) 3762 return PTR_ERR(manager); 3763 3764 /* Sanity check: skip if metadata was already initialized */ 3765 if (manager->default_metadata) 3766 return 0; 3767 3768 err = esw_offloads_metadata_init(esw); 3769 if (err) 3770 return err; 3771 3772 mutex_lock(&esw->state_lock); 3773 /* Manager vport doesn't have a rep/netdev loaded but its ingress ACL 3774 * was programmed with metadata=0 - refresh it explicitly. 3775 */ 3776 err = mlx5_esw_acl_ingress_vport_metadata_update(esw, 3777 esw->manager_vport, 3778 0); 3779 if (err) 3780 goto err_acl; 3781 3782 /* UPLINK is never marked enabled but its ACL is programmed in 3783 * esw_create_offloads_acl_tables(); refresh it explicitly. 3784 */ 3785 err = mlx5_esw_acl_ingress_vport_metadata_update(esw, MLX5_VPORT_UPLINK, 3786 0); 3787 if (err) 3788 goto err_acl; 3789 3790 mlx5_esw_for_each_vport(esw, i, vport) { 3791 if (!vport || !vport->enabled) 3792 continue; 3793 err = mlx5_esw_acl_ingress_vport_metadata_update(esw, 3794 vport->vport, 3795 0); 3796 if (err) 3797 goto err_acl; 3798 } 3799 3800 mutex_unlock(&esw->state_lock); 3801 return 0; 3802 3803 err_acl: 3804 esw_offloads_metadata_uninit(esw); 3805 mutex_unlock(&esw->state_lock); 3806 return err; 3807 } 3808 3809 int 3810 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 3811 struct mlx5_vport *vport) 3812 { 3813 int err; 3814 3815 err = esw_acl_ingress_ofld_setup(esw, vport); 3816 if (err) 3817 return err; 3818 3819 err = esw_acl_egress_ofld_setup(esw, vport); 3820 if (err) 3821 goto egress_err; 3822 3823 return 0; 3824 3825 egress_err: 3826 esw_acl_ingress_ofld_cleanup(esw, vport); 3827 return err; 3828 } 3829 3830 void 3831 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 3832 struct mlx5_vport *vport) 3833 { 3834 esw_acl_egress_ofld_cleanup(vport); 3835 esw_acl_ingress_ofld_cleanup(esw, vport); 3836 } 3837 3838 static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw) 3839 { 3840 struct mlx5_vport *uplink, *manager; 3841 int ret; 3842 3843 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3844 if (IS_ERR(uplink)) 3845 return PTR_ERR(uplink); 3846 3847 ret = esw_vport_create_offloads_acl_tables(esw, uplink); 3848 if (ret) 3849 return ret; 3850 3851 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3852 if (IS_ERR(manager)) { 3853 ret = PTR_ERR(manager); 3854 goto err_manager; 3855 } 3856 3857 ret = esw_vport_create_offloads_acl_tables(esw, manager); 3858 if (ret) 3859 goto err_manager; 3860 3861 return 0; 3862 3863 err_manager: 3864 esw_vport_destroy_offloads_acl_tables(esw, uplink); 3865 return ret; 3866 } 3867 3868 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw) 3869 { 3870 struct mlx5_vport *vport; 3871 3872 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3873 if (!IS_ERR(vport)) 3874 esw_vport_destroy_offloads_acl_tables(esw, vport); 3875 3876 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3877 if (!IS_ERR(vport)) 3878 esw_vport_destroy_offloads_acl_tables(esw, vport); 3879 } 3880 3881 void mlx5_eswitch_unload_reps(struct mlx5_eswitch *esw) 3882 { 3883 struct mlx5_eswitch_rep *rep; 3884 unsigned long i; 3885 3886 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3887 return; 3888 3889 mlx5_esw_for_each_rep(esw, i, rep) { 3890 if (rep->vport == MLX5_VPORT_UPLINK) 3891 continue; 3892 mlx5_esw_offloads_rep_unload(esw, rep->vport); 3893 } 3894 } 3895 3896 int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw) 3897 { 3898 struct mlx5_eswitch_rep *rep; 3899 unsigned long i; 3900 int ret; 3901 3902 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3903 return 0; 3904 3905 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3906 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3907 return 0; 3908 3909 /* SD secondary devices share the primary's uplink and do not 3910 * have their own uplink representor. Only load VF/SF vports. 3911 */ 3912 if (mlx5_sd_is_primary(esw->dev)) { 3913 ret = __esw_offloads_load_rep(esw, rep, REP_IB, NULL); 3914 if (ret) 3915 return ret; 3916 } 3917 3918 mlx5_esw_for_each_rep(esw, i, rep) { 3919 if (!mlx5_sd_is_primary(esw->dev) && 3920 rep->vport == MLX5_VPORT_UPLINK) 3921 continue; 3922 if (rep->vport != MLX5_VPORT_UPLINK && 3923 mlx5_get_sd(esw->dev) && !mlx5_lag_is_active(esw->dev)) 3924 continue; 3925 3926 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3927 __esw_offloads_load_rep(esw, rep, REP_IB, NULL); 3928 } 3929 3930 return 0; 3931 } 3932 3933 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3934 { 3935 struct mlx5_esw_indir_table *indir; 3936 int err; 3937 3938 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3939 mutex_init(&esw->fdb_table.offloads.vports.lock); 3940 hash_init(esw->fdb_table.offloads.vports.table); 3941 atomic64_set(&esw->user_count, 0); 3942 3943 indir = mlx5_esw_indir_table_init(); 3944 if (IS_ERR(indir)) { 3945 err = PTR_ERR(indir); 3946 goto create_indir_err; 3947 } 3948 esw->fdb_table.offloads.indir = indir; 3949 3950 err = esw_create_offloads_acl_tables(esw); 3951 if (err) 3952 goto create_acl_err; 3953 3954 err = esw_create_offloads_table(esw); 3955 if (err) 3956 goto create_offloads_err; 3957 3958 err = esw_create_restore_table(esw); 3959 if (err) 3960 goto create_restore_err; 3961 3962 err = esw_create_offloads_fdb_tables(esw); 3963 if (err) 3964 goto create_fdb_err; 3965 3966 err = esw_create_vport_rx_group(esw); 3967 if (err) 3968 goto create_fg_err; 3969 3970 err = esw_create_vport_rx_drop_group(esw); 3971 if (err) 3972 goto create_rx_drop_fg_err; 3973 3974 err = esw_create_vport_rx_drop_rule(esw); 3975 if (err) 3976 goto create_rx_drop_rule_err; 3977 3978 return 0; 3979 3980 create_rx_drop_rule_err: 3981 esw_destroy_vport_rx_drop_group(esw); 3982 create_rx_drop_fg_err: 3983 esw_destroy_vport_rx_group(esw); 3984 create_fg_err: 3985 esw_destroy_offloads_fdb_tables(esw); 3986 create_fdb_err: 3987 esw_destroy_restore_table(esw); 3988 create_restore_err: 3989 esw_destroy_offloads_table(esw); 3990 create_offloads_err: 3991 esw_destroy_offloads_acl_tables(esw); 3992 create_acl_err: 3993 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3994 create_indir_err: 3995 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3996 return err; 3997 } 3998 3999 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 4000 { 4001 mlx5_esw_fdb_drop_destroy(esw); 4002 if (esw->fdb_table.offloads.drop_root_fc) 4003 mlx5_fc_destroy(esw->dev, esw->fdb_table.offloads.drop_root_fc); 4004 esw->fdb_table.offloads.drop_root_fc = NULL; 4005 esw_destroy_vport_rx_drop_rule(esw); 4006 esw_destroy_vport_rx_drop_group(esw); 4007 esw_destroy_vport_rx_group(esw); 4008 esw_destroy_offloads_fdb_tables(esw); 4009 esw_destroy_restore_table(esw); 4010 esw_destroy_offloads_table(esw); 4011 esw_destroy_offloads_acl_tables(esw); 4012 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 4013 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 4014 } 4015 4016 static void esw_changed_event_handler(struct mlx5_eswitch *esw) 4017 { 4018 struct mlx5_esw_pf_info host_pf_info; 4019 u16 new_num_vfs; 4020 const u32 *out; 4021 4022 out = mlx5_esw_query_functions(esw->dev); 4023 if (IS_ERR(out)) 4024 return; 4025 4026 host_pf_info = mlx5_esw_get_host_pf_info(esw->dev, out); 4027 new_num_vfs = host_pf_info.num_of_vfs; 4028 4029 if (host_pf_info.pf_disabled) { 4030 mlx5_sf_table_esw_changed_event_handler(esw->dev); 4031 mlx5_sf_hw_table_esw_changed_event_handler(esw->dev); 4032 } 4033 4034 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_info.pf_disabled) 4035 goto free; 4036 4037 mlx5_esw_reps_block(esw); 4038 /* Number of VFs can only change from "0 to x" or "x to 0". */ 4039 if (esw->esw_funcs.num_vfs > 0) { 4040 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 4041 } else { 4042 int err; 4043 4044 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 4045 MLX5_VPORT_UC_ADDR_CHANGE); 4046 if (err) 4047 goto unblock; 4048 } 4049 esw->esw_funcs.num_vfs = new_num_vfs; 4050 unblock: 4051 mlx5_esw_reps_unblock(esw); 4052 free: 4053 kvfree(out); 4054 } 4055 4056 static void esw_wq_handler(struct work_struct *work) 4057 { 4058 struct mlx5_host_work *host_work; 4059 struct mlx5_eswitch *esw; 4060 struct devlink *devlink; 4061 int work_gen; 4062 4063 host_work = container_of(work, struct mlx5_host_work, work); 4064 esw = host_work->esw; 4065 work_gen = host_work->work_gen; 4066 devlink = priv_to_devlink(esw->dev); 4067 4068 /* Do not block on devlink lock until stale work is filtered out. 4069 * Teardown can invalidate the generation and then wait for this 4070 * workqueue while holding devlink lock. 4071 */ 4072 for (;;) { 4073 if (work_gen != atomic_read(&esw->generation)) 4074 goto free; 4075 4076 if (devl_trylock(devlink)) 4077 break; 4078 4079 wait_event_timeout(esw->work_queue_wait, 4080 work_gen != atomic_read(&esw->generation), 4081 msecs_to_jiffies(60)); 4082 } 4083 4084 /* Stale work from one or more mode changes ago. Bail out. */ 4085 if (work_gen != atomic_read(&esw->generation)) 4086 goto unlock; 4087 4088 host_work->func(esw); 4089 4090 unlock: 4091 devl_unlock(devlink); 4092 free: 4093 kfree(host_work); 4094 } 4095 4096 static int mlx5_esw_add_work(struct mlx5_eswitch *esw, 4097 void (*func)(struct mlx5_eswitch *esw), 4098 gfp_t gfp) 4099 { 4100 struct mlx5_host_work *host_work; 4101 4102 host_work = kzalloc_obj(*host_work, gfp); 4103 if (!host_work) 4104 return -ENOMEM; 4105 4106 host_work->esw = esw; 4107 host_work->work_gen = atomic_read(&esw->generation); 4108 4109 host_work->func = func; 4110 INIT_WORK(&host_work->work, esw_wq_handler); 4111 queue_work(esw->work_queue, &host_work->work); 4112 4113 return 0; 4114 } 4115 4116 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, 4117 unsigned long type, void *data) 4118 { 4119 struct mlx5_esw_functions *esw_funcs; 4120 struct mlx5_eswitch *esw; 4121 int ret; 4122 4123 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 4124 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 4125 4126 ret = mlx5_esw_add_work(esw, esw_changed_event_handler, GFP_ATOMIC); 4127 if (ret) 4128 return NOTIFY_DONE; 4129 4130 return NOTIFY_OK; 4131 } 4132 4133 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 4134 { 4135 const struct mlx5_esw_functions *esw_funcs; 4136 int i; 4137 4138 /* Local controller is always valid */ 4139 if (controller == 0) 4140 return true; 4141 4142 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 4143 return false; 4144 4145 /* External host number starts with zero in device */ 4146 if (controller == mlx5_esw_get_hpf_host_number(esw->dev) + 1) 4147 return true; 4148 4149 esw_funcs = &esw->esw_funcs; 4150 for (i = 0; i < esw_funcs->num_spfs; i++) { 4151 if (controller == esw_funcs->spfs[i].host_number + 1) 4152 return true; 4153 } 4154 return false; 4155 } 4156 4157 int esw_offloads_enable(struct mlx5_eswitch *esw) 4158 { 4159 u8 mapping_id[MLX5_SW_IMAGE_GUID_MAX_BYTES]; 4160 struct mlx5_devcom_match_attr attr = {}; 4161 struct mapping_ctx *reg_c0_obj_pool; 4162 struct mlx5_vport *vport; 4163 unsigned long i; 4164 u8 id_len; 4165 int err; 4166 4167 mutex_init(&esw->offloads.termtbl_mutex); 4168 mlx5_esw_adjacent_vhcas_setup(esw); 4169 4170 err = mlx5_rdma_enable_roce(esw->dev); 4171 if (err) 4172 goto err_roce; 4173 4174 /* SD devices defer metadata init until SD is ready and 4175 * mlx5_sd_pf_num_get() can return the correct pf_num. 4176 */ 4177 if (!mlx5_get_sd(esw->dev)) { 4178 err = esw_offloads_metadata_init(esw); 4179 if (err) 4180 goto err_metadata; 4181 } 4182 4183 err = esw_set_passing_vport_metadata(esw, true); 4184 if (err) 4185 goto err_vport_metadata; 4186 4187 mlx5_query_nic_sw_system_image_guid(esw->dev, mapping_id, &id_len); 4188 4189 reg_c0_obj_pool = mapping_create_for_id(mapping_id, id_len, 4190 MAPPING_TYPE_CHAIN, 4191 sizeof(struct mlx5_mapped_obj), 4192 ESW_REG_C0_USER_DATA_METADATA_MASK, 4193 true); 4194 4195 if (IS_ERR(reg_c0_obj_pool)) { 4196 err = PTR_ERR(reg_c0_obj_pool); 4197 goto err_pool; 4198 } 4199 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 4200 4201 err = esw_offloads_steering_init(esw); 4202 if (err) 4203 goto err_steering_init; 4204 4205 if (esw->offloads_inactive) 4206 mlx5_esw_fdb_inactive(esw); 4207 else 4208 mlx5_esw_fdb_active(esw); 4209 4210 /* Representor will control the vport link state */ 4211 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 4212 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 4213 if (mlx5_core_ec_sriov_enabled(esw->dev)) 4214 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) 4215 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 4216 4217 /* Uplink vport rep must load first. */ 4218 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 4219 if (err) 4220 goto err_uplink; 4221 4222 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 4223 if (err) 4224 goto err_vports; 4225 4226 memcpy(attr.key.buf, mapping_id, id_len); 4227 attr.flags = MLX5_DEVCOM_MATCH_FLAGS_NS; 4228 attr.net = mlx5_core_net(esw->dev); 4229 mlx5_esw_offloads_devcom_init(esw, &attr); 4230 return 0; 4231 4232 err_vports: 4233 /* rollback to legacy, indicates don't unregister the uplink netdev */ 4234 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; 4235 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 4236 err_uplink: 4237 esw_offloads_steering_cleanup(esw); 4238 err_steering_init: 4239 mapping_destroy(reg_c0_obj_pool); 4240 err_pool: 4241 esw_set_passing_vport_metadata(esw, false); 4242 err_vport_metadata: 4243 esw_offloads_metadata_uninit(esw); 4244 err_metadata: 4245 mlx5_rdma_disable_roce(esw->dev); 4246 err_roce: 4247 mlx5_esw_adjacent_vhcas_cleanup(esw); 4248 mutex_destroy(&esw->offloads.termtbl_mutex); 4249 return err; 4250 } 4251 4252 static int esw_offloads_stop(struct mlx5_eswitch *esw, 4253 struct netlink_ext_ack *extack) 4254 { 4255 int err; 4256 4257 esw_mode_change(esw, MLX5_ESWITCH_LEGACY); 4258 4259 /* If changing from switchdev to legacy mode without sriov enabled, 4260 * no need to create legacy fdb. 4261 */ 4262 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev)) 4263 return 0; 4264 4265 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 4266 if (err) 4267 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 4268 4269 return err; 4270 } 4271 4272 void esw_offloads_disable(struct mlx5_eswitch *esw) 4273 { 4274 mlx5_esw_offloads_devcom_cleanup(esw); 4275 mlx5_eswitch_disable_pf_vf_vports(esw); 4276 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 4277 esw_set_passing_vport_metadata(esw, false); 4278 esw_offloads_steering_cleanup(esw); 4279 mapping_destroy(esw->offloads.reg_c0_obj_pool); 4280 esw_offloads_metadata_uninit(esw); 4281 mlx5_rdma_disable_roce(esw->dev); 4282 mlx5_esw_adjacent_vhcas_cleanup(esw); 4283 /* must be done after vhcas cleanup to avoid adjacent vports connect */ 4284 if (esw->offloads_inactive) 4285 mlx5_esw_fdb_active(esw); /* legacy mode always active */ 4286 mutex_destroy(&esw->offloads.termtbl_mutex); 4287 } 4288 4289 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 4290 { 4291 switch (mode) { 4292 case DEVLINK_ESWITCH_MODE_LEGACY: 4293 *mlx5_mode = MLX5_ESWITCH_LEGACY; 4294 break; 4295 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 4296 case DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE: 4297 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 4298 break; 4299 default: 4300 return -EINVAL; 4301 } 4302 4303 return 0; 4304 } 4305 4306 static int esw_mode_to_devlink(struct mlx5_eswitch *esw, u16 *mode) 4307 { 4308 switch (esw->mode) { 4309 case MLX5_ESWITCH_LEGACY: 4310 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 4311 break; 4312 case MLX5_ESWITCH_OFFLOADS: 4313 if (esw->offloads_inactive) 4314 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE; 4315 else 4316 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 4317 break; 4318 default: 4319 return -EINVAL; 4320 } 4321 4322 return 0; 4323 } 4324 4325 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 4326 { 4327 switch (mode) { 4328 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 4329 *mlx5_mode = MLX5_INLINE_MODE_NONE; 4330 break; 4331 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 4332 *mlx5_mode = MLX5_INLINE_MODE_L2; 4333 break; 4334 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 4335 *mlx5_mode = MLX5_INLINE_MODE_IP; 4336 break; 4337 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 4338 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 4339 break; 4340 default: 4341 return -EINVAL; 4342 } 4343 4344 return 0; 4345 } 4346 4347 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 4348 { 4349 switch (mlx5_mode) { 4350 case MLX5_INLINE_MODE_NONE: 4351 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 4352 break; 4353 case MLX5_INLINE_MODE_L2: 4354 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 4355 break; 4356 case MLX5_INLINE_MODE_IP: 4357 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 4358 break; 4359 case MLX5_INLINE_MODE_TCP_UDP: 4360 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 4361 break; 4362 default: 4363 return -EINVAL; 4364 } 4365 4366 return 0; 4367 } 4368 4369 int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev) 4370 { 4371 struct mlx5_eswitch *esw = dev->priv.eswitch; 4372 int err; 4373 4374 if (!mlx5_esw_allowed(esw)) 4375 return 0; 4376 4377 /* Take TC into account */ 4378 err = mlx5_esw_try_lock(esw); 4379 if (err < 0) 4380 return err; 4381 4382 esw->offloads.num_block_mode++; 4383 mlx5_esw_unlock(esw); 4384 return 0; 4385 } 4386 4387 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev) 4388 { 4389 struct mlx5_eswitch *esw = dev->priv.eswitch; 4390 4391 if (!mlx5_esw_allowed(esw)) 4392 return; 4393 4394 down_write(&esw->mode_lock); 4395 esw->offloads.num_block_mode--; 4396 up_write(&esw->mode_lock); 4397 } 4398 4399 /* Returns false only when uplink netdev exists and its netns is different from 4400 * devlink's netns. True for all others so entering switchdev mode is allowed. 4401 */ 4402 static bool mlx5_devlink_netdev_netns_immutable_set(struct devlink *devlink, 4403 bool immutable) 4404 { 4405 struct mlx5_core_dev *mdev = devlink_priv(devlink); 4406 struct net_device *netdev; 4407 bool ret; 4408 4409 netdev = mlx5_uplink_netdev_get(mdev); 4410 if (!netdev) 4411 return true; 4412 4413 rtnl_lock(); 4414 netdev->netns_immutable = immutable; 4415 ret = net_eq(dev_net(netdev), devlink_net(devlink)); 4416 rtnl_unlock(); 4417 4418 mlx5_uplink_netdev_put(mdev, netdev); 4419 return ret; 4420 } 4421 4422 /* Returns true when only changing between active and inactive switchdev mode */ 4423 static bool mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch *esw, 4424 u16 devlink_mode) 4425 { 4426 /* current mode is not switchdev */ 4427 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 4428 return false; 4429 4430 /* new mode is not switchdev */ 4431 if (devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV && 4432 devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE) 4433 return false; 4434 4435 /* already inactive: no change in current state */ 4436 if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE && 4437 esw->offloads_inactive) 4438 return false; 4439 4440 /* already active: no change in current state */ 4441 if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && 4442 !esw->offloads_inactive) 4443 return false; 4444 4445 down_write(&esw->mode_lock); 4446 esw->offloads_inactive = !esw->offloads_inactive; 4447 esw->eswitch_operation_in_progress = true; 4448 up_write(&esw->mode_lock); 4449 4450 if (esw->offloads_inactive) 4451 mlx5_esw_fdb_inactive(esw); 4452 else 4453 mlx5_esw_fdb_active(esw); 4454 4455 down_write(&esw->mode_lock); 4456 esw->eswitch_operation_in_progress = false; 4457 up_write(&esw->mode_lock); 4458 return true; 4459 } 4460 4461 #define MLX5_ESW_HOLD_TIMEOUT_MS 7000 4462 #define MLX5_ESW_HOLD_RETRY_DELAY_MS 500 4463 4464 void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev) 4465 { 4466 unsigned long timeout; 4467 bool hold_esw = true; 4468 4469 /* Wait for any concurrent eswitch mode transition to complete. */ 4470 if (!mlx5_esw_hold(dev)) { 4471 timeout = jiffies + msecs_to_jiffies(MLX5_ESW_HOLD_TIMEOUT_MS); 4472 while (!mlx5_esw_hold(dev)) { 4473 if (!time_before(jiffies, timeout)) { 4474 hold_esw = false; 4475 break; 4476 } 4477 msleep(MLX5_ESW_HOLD_RETRY_DELAY_MS); 4478 } 4479 } 4480 if (hold_esw) { 4481 if (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS) 4482 mlx5_core_reps_aux_devs_remove(dev); 4483 mlx5_esw_release(dev); 4484 } 4485 } 4486 4487 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 4488 struct netlink_ext_ack *extack) 4489 { 4490 u16 cur_mlx5_mode, mlx5_mode = 0; 4491 struct mlx5_eswitch *esw; 4492 int err = 0; 4493 4494 esw = mlx5_devlink_eswitch_get(devlink); 4495 if (IS_ERR(esw)) 4496 return PTR_ERR(esw); 4497 4498 if (mlx5_fw_reset_in_progress(esw->dev)) { 4499 NL_SET_ERR_MSG_MOD(extack, "Can't change eswitch mode during firmware reset"); 4500 return -EBUSY; 4501 } 4502 4503 if (esw_mode_from_devlink(mode, &mlx5_mode)) 4504 return -EINVAL; 4505 4506 /* Avoid try_lock, active/inactive mode change is not restricted */ 4507 if (mlx5_devlink_switchdev_active_mode_change(esw, mode)) 4508 return 0; 4509 4510 mlx5_lag_disable_change(esw->dev); 4511 err = mlx5_esw_try_lock(esw); 4512 if (err < 0) { 4513 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 4514 goto enable_lag; 4515 } 4516 cur_mlx5_mode = err; 4517 err = 0; 4518 4519 if (cur_mlx5_mode == mlx5_mode) 4520 goto unlock; 4521 4522 if (esw->offloads.num_block_mode) { 4523 NL_SET_ERR_MSG_MOD(extack, 4524 "Can't change eswitch mode when IPsec SA and/or policies are configured"); 4525 err = -EOPNOTSUPP; 4526 goto unlock; 4527 } 4528 4529 /* Keep mode_lock and reps_lock unnested. The operation flag excludes 4530 * mode users while mode_lock is dropped before taking reps_lock. 4531 */ 4532 esw->eswitch_operation_in_progress = true; 4533 up_write(&esw->mode_lock); 4534 4535 mlx5_esw_reps_block(esw); 4536 4537 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && 4538 !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) { 4539 NL_SET_ERR_MSG_MOD(extack, 4540 "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's."); 4541 err = -EINVAL; 4542 goto skip; 4543 } 4544 4545 if (mlx5_mode == MLX5_ESWITCH_LEGACY) 4546 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; 4547 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) 4548 esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_SWITCH_LEGACY; 4549 mlx5_eswitch_disable_locked(esw); 4550 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) { 4551 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 4552 NL_SET_ERR_MSG_MOD(extack, 4553 "Can't change mode while devlink traps are active"); 4554 err = -EOPNOTSUPP; 4555 goto skip; 4556 } 4557 esw->offloads_inactive = 4558 (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE); 4559 err = esw_offloads_start(esw, extack); 4560 } else if (mlx5_mode == MLX5_ESWITCH_LEGACY) { 4561 err = esw_offloads_stop(esw, extack); 4562 } else { 4563 err = -EINVAL; 4564 } 4565 4566 skip: 4567 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && err) 4568 mlx5_devlink_netdev_netns_immutable_set(devlink, false); 4569 /* Reconfiguration is done; drop reps_lock before taking mode_lock again 4570 * to clear the operation flag. 4571 */ 4572 mlx5_esw_reps_unblock(esw); 4573 down_write(&esw->mode_lock); 4574 esw->eswitch_operation_in_progress = false; 4575 unlock: 4576 mlx5_esw_unlock(esw); 4577 enable_lag: 4578 mlx5_lag_enable_change(esw->dev); 4579 /* Shared FDB activation is creating LAG which is changing reps. */ 4580 if (!err) 4581 mlx5_sd_eswitch_mode_set(esw->dev, mlx5_mode); 4582 return err; 4583 } 4584 4585 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 4586 { 4587 struct mlx5_eswitch *esw; 4588 4589 esw = mlx5_devlink_eswitch_get(devlink); 4590 if (IS_ERR(esw)) 4591 return PTR_ERR(esw); 4592 4593 return esw_mode_to_devlink(esw, mode); 4594 } 4595 4596 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 4597 struct netlink_ext_ack *extack) 4598 { 4599 struct mlx5_core_dev *dev = esw->dev; 4600 struct mlx5_vport *vport; 4601 u16 err_vport_num = 0; 4602 unsigned long i; 4603 int err = 0; 4604 4605 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 4606 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 4607 if (err) { 4608 err_vport_num = vport->vport; 4609 NL_SET_ERR_MSG_MOD(extack, 4610 "Failed to set min inline on vport"); 4611 goto revert_inline_mode; 4612 } 4613 } 4614 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 4615 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 4616 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 4617 if (err) { 4618 err_vport_num = vport->vport; 4619 NL_SET_ERR_MSG_MOD(extack, 4620 "Failed to set min inline on vport"); 4621 goto revert_ec_vf_inline_mode; 4622 } 4623 } 4624 } 4625 return 0; 4626 4627 revert_ec_vf_inline_mode: 4628 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 4629 if (vport->vport == err_vport_num) 4630 break; 4631 mlx5_modify_nic_vport_min_inline(dev, 4632 vport->vport, 4633 esw->offloads.inline_mode); 4634 } 4635 revert_inline_mode: 4636 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 4637 if (vport->vport == err_vport_num) 4638 break; 4639 mlx5_modify_nic_vport_min_inline(dev, 4640 vport->vport, 4641 esw->offloads.inline_mode); 4642 } 4643 return err; 4644 } 4645 4646 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 4647 struct netlink_ext_ack *extack) 4648 { 4649 struct mlx5_core_dev *dev = devlink_priv(devlink); 4650 struct mlx5_eswitch *esw; 4651 u8 mlx5_mode; 4652 int err; 4653 4654 esw = mlx5_devlink_eswitch_get(devlink); 4655 if (IS_ERR(esw)) 4656 return PTR_ERR(esw); 4657 4658 down_write(&esw->mode_lock); 4659 4660 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 4661 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 4662 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 4663 err = 0; 4664 goto out; 4665 } 4666 4667 fallthrough; 4668 case MLX5_CAP_INLINE_MODE_L2: 4669 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 4670 err = -EOPNOTSUPP; 4671 goto out; 4672 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 4673 break; 4674 } 4675 4676 if (atomic64_read(&esw->offloads.num_flows) > 0) { 4677 NL_SET_ERR_MSG_MOD(extack, 4678 "Can't set inline mode when flows are configured"); 4679 err = -EOPNOTSUPP; 4680 goto out; 4681 } 4682 4683 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 4684 if (err) 4685 goto out; 4686 4687 esw->eswitch_operation_in_progress = true; 4688 up_write(&esw->mode_lock); 4689 4690 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 4691 if (!err) 4692 esw->offloads.inline_mode = mlx5_mode; 4693 4694 down_write(&esw->mode_lock); 4695 esw->eswitch_operation_in_progress = false; 4696 up_write(&esw->mode_lock); 4697 return 0; 4698 4699 out: 4700 up_write(&esw->mode_lock); 4701 return err; 4702 } 4703 4704 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 4705 { 4706 struct mlx5_eswitch *esw; 4707 4708 esw = mlx5_devlink_eswitch_get(devlink); 4709 if (IS_ERR(esw)) 4710 return PTR_ERR(esw); 4711 4712 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 4713 } 4714 4715 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev, bool from_fdb) 4716 { 4717 struct mlx5_eswitch *esw = dev->priv.eswitch; 4718 enum devlink_eswitch_encap_mode encap; 4719 bool allow_tunnel = false; 4720 4721 if (!mlx5_esw_allowed(esw)) 4722 return true; 4723 4724 down_write(&esw->mode_lock); 4725 encap = esw->offloads.encap; 4726 if (esw->mode == MLX5_ESWITCH_LEGACY || 4727 (encap == DEVLINK_ESWITCH_ENCAP_MODE_NONE && !from_fdb)) { 4728 allow_tunnel = true; 4729 esw->offloads.num_block_encap++; 4730 } 4731 up_write(&esw->mode_lock); 4732 4733 return allow_tunnel; 4734 } 4735 4736 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev) 4737 { 4738 struct mlx5_eswitch *esw = dev->priv.eswitch; 4739 4740 if (!mlx5_esw_allowed(esw)) 4741 return; 4742 4743 down_write(&esw->mode_lock); 4744 esw->offloads.num_block_encap--; 4745 up_write(&esw->mode_lock); 4746 } 4747 4748 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 4749 enum devlink_eswitch_encap_mode encap, 4750 struct netlink_ext_ack *extack) 4751 { 4752 struct mlx5_core_dev *dev = devlink_priv(devlink); 4753 struct mlx5_eswitch *esw; 4754 int err = 0; 4755 4756 esw = mlx5_devlink_eswitch_get(devlink); 4757 if (IS_ERR(esw)) 4758 return PTR_ERR(esw); 4759 4760 down_write(&esw->mode_lock); 4761 4762 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 4763 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 4764 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 4765 err = -EOPNOTSUPP; 4766 goto unlock; 4767 } 4768 4769 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 4770 err = -EOPNOTSUPP; 4771 goto unlock; 4772 } 4773 4774 if (esw->mode == MLX5_ESWITCH_LEGACY) { 4775 esw->offloads.encap = encap; 4776 goto unlock; 4777 } 4778 4779 if (esw->offloads.encap == encap) 4780 goto unlock; 4781 4782 if (atomic64_read(&esw->offloads.num_flows) > 0) { 4783 NL_SET_ERR_MSG_MOD(extack, 4784 "Can't set encapsulation when flows are configured"); 4785 err = -EOPNOTSUPP; 4786 goto unlock; 4787 } 4788 4789 if (esw->offloads.num_block_encap) { 4790 NL_SET_ERR_MSG_MOD(extack, 4791 "Can't set encapsulation when IPsec SA and/or policies are configured"); 4792 err = -EOPNOTSUPP; 4793 goto unlock; 4794 } 4795 4796 esw->eswitch_operation_in_progress = true; 4797 up_write(&esw->mode_lock); 4798 4799 esw_destroy_offloads_fdb_tables(esw); 4800 4801 esw->offloads.encap = encap; 4802 4803 err = esw_create_offloads_fdb_tables(esw); 4804 4805 if (err) { 4806 NL_SET_ERR_MSG_MOD(extack, 4807 "Failed re-creating fast FDB table"); 4808 esw->offloads.encap = !encap; 4809 (void)esw_create_offloads_fdb_tables(esw); 4810 } 4811 4812 down_write(&esw->mode_lock); 4813 esw->eswitch_operation_in_progress = false; 4814 4815 unlock: 4816 up_write(&esw->mode_lock); 4817 return err; 4818 } 4819 4820 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 4821 enum devlink_eswitch_encap_mode *encap) 4822 { 4823 struct mlx5_eswitch *esw; 4824 4825 esw = mlx5_devlink_eswitch_get(devlink); 4826 if (IS_ERR(esw)) 4827 return PTR_ERR(esw); 4828 4829 *encap = esw->offloads.encap; 4830 return 0; 4831 } 4832 4833 static bool 4834 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 4835 { 4836 /* Currently, only ECPF based device has representor for host PF. */ 4837 if (vport_num == MLX5_VPORT_HOST_PF && 4838 (!mlx5_core_is_ecpf_esw_manager(esw->dev) || 4839 !mlx5_esw_host_functions_enabled(esw->dev))) 4840 return false; 4841 4842 if (vport_num == MLX5_VPORT_ECPF && 4843 !mlx5_ecpf_vport_exists(esw->dev)) 4844 return false; 4845 4846 return true; 4847 } 4848 4849 static void 4850 mlx5_eswitch_register_vport_reps_blocked(struct mlx5_eswitch *esw, 4851 const struct mlx5_eswitch_rep_ops *ops, 4852 u8 rep_type) 4853 { 4854 struct mlx5_eswitch_rep_data *rep_data; 4855 struct mlx5_eswitch_rep *rep; 4856 unsigned long i; 4857 4858 esw->offloads.rep_ops[rep_type] = ops; 4859 mlx5_esw_for_each_rep(esw, i, rep) { 4860 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 4861 rep->esw = esw; 4862 rep_data = &rep->rep_data[rep_type]; 4863 atomic_set(&rep_data->state, REP_REGISTERED); 4864 } 4865 } 4866 } 4867 4868 static void mlx5_eswitch_reload_reps_blocked(struct mlx5_eswitch *esw) 4869 { 4870 struct mlx5_eswitch_rep *uplink; 4871 struct mlx5_vport *vport; 4872 bool newly_loaded; 4873 unsigned long i; 4874 4875 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 4876 return; 4877 4878 uplink = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 4879 if (__esw_offloads_load_rep(esw, uplink, REP_ETH, &newly_loaded)) 4880 return; 4881 if (mlx5_sd_is_primary(esw->dev) && 4882 __esw_offloads_load_rep(esw, uplink, REP_IB, NULL)) { 4883 if (newly_loaded) 4884 __esw_offloads_unload_rep(esw, uplink, REP_ETH); 4885 return; 4886 } 4887 4888 if (mlx5_get_sd(esw->dev) && !mlx5_lag_is_active(esw->dev)) 4889 return; 4890 4891 mlx5_esw_for_each_vport(esw, i, vport) { 4892 if (!vport) 4893 continue; 4894 if (!vport->enabled) 4895 continue; 4896 if (vport->vport == MLX5_VPORT_UPLINK) 4897 continue; 4898 if (!mlx5_eswitch_vport_has_rep(esw, vport->vport)) 4899 continue; 4900 4901 mlx5_esw_offloads_rep_load(esw, vport->vport); 4902 } 4903 } 4904 4905 static void mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw) 4906 { 4907 mlx5_esw_reps_block(esw); 4908 mlx5_eswitch_reload_reps_blocked(esw); 4909 mlx5_esw_reps_unblock(esw); 4910 } 4911 4912 static void 4913 mlx5_eswitch_register_vport_reps_locked(struct mlx5_eswitch *esw, 4914 const struct mlx5_eswitch_rep_ops *ops, 4915 u8 rep_type, bool nested) 4916 { 4917 if (nested) 4918 mlx5_esw_reps_block_nested(esw); 4919 else 4920 mlx5_esw_reps_block(esw); 4921 mlx5_eswitch_register_vport_reps_blocked(esw, ops, rep_type); 4922 mlx5_esw_reps_unblock(esw); 4923 4924 mlx5_esw_add_work(esw, mlx5_eswitch_reload_reps, GFP_KERNEL); 4925 } 4926 4927 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 4928 const struct mlx5_eswitch_rep_ops *ops, 4929 u8 rep_type) 4930 { 4931 mlx5_eswitch_register_vport_reps_locked(esw, ops, rep_type, false); 4932 } 4933 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 4934 4935 void 4936 mlx5_eswitch_register_vport_reps_nested(struct mlx5_eswitch *esw, 4937 const struct mlx5_eswitch_rep_ops *ops, 4938 u8 rep_type) 4939 { 4940 mlx5_eswitch_register_vport_reps_locked(esw, ops, rep_type, true); 4941 } 4942 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps_nested); 4943 4944 static void 4945 mlx5_eswitch_unregister_vport_reps_blocked(struct mlx5_eswitch *esw, 4946 u8 rep_type) 4947 { 4948 struct mlx5_eswitch_rep *rep; 4949 unsigned long i; 4950 4951 __unload_reps_all_vport(esw, rep_type); 4952 4953 mlx5_esw_for_each_rep(esw, i, rep) 4954 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 4955 4956 esw->offloads.rep_ops[rep_type] = NULL; 4957 } 4958 4959 static void 4960 mlx5_eswitch_unregister_vport_reps_locked(struct mlx5_eswitch *esw, 4961 u8 rep_type, bool nested) 4962 { 4963 if (nested) 4964 mlx5_esw_reps_block_nested(esw); 4965 else 4966 mlx5_esw_reps_block(esw); 4967 mlx5_eswitch_unregister_vport_reps_blocked(esw, rep_type); 4968 mlx5_esw_reps_unblock(esw); 4969 } 4970 4971 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 4972 { 4973 mlx5_eswitch_unregister_vport_reps_locked(esw, rep_type, false); 4974 } 4975 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 4976 4977 void mlx5_eswitch_unregister_vport_reps_nested(struct mlx5_eswitch *esw, 4978 u8 rep_type) 4979 { 4980 mlx5_eswitch_unregister_vport_reps_locked(esw, rep_type, true); 4981 } 4982 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps_nested); 4983 4984 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 4985 { 4986 struct mlx5_core_dev *primary = mlx5_sd_get_primary(esw->dev); 4987 struct mlx5_eswitch_rep *rep; 4988 4989 if (primary) 4990 esw = primary->priv.eswitch; 4991 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 4992 return rep->rep_data[rep_type].priv; 4993 } 4994 4995 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 4996 u16 vport, 4997 u8 rep_type) 4998 { 4999 struct mlx5_eswitch_rep *rep; 5000 5001 rep = mlx5_eswitch_get_rep(esw, vport); 5002 5003 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 5004 esw->offloads.rep_ops[rep_type]->get_proto_dev) 5005 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 5006 return NULL; 5007 } 5008 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 5009 5010 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 5011 { 5012 struct mlx5_core_dev *primary = mlx5_sd_get_primary(esw->dev); 5013 5014 if (primary) 5015 esw = primary->priv.eswitch; 5016 5017 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 5018 } 5019 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 5020 5021 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 5022 u16 vport) 5023 { 5024 return mlx5_eswitch_get_rep(esw, vport); 5025 } 5026 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 5027 5028 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 5029 { 5030 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 5031 } 5032 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 5033 5034 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 5035 { 5036 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 5037 } 5038 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 5039 5040 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 5041 u16 vport_num) 5042 { 5043 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 5044 5045 if (WARN_ON_ONCE(IS_ERR(vport))) 5046 return 0; 5047 5048 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 5049 } 5050 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 5051 5052 int mlx5_esw_vport_vhca_id_map(struct mlx5_eswitch *esw, 5053 struct mlx5_vport *vport) 5054 { 5055 u16 *old_entry, *vhca_map_entry, vhca_id; 5056 5057 if (WARN_ONCE(MLX5_VPORT_INVAL_VHCA_ID(vport), 5058 "vport %d vhca_id is not set", vport->vport)) { 5059 int err; 5060 5061 err = mlx5_vport_get_vhca_id(vport->dev, vport->vport, 5062 &vhca_id); 5063 if (err) 5064 return err; 5065 vport->vhca_id = vhca_id; 5066 } 5067 5068 vhca_id = vport->vhca_id; 5069 vhca_map_entry = kmalloc_obj(*vhca_map_entry); 5070 if (!vhca_map_entry) 5071 return -ENOMEM; 5072 5073 *vhca_map_entry = vport->vport; 5074 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 5075 if (xa_is_err(old_entry)) { 5076 kfree(vhca_map_entry); 5077 return xa_err(old_entry); 5078 } 5079 kfree(old_entry); 5080 return 0; 5081 } 5082 5083 void mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch *esw, 5084 struct mlx5_vport *vport) 5085 { 5086 u16 *vhca_map_entry; 5087 5088 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vport->vhca_id); 5089 kfree(vhca_map_entry); 5090 } 5091 5092 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 5093 { 5094 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 5095 5096 if (!res) 5097 return -ENOENT; 5098 5099 *vport_num = *res; 5100 return 0; 5101 } 5102 5103 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 5104 u16 vport_num) 5105 { 5106 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 5107 5108 if (WARN_ON_ONCE(IS_ERR(vport))) 5109 return 0; 5110 5111 return vport->metadata; 5112 } 5113 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 5114 5115 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port, 5116 u8 *hw_addr, int *hw_addr_len, 5117 struct netlink_ext_ack *extack) 5118 { 5119 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5120 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5121 5122 mutex_lock(&esw->state_lock); 5123 5124 mlx5_query_nic_vport_mac_address(esw->dev, vport->vport, true, 5125 vport->info.mac); 5126 ether_addr_copy(hw_addr, vport->info.mac); 5127 *hw_addr_len = ETH_ALEN; 5128 mutex_unlock(&esw->state_lock); 5129 return 0; 5130 } 5131 5132 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port, 5133 const u8 *hw_addr, int hw_addr_len, 5134 struct netlink_ext_ack *extack) 5135 { 5136 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5137 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5138 5139 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr); 5140 } 5141 5142 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, 5143 struct netlink_ext_ack *extack) 5144 { 5145 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5146 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5147 5148 if (!MLX5_CAP_GEN(esw->dev, migration)) { 5149 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 5150 return -EOPNOTSUPP; 5151 } 5152 5153 if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) { 5154 NL_SET_ERR_MSG_MOD(extack, 5155 "Device doesn't support vport group management"); 5156 return -EOPNOTSUPP; 5157 } 5158 5159 mutex_lock(&esw->state_lock); 5160 *is_enabled = vport->info.mig_enabled; 5161 mutex_unlock(&esw->state_lock); 5162 return 0; 5163 } 5164 5165 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, 5166 struct netlink_ext_ack *extack) 5167 { 5168 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5169 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5170 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 5171 void *query_ctx; 5172 void *hca_caps; 5173 int err; 5174 5175 if (!MLX5_CAP_GEN(esw->dev, migration)) { 5176 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 5177 return -EOPNOTSUPP; 5178 } 5179 5180 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 5181 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 5182 return -EOPNOTSUPP; 5183 } 5184 5185 mutex_lock(&esw->state_lock); 5186 5187 if (vport->info.mig_enabled == enable) { 5188 err = 0; 5189 goto out; 5190 } 5191 5192 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 5193 if (!query_ctx) { 5194 err = -ENOMEM; 5195 goto out; 5196 } 5197 5198 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, 5199 MLX5_CAP_GENERAL_2); 5200 if (err) { 5201 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 5202 goto out_free; 5203 } 5204 5205 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 5206 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable); 5207 5208 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, 5209 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 5210 if (err) { 5211 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); 5212 goto out_free; 5213 } 5214 5215 vport->info.mig_enabled = enable; 5216 5217 out_free: 5218 kfree(query_ctx); 5219 out: 5220 mutex_unlock(&esw->state_lock); 5221 return err; 5222 } 5223 5224 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, 5225 struct netlink_ext_ack *extack) 5226 { 5227 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5228 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5229 5230 if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) { 5231 NL_SET_ERR_MSG_MOD(extack, 5232 "Device doesn't support vport group management"); 5233 return -EOPNOTSUPP; 5234 } 5235 5236 mutex_lock(&esw->state_lock); 5237 *is_enabled = vport->info.roce_enabled; 5238 mutex_unlock(&esw->state_lock); 5239 return 0; 5240 } 5241 5242 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, 5243 struct netlink_ext_ack *extack) 5244 { 5245 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5246 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5247 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 5248 u16 vport_num = vport->vport; 5249 void *query_ctx; 5250 void *hca_caps; 5251 int err; 5252 5253 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 5254 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 5255 return -EOPNOTSUPP; 5256 } 5257 5258 mutex_lock(&esw->state_lock); 5259 5260 if (vport->info.roce_enabled == enable) { 5261 err = 0; 5262 goto out; 5263 } 5264 5265 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 5266 if (!query_ctx) { 5267 err = -ENOMEM; 5268 goto out; 5269 } 5270 5271 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 5272 MLX5_CAP_GENERAL); 5273 if (err) { 5274 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 5275 goto out_free; 5276 } 5277 5278 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 5279 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); 5280 5281 err = mlx5_vport_set_other_func_general_cap(esw->dev, hca_caps, 5282 vport_num); 5283 if (err) { 5284 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); 5285 goto out_free; 5286 } 5287 5288 vport->info.roce_enabled = enable; 5289 5290 out_free: 5291 kfree(query_ctx); 5292 out: 5293 mutex_unlock(&esw->state_lock); 5294 return err; 5295 } 5296 5297 int mlx5_devlink_pf_port_fn_state_get(struct devlink_port *port, 5298 enum devlink_port_fn_state *state, 5299 enum devlink_port_fn_opstate *opstate, 5300 struct netlink_ext_ack *extack) 5301 { 5302 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5303 struct mlx5_eswitch *esw = vport->dev->priv.eswitch; 5304 const u32 *query_out; 5305 bool pf_disabled; 5306 5307 if (mlx5_eswitch_is_vf_vport(esw, vport->vport)) { 5308 NL_SET_ERR_MSG_MOD(extack, "State get is not supported for VF"); 5309 return -EOPNOTSUPP; 5310 } 5311 5312 *state = vport->pf_activated ? 5313 DEVLINK_PORT_FN_STATE_ACTIVE : DEVLINK_PORT_FN_STATE_INACTIVE; 5314 5315 query_out = mlx5_esw_query_functions(vport->dev); 5316 if (IS_ERR(query_out)) 5317 return PTR_ERR(query_out); 5318 5319 if (vport->vport == MLX5_VPORT_HOST_PF) { 5320 struct mlx5_esw_pf_info host_pf_info; 5321 5322 host_pf_info = mlx5_esw_get_host_pf_info(vport->dev, 5323 query_out); 5324 pf_disabled = host_pf_info.pf_disabled; 5325 } else { 5326 pf_disabled = mlx5_esw_get_spf_disabled(vport->dev, query_out, 5327 vport->vhca_id); 5328 } 5329 5330 *opstate = pf_disabled ? DEVLINK_PORT_FN_OPSTATE_DETACHED : 5331 DEVLINK_PORT_FN_OPSTATE_ATTACHED; 5332 5333 kvfree(query_out); 5334 return 0; 5335 } 5336 5337 int mlx5_devlink_pf_port_fn_state_set(struct devlink_port *port, 5338 enum devlink_port_fn_state state, 5339 struct netlink_ext_ack *extack) 5340 { 5341 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5342 struct mlx5_eswitch *esw = vport->dev->priv.eswitch; 5343 struct mlx5_core_dev *dev; 5344 5345 if (mlx5_eswitch_is_vf_vport(esw, vport->vport)) { 5346 NL_SET_ERR_MSG_MOD(extack, "State set is not supported for VF"); 5347 return -EOPNOTSUPP; 5348 } 5349 5350 dev = vport->dev; 5351 5352 switch (state) { 5353 case DEVLINK_PORT_FN_STATE_ACTIVE: 5354 return mlx5_esw_pf_enable_hca(dev, vport->vport); 5355 case DEVLINK_PORT_FN_STATE_INACTIVE: 5356 return mlx5_esw_pf_disable_hca(dev, vport->vport); 5357 default: 5358 return -EOPNOTSUPP; 5359 } 5360 } 5361 5362 int 5363 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule, 5364 struct mlx5_esw_flow_attr *esw_attr, int attr_idx) 5365 { 5366 struct mlx5_flow_destination new_dest = {}; 5367 struct mlx5_flow_destination old_dest = {}; 5368 5369 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 5370 return 0; 5371 5372 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false); 5373 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false); 5374 5375 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest); 5376 } 5377 5378 #ifdef CONFIG_XFRM_OFFLOAD 5379 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled, 5380 struct netlink_ext_ack *extack) 5381 { 5382 struct mlx5_eswitch *esw; 5383 struct mlx5_vport *vport; 5384 int err = 0; 5385 5386 esw = mlx5_devlink_eswitch_get(port->devlink); 5387 if (IS_ERR(esw)) 5388 return PTR_ERR(esw); 5389 5390 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 5391 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto"); 5392 return -EOPNOTSUPP; 5393 } 5394 5395 vport = mlx5_devlink_port_vport_get(port); 5396 5397 mutex_lock(&esw->state_lock); 5398 if (!vport->enabled) { 5399 err = -EOPNOTSUPP; 5400 goto unlock; 5401 } 5402 5403 *is_enabled = vport->info.ipsec_crypto_enabled; 5404 unlock: 5405 mutex_unlock(&esw->state_lock); 5406 return err; 5407 } 5408 5409 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable, 5410 struct netlink_ext_ack *extack) 5411 { 5412 struct mlx5_eswitch *esw; 5413 struct mlx5_vport *vport; 5414 u16 vport_num; 5415 int err; 5416 5417 esw = mlx5_devlink_eswitch_get(port->devlink); 5418 if (IS_ERR(esw)) 5419 return PTR_ERR(esw); 5420 5421 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 5422 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num); 5423 if (err) { 5424 NL_SET_ERR_MSG_MOD(extack, 5425 "Device doesn't support IPsec crypto"); 5426 return err; 5427 } 5428 5429 vport = mlx5_devlink_port_vport_get(port); 5430 5431 mutex_lock(&esw->state_lock); 5432 if (!vport->enabled) { 5433 err = -EOPNOTSUPP; 5434 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 5435 goto unlock; 5436 } 5437 5438 if (vport->info.ipsec_crypto_enabled == enable) 5439 goto unlock; 5440 5441 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 5442 err = -EBUSY; 5443 goto unlock; 5444 } 5445 5446 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable); 5447 if (err) { 5448 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto"); 5449 goto unlock; 5450 } 5451 5452 vport->info.ipsec_crypto_enabled = enable; 5453 if (enable) 5454 esw->enabled_ipsec_vf_count++; 5455 else 5456 esw->enabled_ipsec_vf_count--; 5457 unlock: 5458 mutex_unlock(&esw->state_lock); 5459 return err; 5460 } 5461 5462 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled, 5463 struct netlink_ext_ack *extack) 5464 { 5465 struct mlx5_eswitch *esw; 5466 struct mlx5_vport *vport; 5467 int err = 0; 5468 5469 esw = mlx5_devlink_eswitch_get(port->devlink); 5470 if (IS_ERR(esw)) 5471 return PTR_ERR(esw); 5472 5473 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 5474 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet"); 5475 return -EOPNOTSUPP; 5476 } 5477 5478 vport = mlx5_devlink_port_vport_get(port); 5479 5480 mutex_lock(&esw->state_lock); 5481 if (!vport->enabled) { 5482 err = -EOPNOTSUPP; 5483 goto unlock; 5484 } 5485 5486 *is_enabled = vport->info.ipsec_packet_enabled; 5487 unlock: 5488 mutex_unlock(&esw->state_lock); 5489 return err; 5490 } 5491 5492 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, 5493 bool enable, 5494 struct netlink_ext_ack *extack) 5495 { 5496 struct mlx5_eswitch *esw; 5497 struct mlx5_vport *vport; 5498 u16 vport_num; 5499 int err; 5500 5501 esw = mlx5_devlink_eswitch_get(port->devlink); 5502 if (IS_ERR(esw)) 5503 return PTR_ERR(esw); 5504 5505 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 5506 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num); 5507 if (err) { 5508 NL_SET_ERR_MSG_MOD(extack, 5509 "Device doesn't support IPsec packet mode"); 5510 return err; 5511 } 5512 5513 vport = mlx5_devlink_port_vport_get(port); 5514 mutex_lock(&esw->state_lock); 5515 if (!vport->enabled) { 5516 err = -EOPNOTSUPP; 5517 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 5518 goto unlock; 5519 } 5520 5521 if (vport->info.ipsec_packet_enabled == enable) 5522 goto unlock; 5523 5524 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 5525 err = -EBUSY; 5526 goto unlock; 5527 } 5528 5529 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable); 5530 if (err) { 5531 NL_SET_ERR_MSG_MOD(extack, 5532 "Failed to set IPsec packet mode"); 5533 goto unlock; 5534 } 5535 5536 vport->info.ipsec_packet_enabled = enable; 5537 if (enable) 5538 esw->enabled_ipsec_vf_count++; 5539 else 5540 esw->enabled_ipsec_vf_count--; 5541 unlock: 5542 mutex_unlock(&esw->state_lock); 5543 return err; 5544 } 5545 #endif /* CONFIG_XFRM_OFFLOAD */ 5546 5547 int 5548 mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs, 5549 struct netlink_ext_ack *extack) 5550 { 5551 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5552 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 5553 u16 vport_num = vport->vport; 5554 struct mlx5_eswitch *esw; 5555 void *query_ctx; 5556 void *hca_caps; 5557 u32 max_eqs; 5558 int err; 5559 5560 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5561 if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) { 5562 NL_SET_ERR_MSG_MOD(extack, 5563 "Device doesn't support vport group management"); 5564 return -EOPNOTSUPP; 5565 } 5566 5567 if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) { 5568 NL_SET_ERR_MSG_MOD(extack, 5569 "Device doesn't support getting the max number of EQs"); 5570 return -EOPNOTSUPP; 5571 } 5572 5573 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 5574 if (!query_ctx) 5575 return -ENOMEM; 5576 5577 mutex_lock(&esw->state_lock); 5578 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 5579 MLX5_CAP_GENERAL_2); 5580 if (err) { 5581 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 5582 goto out; 5583 } 5584 5585 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 5586 max_eqs = MLX5_GET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b); 5587 if (max_eqs < MLX5_ESW_MAX_CTRL_EQS) 5588 *max_io_eqs = 0; 5589 else 5590 *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS; 5591 out: 5592 mutex_unlock(&esw->state_lock); 5593 kfree(query_ctx); 5594 return err; 5595 } 5596 5597 int 5598 mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs, 5599 struct netlink_ext_ack *extack) 5600 { 5601 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 5602 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 5603 u16 vport_num = vport->vport; 5604 struct mlx5_eswitch *esw; 5605 void *query_ctx; 5606 void *hca_caps; 5607 u16 max_eqs; 5608 int err; 5609 5610 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 5611 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 5612 NL_SET_ERR_MSG_MOD(extack, 5613 "Device doesn't support VHCA management"); 5614 return -EOPNOTSUPP; 5615 } 5616 5617 if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) { 5618 NL_SET_ERR_MSG_MOD(extack, 5619 "Device doesn't support changing the max number of EQs"); 5620 return -EOPNOTSUPP; 5621 } 5622 5623 if (check_add_overflow(max_io_eqs, MLX5_ESW_MAX_CTRL_EQS, &max_eqs)) { 5624 NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range"); 5625 return -EINVAL; 5626 } 5627 5628 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 5629 if (!query_ctx) 5630 return -ENOMEM; 5631 5632 mutex_lock(&esw->state_lock); 5633 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 5634 MLX5_CAP_GENERAL_2); 5635 if (err) { 5636 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 5637 goto out; 5638 } 5639 5640 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 5641 MLX5_SET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b, max_eqs); 5642 5643 if (mlx5_esw_is_sf_vport(esw, vport_num)) 5644 MLX5_SET(cmd_hca_cap_2, hca_caps, sf_eq_usage, 1); 5645 5646 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 5647 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 5648 if (err) 5649 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA caps"); 5650 vport->max_eqs_set = true; 5651 out: 5652 mutex_unlock(&esw->state_lock); 5653 kfree(query_ctx); 5654 return err; 5655 } 5656 5657 int 5658 mlx5_devlink_port_fn_max_io_eqs_set_sf_default(struct devlink_port *port, 5659 struct netlink_ext_ack *extack) 5660 { 5661 return mlx5_devlink_port_fn_max_io_eqs_set(port, 5662 MLX5_ESW_DEFAULT_SF_COMP_EQS, 5663 extack); 5664 } 5665