1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ 3 4 #include "lib/sd.h" 5 #include "mlx5_core.h" 6 #include "lib/mlx5.h" 7 #include "fs_cmd.h" 8 #include <linux/mlx5/vport.h> 9 #include <linux/debugfs.h> 10 11 #define sd_info(__dev, format, ...) \ 12 dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) 13 #define sd_warn(__dev, format, ...) \ 14 dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) 15 16 struct mlx5_sd { 17 u32 group_id; 18 u8 host_buses; 19 struct mlx5_devcom_comp_dev *devcom; 20 struct dentry *dfs; 21 bool primary; 22 union { 23 struct { /* primary */ 24 struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1]; 25 struct mlx5_flow_table *tx_ft; 26 }; 27 struct { /* secondary */ 28 struct mlx5_core_dev *primary_dev; 29 u32 alias_obj_id; 30 }; 31 }; 32 }; 33 34 static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) 35 { 36 struct mlx5_sd *sd = mlx5_get_sd(dev); 37 38 if (!sd) 39 return 1; 40 41 return sd->host_buses; 42 } 43 44 static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev) 45 { 46 struct mlx5_sd *sd = mlx5_get_sd(dev); 47 48 if (!sd) 49 return dev; 50 51 return sd->primary ? dev : sd->primary_dev; 52 } 53 54 struct mlx5_core_dev * 55 mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx) 56 { 57 struct mlx5_sd *sd; 58 59 if (idx == 0) 60 return primary; 61 62 if (idx >= mlx5_sd_get_host_buses(primary)) 63 return NULL; 64 65 sd = mlx5_get_sd(primary); 66 return sd->secondaries[idx - 1]; 67 } 68 69 int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix) 70 { 71 return ch_ix % mlx5_sd_get_host_buses(dev); 72 } 73 74 int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix) 75 { 76 return ch_ix / mlx5_sd_get_host_buses(dev); 77 } 78 79 struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix) 80 { 81 int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix); 82 83 return mlx5_sd_primary_get_peer(primary, mdev_idx); 84 } 85 86 static bool ft_create_alias_supported(struct mlx5_core_dev *dev) 87 { 88 u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access); 89 u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported); 90 91 if (!(obj_supp & 92 MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE)) 93 return false; 94 95 if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE)) 96 return false; 97 98 return true; 99 } 100 101 static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses) 102 { 103 /* Honor the SW implementation limit */ 104 if (host_buses > MLX5_SD_MAX_GROUP_SZ) 105 return false; 106 107 /* Disconnect secondaries from the network */ 108 if (!MLX5_CAP_GEN(dev, eswitch_manager)) 109 return false; 110 if (!MLX5_CAP_GEN(dev, silent_mode_set)) 111 return false; 112 113 /* RX steering from primary to secondaries */ 114 if (!MLX5_CAP_GEN(dev, cross_vhca_rqt)) 115 return false; 116 if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id)) 117 return false; 118 119 /* TX steering from secondaries to primary */ 120 if (!ft_create_alias_supported(dev)) 121 return false; 122 if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default)) 123 return false; 124 125 return true; 126 } 127 128 static int mlx5_query_sd(struct mlx5_core_dev *dev, bool *sdm, 129 u8 *host_buses) 130 { 131 u32 out[MLX5_ST_SZ_DW(mpir_reg)]; 132 int err; 133 134 err = mlx5_query_mpir_reg(dev, out); 135 if (err) 136 return err; 137 138 *sdm = MLX5_GET(mpir_reg, out, sdm); 139 *host_buses = MLX5_GET(mpir_reg, out, host_buses); 140 141 return 0; 142 } 143 144 static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group) 145 { 146 return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) | sd_group); 147 } 148 149 static int sd_init(struct mlx5_core_dev *dev) 150 { 151 u8 host_buses, sd_group; 152 struct mlx5_sd *sd; 153 u32 group_id; 154 bool sdm; 155 int err; 156 157 /* Feature is currently implemented for PFs only */ 158 if (!mlx5_core_is_pf(dev)) 159 return 0; 160 161 /* Block on embedded CPU PFs */ 162 if (mlx5_core_is_ecpf(dev)) 163 return 0; 164 165 err = mlx5_query_nic_vport_sd_group(dev, &sd_group); 166 if (err) 167 return err; 168 169 if (!sd_group) 170 return 0; 171 172 if (!MLX5_CAP_MCAM_REG(dev, mpir)) 173 return 0; 174 175 err = mlx5_query_sd(dev, &sdm, &host_buses); 176 if (err) 177 return err; 178 179 if (!sdm) 180 return 0; 181 182 group_id = mlx5_sd_group_id(dev, sd_group); 183 184 if (!mlx5_sd_is_supported(dev, host_buses)) { 185 sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n", 186 group_id); 187 return 0; 188 } 189 190 sd = kzalloc_obj(*sd); 191 if (!sd) 192 return -ENOMEM; 193 194 sd->host_buses = host_buses; 195 sd->group_id = group_id; 196 197 mlx5_set_sd(dev, sd); 198 199 return 0; 200 } 201 202 static void sd_cleanup(struct mlx5_core_dev *dev) 203 { 204 struct mlx5_sd *sd = mlx5_get_sd(dev); 205 206 mlx5_set_sd(dev, NULL); 207 kfree(sd); 208 } 209 210 static int sd_register(struct mlx5_core_dev *dev) 211 { 212 struct mlx5_devcom_comp_dev *devcom, *pos; 213 struct mlx5_devcom_match_attr attr = {}; 214 struct mlx5_core_dev *peer, *primary; 215 struct mlx5_sd *sd, *primary_sd; 216 int err, i; 217 218 sd = mlx5_get_sd(dev); 219 attr.key.val = sd->group_id; 220 attr.flags = MLX5_DEVCOM_MATCH_FLAGS_NS; 221 attr.net = mlx5_core_net(dev); 222 devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP, 223 &attr, NULL, dev); 224 if (!devcom) 225 return -EINVAL; 226 227 sd->devcom = devcom; 228 229 if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses) 230 return 0; 231 232 mlx5_devcom_comp_lock(devcom); 233 mlx5_devcom_comp_set_ready(devcom, true); 234 mlx5_devcom_comp_unlock(devcom); 235 236 if (!mlx5_devcom_for_each_peer_begin(devcom)) { 237 err = -ENODEV; 238 goto err_devcom_unreg; 239 } 240 241 primary = dev; 242 mlx5_devcom_for_each_peer_entry(devcom, peer, pos) 243 if (peer->pdev->bus->number < primary->pdev->bus->number) 244 primary = peer; 245 246 primary_sd = mlx5_get_sd(primary); 247 primary_sd->primary = true; 248 i = 0; 249 /* loop the secondaries */ 250 mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) { 251 struct mlx5_sd *peer_sd = mlx5_get_sd(peer); 252 253 primary_sd->secondaries[i++] = peer; 254 peer_sd->primary = false; 255 peer_sd->primary_dev = primary; 256 } 257 258 mlx5_devcom_for_each_peer_end(devcom); 259 return 0; 260 261 err_devcom_unreg: 262 mlx5_devcom_comp_lock(sd->devcom); 263 mlx5_devcom_comp_set_ready(sd->devcom, false); 264 mlx5_devcom_comp_unlock(sd->devcom); 265 mlx5_devcom_unregister_component(sd->devcom); 266 return err; 267 } 268 269 static void sd_unregister(struct mlx5_core_dev *dev) 270 { 271 struct mlx5_sd *sd = mlx5_get_sd(dev); 272 273 mlx5_devcom_comp_lock(sd->devcom); 274 mlx5_devcom_comp_set_ready(sd->devcom, false); 275 mlx5_devcom_comp_unlock(sd->devcom); 276 mlx5_devcom_unregister_component(sd->devcom); 277 } 278 279 static int sd_cmd_set_primary(struct mlx5_core_dev *primary, u8 *alias_key) 280 { 281 struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {}; 282 struct mlx5_sd *sd = mlx5_get_sd(primary); 283 struct mlx5_flow_table_attr ft_attr = {}; 284 struct mlx5_flow_namespace *nic_ns; 285 struct mlx5_flow_table *ft; 286 int err; 287 288 nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS); 289 if (!nic_ns) 290 return -EOPNOTSUPP; 291 292 ft = mlx5_create_flow_table(nic_ns, &ft_attr); 293 if (IS_ERR(ft)) { 294 err = PTR_ERR(ft); 295 return err; 296 } 297 sd->tx_ft = ft; 298 memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN); 299 allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; 300 allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; 301 302 err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr); 303 if (err) { 304 mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n", 305 err); 306 mlx5_destroy_flow_table(ft); 307 return err; 308 } 309 310 return 0; 311 } 312 313 static void sd_cmd_unset_primary(struct mlx5_core_dev *primary) 314 { 315 struct mlx5_sd *sd = mlx5_get_sd(primary); 316 317 mlx5_destroy_flow_table(sd->tx_ft); 318 } 319 320 static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary, 321 struct mlx5_core_dev *primary, 322 struct mlx5_flow_table *ft, 323 u32 *obj_id, u8 *alias_key) 324 { 325 u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; 326 u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id); 327 struct mlx5_cmd_alias_obj_create_attr alias_attr = {}; 328 int ret; 329 330 memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN); 331 alias_attr.obj_id = aliased_object_id; 332 alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; 333 alias_attr.vhca_id = vhca_id_to_be_accessed; 334 ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id); 335 if (ret) { 336 mlx5_core_err(secondary, "Failed to create alias object err=%d\n", 337 ret); 338 return ret; 339 } 340 341 return 0; 342 } 343 344 static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary) 345 { 346 struct mlx5_sd *sd = mlx5_get_sd(secondary); 347 348 mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id, 349 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); 350 } 351 352 static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary, 353 struct mlx5_core_dev *primary, 354 u8 *alias_key) 355 { 356 struct mlx5_sd *primary_sd = mlx5_get_sd(primary); 357 struct mlx5_sd *sd = mlx5_get_sd(secondary); 358 int err; 359 360 err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1); 361 if (err) 362 return err; 363 364 err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft, 365 &sd->alias_obj_id, alias_key); 366 if (err) 367 goto err_unset_silent; 368 369 err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false); 370 if (err) 371 goto err_destroy_alias_ft; 372 373 return 0; 374 375 err_destroy_alias_ft: 376 sd_secondary_destroy_alias_ft(secondary); 377 err_unset_silent: 378 mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); 379 return err; 380 } 381 382 static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary) 383 { 384 mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true); 385 sd_secondary_destroy_alias_ft(secondary); 386 mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); 387 } 388 389 static void sd_print_group(struct mlx5_core_dev *primary) 390 { 391 struct mlx5_sd *sd = mlx5_get_sd(primary); 392 struct mlx5_core_dev *pos; 393 int i; 394 395 sd_info(primary, "group id %#x, primary %s, vhca %#x\n", 396 sd->group_id, pci_name(primary->pdev), 397 MLX5_CAP_GEN(primary, vhca_id)); 398 mlx5_sd_for_each_secondary(i, primary, pos) 399 sd_info(primary, "group id %#x, secondary_%d %s, vhca %#x\n", 400 sd->group_id, i - 1, pci_name(pos->pdev), 401 MLX5_CAP_GEN(pos, vhca_id)); 402 } 403 404 static ssize_t dev_read(struct file *filp, char __user *buf, size_t count, 405 loff_t *pos) 406 { 407 struct mlx5_core_dev *dev; 408 char tbuf[32]; 409 int ret; 410 411 dev = filp->private_data; 412 ret = snprintf(tbuf, sizeof(tbuf), "%s vhca %#x\n", pci_name(dev->pdev), 413 MLX5_CAP_GEN(dev, vhca_id)); 414 415 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 416 } 417 418 static const struct file_operations dev_fops = { 419 .owner = THIS_MODULE, 420 .open = simple_open, 421 .read = dev_read, 422 }; 423 424 int mlx5_sd_init(struct mlx5_core_dev *dev) 425 { 426 struct mlx5_core_dev *primary, *pos, *to; 427 struct mlx5_sd *sd = mlx5_get_sd(dev); 428 u8 alias_key[ACCESS_KEY_LEN]; 429 int err, i; 430 431 err = sd_init(dev); 432 if (err) 433 return err; 434 435 sd = mlx5_get_sd(dev); 436 if (!sd) 437 return 0; 438 439 err = sd_register(dev); 440 if (err) 441 goto err_sd_cleanup; 442 443 if (!mlx5_devcom_comp_is_ready(sd->devcom)) 444 return 0; 445 446 primary = mlx5_sd_get_primary(dev); 447 448 for (i = 0; i < ACCESS_KEY_LEN; i++) 449 alias_key[i] = get_random_u8(); 450 451 err = sd_cmd_set_primary(primary, alias_key); 452 if (err) 453 goto err_sd_unregister; 454 455 sd->dfs = debugfs_create_dir("multi-pf", mlx5_debugfs_get_dev_root(primary)); 456 debugfs_create_x32("group_id", 0400, sd->dfs, &sd->group_id); 457 debugfs_create_file("primary", 0400, sd->dfs, primary, &dev_fops); 458 459 mlx5_sd_for_each_secondary(i, primary, pos) { 460 char name[32]; 461 462 err = sd_cmd_set_secondary(pos, primary, alias_key); 463 if (err) 464 goto err_unset_secondaries; 465 466 snprintf(name, sizeof(name), "secondary_%d", i - 1); 467 debugfs_create_file(name, 0400, sd->dfs, pos, &dev_fops); 468 469 } 470 471 sd_info(primary, "group id %#x, size %d, combined\n", 472 sd->group_id, mlx5_devcom_comp_get_size(sd->devcom)); 473 sd_print_group(primary); 474 475 return 0; 476 477 err_unset_secondaries: 478 to = pos; 479 mlx5_sd_for_each_secondary_to(i, primary, to, pos) 480 sd_cmd_unset_secondary(pos); 481 sd_cmd_unset_primary(primary); 482 debugfs_remove_recursive(sd->dfs); 483 err_sd_unregister: 484 sd_unregister(dev); 485 err_sd_cleanup: 486 sd_cleanup(dev); 487 return err; 488 } 489 490 void mlx5_sd_cleanup(struct mlx5_core_dev *dev) 491 { 492 struct mlx5_sd *sd = mlx5_get_sd(dev); 493 struct mlx5_core_dev *primary, *pos; 494 int i; 495 496 if (!sd) 497 return; 498 499 if (!mlx5_devcom_comp_is_ready(sd->devcom)) 500 goto out; 501 502 primary = mlx5_sd_get_primary(dev); 503 mlx5_sd_for_each_secondary(i, primary, pos) 504 sd_cmd_unset_secondary(pos); 505 sd_cmd_unset_primary(primary); 506 debugfs_remove_recursive(sd->dfs); 507 508 sd_info(primary, "group id %#x, uncombined\n", sd->group_id); 509 out: 510 sd_unregister(dev); 511 sd_cleanup(dev); 512 } 513 514 struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, 515 struct auxiliary_device *adev, 516 int idx) 517 { 518 struct mlx5_sd *sd = mlx5_get_sd(dev); 519 struct mlx5_core_dev *primary; 520 521 if (!sd) 522 return adev; 523 524 if (!mlx5_devcom_comp_is_ready(sd->devcom)) 525 return NULL; 526 527 primary = mlx5_sd_get_primary(dev); 528 if (dev == primary) 529 return adev; 530 531 return &primary->priv.adev[idx]->adev; 532 } 533