1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020 Mellanox Technologies Ltd */ 3 4 #include <linux/mlx5/driver.h> 5 #include "eswitch.h" 6 #include "priv.h" 7 #include "sf/dev/dev.h" 8 #include "mlx5_ifc_vhca_event.h" 9 #include "vhca_event.h" 10 #include "ecpf.h" 11 #define CREATE_TRACE_POINTS 12 #include "diag/sf_tracepoint.h" 13 14 struct mlx5_sf { 15 struct mlx5_devlink_port dl_port; 16 unsigned int port_index; 17 u32 controller; 18 u16 id; 19 u16 hw_fn_id; 20 u16 hw_state; 21 }; 22 23 static void *mlx5_sf_by_dl_port(struct devlink_port *dl_port) 24 { 25 struct mlx5_devlink_port *mlx5_dl_port = mlx5_devlink_port_get(dl_port); 26 27 return container_of(mlx5_dl_port, struct mlx5_sf, dl_port); 28 } 29 30 struct mlx5_sf_table { 31 struct mlx5_core_dev *dev; /* To refer from notifier context. */ 32 struct xarray function_ids; /* function id based lookup. */ 33 struct mutex sf_state_lock; /* Serializes sf state among user cmds & vhca event handler. */ 34 }; 35 36 static struct mlx5_sf * 37 mlx5_sf_lookup_by_function_id(struct mlx5_sf_table *table, unsigned int fn_id) 38 { 39 return xa_load(&table->function_ids, fn_id); 40 } 41 42 static int mlx5_sf_function_id_insert(struct mlx5_sf_table *table, struct mlx5_sf *sf) 43 { 44 return xa_insert(&table->function_ids, sf->hw_fn_id, sf, GFP_KERNEL); 45 } 46 47 static void mlx5_sf_function_id_erase(struct mlx5_sf_table *table, struct mlx5_sf *sf) 48 { 49 xa_erase(&table->function_ids, sf->hw_fn_id); 50 } 51 52 static struct mlx5_sf * 53 mlx5_sf_alloc(struct mlx5_sf_table *table, struct mlx5_eswitch *esw, 54 u32 controller, u32 sfnum, struct netlink_ext_ack *extack) 55 { 56 unsigned int dl_port_index; 57 struct mlx5_sf *sf; 58 u16 hw_fn_id; 59 int id_err; 60 int err; 61 62 if (!mlx5_esw_offloads_controller_valid(esw, controller)) { 63 NL_SET_ERR_MSG_MOD(extack, "Invalid controller number"); 64 return ERR_PTR(-EINVAL); 65 } 66 67 id_err = mlx5_sf_hw_table_sf_alloc(table->dev, controller, sfnum); 68 if (id_err < 0) { 69 err = id_err; 70 goto id_err; 71 } 72 73 sf = kzalloc_obj(*sf); 74 if (!sf) { 75 err = -ENOMEM; 76 goto alloc_err; 77 } 78 sf->id = id_err; 79 hw_fn_id = mlx5_sf_sw_to_hw_id(table->dev, controller, sf->id); 80 dl_port_index = mlx5_esw_vport_to_devlink_port_index(table->dev, hw_fn_id); 81 sf->port_index = dl_port_index; 82 sf->hw_fn_id = hw_fn_id; 83 sf->hw_state = MLX5_VHCA_STATE_ALLOCATED; 84 sf->controller = controller; 85 86 err = mlx5_sf_function_id_insert(table, sf); 87 if (err) 88 goto insert_err; 89 90 return sf; 91 92 insert_err: 93 kfree(sf); 94 alloc_err: 95 mlx5_sf_hw_table_sf_free(table->dev, controller, id_err); 96 id_err: 97 if (err == -EEXIST) 98 NL_SET_ERR_MSG_MOD(extack, "SF already exist. Choose different sfnum"); 99 return ERR_PTR(err); 100 } 101 102 static void mlx5_sf_free(struct mlx5_sf_table *table, struct mlx5_sf *sf) 103 { 104 mlx5_sf_hw_table_sf_free(table->dev, sf->controller, sf->id); 105 trace_mlx5_sf_free(table->dev, sf->port_index, sf->controller, sf->hw_fn_id); 106 kfree(sf); 107 } 108 109 static enum devlink_port_fn_state mlx5_sf_to_devlink_state(u8 hw_state) 110 { 111 switch (hw_state) { 112 case MLX5_VHCA_STATE_ACTIVE: 113 case MLX5_VHCA_STATE_IN_USE: 114 return DEVLINK_PORT_FN_STATE_ACTIVE; 115 case MLX5_VHCA_STATE_INVALID: 116 case MLX5_VHCA_STATE_ALLOCATED: 117 case MLX5_VHCA_STATE_TEARDOWN_REQUEST: 118 default: 119 return DEVLINK_PORT_FN_STATE_INACTIVE; 120 } 121 } 122 123 static enum devlink_port_fn_opstate mlx5_sf_to_devlink_opstate(u8 hw_state) 124 { 125 switch (hw_state) { 126 case MLX5_VHCA_STATE_IN_USE: 127 case MLX5_VHCA_STATE_TEARDOWN_REQUEST: 128 return DEVLINK_PORT_FN_OPSTATE_ATTACHED; 129 case MLX5_VHCA_STATE_INVALID: 130 case MLX5_VHCA_STATE_ALLOCATED: 131 case MLX5_VHCA_STATE_ACTIVE: 132 default: 133 return DEVLINK_PORT_FN_OPSTATE_DETACHED; 134 } 135 } 136 137 static bool mlx5_sf_is_active(const struct mlx5_sf *sf) 138 { 139 return sf->hw_state == MLX5_VHCA_STATE_ACTIVE || sf->hw_state == MLX5_VHCA_STATE_IN_USE; 140 } 141 142 int mlx5_devlink_sf_port_fn_state_get(struct devlink_port *dl_port, 143 enum devlink_port_fn_state *state, 144 enum devlink_port_fn_opstate *opstate, 145 struct netlink_ext_ack *extack) 146 { 147 struct mlx5_core_dev *dev = devlink_priv(dl_port->devlink); 148 struct mlx5_sf_table *table = dev->priv.sf_table; 149 struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); 150 151 mutex_lock(&table->sf_state_lock); 152 *state = mlx5_sf_to_devlink_state(sf->hw_state); 153 *opstate = mlx5_sf_to_devlink_opstate(sf->hw_state); 154 mutex_unlock(&table->sf_state_lock); 155 return 0; 156 } 157 158 static int mlx5_sf_activate(struct mlx5_core_dev *dev, struct mlx5_sf *sf, 159 struct netlink_ext_ack *extack) 160 { 161 struct mlx5_vport *vport; 162 int err; 163 164 if (mlx5_sf_is_active(sf)) 165 return 0; 166 if (sf->hw_state != MLX5_VHCA_STATE_ALLOCATED) { 167 NL_SET_ERR_MSG_MOD(extack, "SF is inactivated but it is still attached"); 168 return -EBUSY; 169 } 170 171 vport = mlx5_devlink_port_vport_get(&sf->dl_port.dl_port); 172 if (!vport->max_eqs_set && MLX5_CAP_GEN_2(dev, max_num_eqs_24b)) { 173 err = mlx5_devlink_port_fn_max_io_eqs_set_sf_default(&sf->dl_port.dl_port, 174 extack); 175 if (err) 176 return err; 177 } 178 err = mlx5_cmd_sf_enable_hca(dev, sf->hw_fn_id); 179 if (err) 180 return err; 181 182 sf->hw_state = MLX5_VHCA_STATE_ACTIVE; 183 trace_mlx5_sf_activate(dev, sf->port_index, sf->controller, sf->hw_fn_id); 184 return 0; 185 } 186 187 static int mlx5_sf_deactivate(struct mlx5_core_dev *dev, struct mlx5_sf *sf) 188 { 189 int err; 190 191 if (!mlx5_sf_is_active(sf)) 192 return 0; 193 194 err = mlx5_cmd_sf_disable_hca(dev, sf->hw_fn_id); 195 if (err) 196 return err; 197 198 sf->hw_state = MLX5_VHCA_STATE_TEARDOWN_REQUEST; 199 trace_mlx5_sf_deactivate(dev, sf->port_index, sf->controller, sf->hw_fn_id); 200 return 0; 201 } 202 203 static int mlx5_sf_state_set(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, 204 struct mlx5_sf *sf, 205 enum devlink_port_fn_state state, 206 struct netlink_ext_ack *extack) 207 { 208 int err = 0; 209 210 mutex_lock(&table->sf_state_lock); 211 if (state == mlx5_sf_to_devlink_state(sf->hw_state)) 212 goto out; 213 if (state == DEVLINK_PORT_FN_STATE_ACTIVE) 214 err = mlx5_sf_activate(dev, sf, extack); 215 else if (state == DEVLINK_PORT_FN_STATE_INACTIVE) 216 err = mlx5_sf_deactivate(dev, sf); 217 else 218 err = -EINVAL; 219 out: 220 mutex_unlock(&table->sf_state_lock); 221 return err; 222 } 223 224 int mlx5_devlink_sf_port_fn_state_set(struct devlink_port *dl_port, 225 enum devlink_port_fn_state state, 226 struct netlink_ext_ack *extack) 227 { 228 struct mlx5_core_dev *dev = devlink_priv(dl_port->devlink); 229 struct mlx5_sf_table *table = dev->priv.sf_table; 230 struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); 231 232 return mlx5_sf_state_set(dev, table, sf, state, extack); 233 } 234 235 static int mlx5_sf_add(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, 236 const struct devlink_port_new_attrs *new_attr, 237 struct netlink_ext_ack *extack, 238 struct devlink_port **dl_port) 239 { 240 struct mlx5_eswitch *esw = dev->priv.eswitch; 241 struct mlx5_sf *sf; 242 int err; 243 244 sf = mlx5_sf_alloc(table, esw, new_attr->controller, new_attr->sfnum, extack); 245 if (IS_ERR(sf)) 246 return PTR_ERR(sf); 247 248 mlx5_esw_reps_block(esw); 249 err = mlx5_eswitch_load_sf_vport(esw, sf->hw_fn_id, MLX5_VPORT_UC_ADDR_CHANGE, 250 &sf->dl_port, new_attr->controller, new_attr->sfnum); 251 mlx5_esw_reps_unblock(esw); 252 if (err) 253 goto esw_err; 254 *dl_port = &sf->dl_port.dl_port; 255 trace_mlx5_sf_add(dev, sf->port_index, sf->controller, sf->hw_fn_id, new_attr->sfnum); 256 return 0; 257 258 esw_err: 259 mlx5_sf_function_id_erase(table, sf); 260 mlx5_sf_free(table, sf); 261 return err; 262 } 263 264 static int 265 mlx5_sf_new_check_attr(struct mlx5_core_dev *dev, const struct devlink_port_new_attrs *new_attr, 266 struct netlink_ext_ack *extack) 267 { 268 u32 controller; 269 270 if (new_attr->flavour != DEVLINK_PORT_FLAVOUR_PCI_SF) { 271 NL_SET_ERR_MSG_MOD(extack, "Driver supports only SF port addition"); 272 return -EOPNOTSUPP; 273 } 274 if (new_attr->port_index_valid) { 275 NL_SET_ERR_MSG_MOD(extack, 276 "Driver does not support user defined port index assignment"); 277 return -EOPNOTSUPP; 278 } 279 if (!new_attr->sfnum_valid) { 280 NL_SET_ERR_MSG_MOD(extack, 281 "User must provide unique sfnum. Driver does not support auto assignment"); 282 return -EOPNOTSUPP; 283 } 284 if (new_attr->controller_valid && new_attr->controller && 285 !mlx5_core_is_ecpf_esw_manager(dev)) { 286 NL_SET_ERR_MSG_MOD(extack, "External controller is unsupported"); 287 return -EOPNOTSUPP; 288 } 289 controller = new_attr->controller_valid ? new_attr->controller : 0; 290 if (new_attr->pfnum != 291 mlx5_esw_sf_controller_to_pfnum(dev, controller)) { 292 NL_SET_ERR_MSG_MOD(extack, "Invalid pfnum supplied"); 293 return -EOPNOTSUPP; 294 } 295 return 0; 296 } 297 298 static bool mlx5_sf_table_supported(const struct mlx5_core_dev *dev) 299 { 300 return dev->priv.eswitch && MLX5_ESWITCH_MANAGER(dev) && 301 mlx5_sf_hw_table_supported(dev); 302 } 303 304 int mlx5_devlink_sf_port_new(struct devlink *devlink, 305 const struct devlink_port_new_attrs *new_attr, 306 struct netlink_ext_ack *extack, 307 struct devlink_port **dl_port) 308 { 309 struct mlx5_core_dev *dev = devlink_priv(devlink); 310 struct mlx5_sf_table *table = dev->priv.sf_table; 311 int err; 312 313 if (!mlx5_sf_table_supported(dev)) { 314 NL_SET_ERR_MSG_MOD(extack, "SF ports are not supported."); 315 return -EOPNOTSUPP; 316 } 317 318 if (!is_mdev_switchdev_mode(dev)) { 319 NL_SET_ERR_MSG_MOD(extack, 320 "SF ports are only supported in eswitch switchdev mode."); 321 return -EOPNOTSUPP; 322 } 323 324 err = mlx5_sf_new_check_attr(dev, new_attr, extack); 325 if (err) 326 return err; 327 328 return mlx5_sf_add(dev, table, new_attr, extack, dl_port); 329 } 330 331 static void mlx5_sf_dealloc(struct mlx5_sf_table *table, struct mlx5_sf *sf) 332 { 333 struct mlx5_vport *vport; 334 335 mutex_lock(&table->sf_state_lock); 336 vport = mlx5_devlink_port_vport_get(&sf->dl_port.dl_port); 337 vport->max_eqs_set = false; 338 339 mlx5_sf_function_id_erase(table, sf); 340 341 if (sf->hw_state == MLX5_VHCA_STATE_ALLOCATED) { 342 mlx5_sf_free(table, sf); 343 } else if (mlx5_sf_is_active(sf)) { 344 /* Even if its active, it is treated as in_use because by the time, 345 * it is disabled here, it may getting used. So it is safe to 346 * always look for the event to ensure that it is recycled only after 347 * firmware gives confirmation that it is detached by the driver. 348 */ 349 mlx5_cmd_sf_disable_hca(table->dev, sf->hw_fn_id); 350 mlx5_sf_hw_table_sf_deferred_free(table->dev, sf->controller, sf->id); 351 kfree(sf); 352 } else { 353 mlx5_sf_hw_table_sf_deferred_free(table->dev, sf->controller, sf->id); 354 kfree(sf); 355 } 356 357 mutex_unlock(&table->sf_state_lock); 358 } 359 360 static void mlx5_sf_del(struct mlx5_sf_table *table, struct mlx5_sf *sf) 361 { 362 struct mlx5_eswitch *esw = table->dev->priv.eswitch; 363 364 mlx5_eswitch_unload_sf_vport(esw, sf->hw_fn_id); 365 mlx5_sf_dealloc(table, sf); 366 } 367 368 int mlx5_devlink_sf_port_del(struct devlink *devlink, 369 struct devlink_port *dl_port, 370 struct netlink_ext_ack *extack) 371 { 372 struct mlx5_core_dev *dev = devlink_priv(devlink); 373 struct mlx5_sf_table *table = dev->priv.sf_table; 374 struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); 375 376 mlx5_esw_reps_block(dev->priv.eswitch); 377 mlx5_sf_del(table, sf); 378 mlx5_esw_reps_unblock(dev->priv.eswitch); 379 380 return 0; 381 } 382 383 static bool mlx5_sf_state_update_check(const struct mlx5_sf *sf, u8 new_state) 384 { 385 if (sf->hw_state == MLX5_VHCA_STATE_ACTIVE && new_state == MLX5_VHCA_STATE_IN_USE) 386 return true; 387 388 if (sf->hw_state == MLX5_VHCA_STATE_IN_USE && new_state == MLX5_VHCA_STATE_ACTIVE) 389 return true; 390 391 if (sf->hw_state == MLX5_VHCA_STATE_TEARDOWN_REQUEST && 392 new_state == MLX5_VHCA_STATE_ALLOCATED) 393 return true; 394 395 return false; 396 } 397 398 static int mlx5_sf_vhca_event(struct notifier_block *nb, unsigned long opcode, void *data) 399 { 400 struct mlx5_core_dev *dev = container_of(nb, struct mlx5_core_dev, 401 priv.sf_table_vhca_nb); 402 struct mlx5_sf_table *table = dev->priv.sf_table; 403 const struct mlx5_vhca_state_event *event = data; 404 bool update = false; 405 struct mlx5_sf *sf; 406 407 if (!table) 408 return 0; 409 410 mutex_lock(&table->sf_state_lock); 411 sf = mlx5_sf_lookup_by_function_id(table, event->function_id); 412 if (!sf) 413 goto unlock; 414 415 /* When driver is attached or detached to a function, an event 416 * notifies such state change. 417 */ 418 update = mlx5_sf_state_update_check(sf, event->new_vhca_state); 419 if (update) 420 sf->hw_state = event->new_vhca_state; 421 trace_mlx5_sf_update_state(dev, sf->port_index, sf->controller, 422 sf->hw_fn_id, sf->hw_state); 423 unlock: 424 mutex_unlock(&table->sf_state_lock); 425 return 0; 426 } 427 428 static void mlx5_sf_del_all(struct mlx5_sf_table *table) 429 { 430 unsigned long index; 431 struct mlx5_sf *sf; 432 433 xa_for_each(&table->function_ids, index, sf) 434 mlx5_sf_del(table, sf); 435 } 436 437 static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, void *data) 438 { 439 struct mlx5_core_dev *dev = container_of(nb, struct mlx5_core_dev, 440 priv.sf_table_esw_nb); 441 const struct mlx5_esw_event_info *mode = data; 442 443 if (!dev->priv.sf_table) 444 return 0; 445 446 switch (mode->new_mode) { 447 case MLX5_ESWITCH_LEGACY: 448 mlx5_sf_del_all(dev->priv.sf_table); 449 break; 450 default: 451 break; 452 } 453 454 return 0; 455 } 456 457 static int mlx5_sf_mdev_event(struct notifier_block *nb, unsigned long event, void *data) 458 { 459 struct mlx5_core_dev *dev = container_of(nb, struct mlx5_core_dev, 460 priv.sf_table_mdev_nb); 461 struct mlx5_sf_peer_devlink_event_ctx *event_ctx = data; 462 struct mlx5_sf_table *table = dev->priv.sf_table; 463 int ret = NOTIFY_DONE; 464 struct mlx5_sf *sf; 465 466 if (!table || event != MLX5_DRIVER_EVENT_SF_PEER_DEVLINK) 467 return NOTIFY_DONE; 468 469 mutex_lock(&table->sf_state_lock); 470 sf = mlx5_sf_lookup_by_function_id(table, event_ctx->fn_id); 471 if (!sf) 472 goto out; 473 474 event_ctx->err = devl_port_fn_devlink_set(&sf->dl_port.dl_port, 475 event_ctx->devlink); 476 477 ret = NOTIFY_OK; 478 out: 479 mutex_unlock(&table->sf_state_lock); 480 return ret; 481 } 482 483 int mlx5_sf_notifiers_init(struct mlx5_core_dev *dev) 484 { 485 int err; 486 487 if (mlx5_core_is_sf(dev)) 488 return 0; 489 490 dev->priv.sf_table_esw_nb.notifier_call = mlx5_sf_esw_event; 491 err = mlx5_esw_event_notifier_register(dev, &dev->priv.sf_table_esw_nb); 492 if (err) 493 return err; 494 495 dev->priv.sf_table_vhca_nb.notifier_call = mlx5_sf_vhca_event; 496 err = mlx5_vhca_event_notifier_register(dev, 497 &dev->priv.sf_table_vhca_nb); 498 if (err) 499 goto vhca_err; 500 501 dev->priv.sf_table_mdev_nb.notifier_call = mlx5_sf_mdev_event; 502 err = mlx5_blocking_notifier_register(dev, &dev->priv.sf_table_mdev_nb); 503 if (err) 504 goto mdev_err; 505 506 return 0; 507 mdev_err: 508 mlx5_vhca_event_notifier_unregister(dev, &dev->priv.sf_table_vhca_nb); 509 vhca_err: 510 mlx5_esw_event_notifier_unregister(dev, &dev->priv.sf_table_esw_nb); 511 return err; 512 } 513 514 int mlx5_sf_table_init(struct mlx5_core_dev *dev) 515 { 516 struct mlx5_sf_table *table; 517 518 if (!mlx5_sf_table_supported(dev) || !mlx5_vhca_event_supported(dev)) 519 return 0; 520 521 table = kzalloc_obj(*table); 522 if (!table) 523 return -ENOMEM; 524 525 mutex_init(&table->sf_state_lock); 526 table->dev = dev; 527 xa_init(&table->function_ids); 528 dev->priv.sf_table = table; 529 530 return 0; 531 } 532 533 void mlx5_sf_notifiers_cleanup(struct mlx5_core_dev *dev) 534 { 535 if (mlx5_core_is_sf(dev)) 536 return; 537 538 mlx5_blocking_notifier_unregister(dev, &dev->priv.sf_table_mdev_nb); 539 mlx5_vhca_event_notifier_unregister(dev, &dev->priv.sf_table_vhca_nb); 540 mlx5_esw_event_notifier_unregister(dev, &dev->priv.sf_table_esw_nb); 541 } 542 543 void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev) 544 { 545 struct mlx5_sf_table *table = dev->priv.sf_table; 546 547 if (!table) 548 return; 549 550 mutex_destroy(&table->sf_state_lock); 551 WARN_ON(!xa_empty(&table->function_ids)); 552 kfree(table); 553 } 554 555 bool mlx5_sf_table_empty(const struct mlx5_core_dev *dev) 556 { 557 struct mlx5_sf_table *table = dev->priv.sf_table; 558 559 if (!table) 560 return true; 561 562 return xa_empty(&table->function_ids); 563 } 564