1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020 Mellanox Technologies Ltd */ 3 #include <linux/mlx5/driver.h> 4 #include "vhca_event.h" 5 #include "priv.h" 6 #include "sf.h" 7 #include "mlx5_ifc_vhca_event.h" 8 #include "ecpf.h" 9 #include "mlx5_core.h" 10 #include "eswitch.h" 11 #include "diag/sf_tracepoint.h" 12 #include "devlink.h" 13 14 struct mlx5_sf_hw { 15 u32 usr_sfnum; 16 u8 allocated: 1; 17 u8 pending_delete: 1; 18 }; 19 20 struct mlx5_sf_hwc_table { 21 struct mlx5_sf_hw *sfs; 22 int max_fn; 23 u16 start_fn_id; 24 u32 controller; 25 }; 26 27 enum { 28 MLX5_SF_HWC_LOCAL, 29 MLX5_SF_HWC_EXT_HOST, 30 MLX5_SF_HWC_FIRST_SPF, 31 }; 32 33 struct mlx5_sf_hw_table { 34 struct mutex table_lock; /* Serializes sf deletion and vhca state change handler. */ 35 struct mlx5_sf_hwc_table *hwc; 36 int num_hwc; 37 }; 38 39 static struct mlx5_sf_hwc_table * 40 mlx5_sf_controller_to_hwc(struct mlx5_core_dev *dev, u32 controller) 41 { 42 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; 43 int i; 44 45 for (i = MLX5_SF_HWC_FIRST_SPF; i < table->num_hwc; i++) { 46 if (table->hwc[i].controller == controller) 47 return &table->hwc[i]; 48 } 49 50 return &table->hwc[!!controller]; 51 } 52 53 u16 mlx5_sf_sw_to_hw_id(struct mlx5_core_dev *dev, u32 controller, u16 sw_id) 54 { 55 struct mlx5_sf_hwc_table *hwc; 56 57 hwc = mlx5_sf_controller_to_hwc(dev, controller); 58 return hwc->start_fn_id + sw_id; 59 } 60 61 static u16 mlx5_sf_hw_to_sw_id(struct mlx5_sf_hwc_table *hwc, u16 hw_id) 62 { 63 return hw_id - hwc->start_fn_id; 64 } 65 66 static struct mlx5_sf_hwc_table * 67 mlx5_sf_table_fn_to_hwc(struct mlx5_sf_hw_table *table, u16 fn_id) 68 { 69 int i; 70 71 for (i = 0; i < table->num_hwc; i++) { 72 if (table->hwc[i].max_fn && 73 fn_id >= table->hwc[i].start_fn_id && 74 fn_id < (table->hwc[i].start_fn_id + table->hwc[i].max_fn)) 75 return &table->hwc[i]; 76 } 77 return NULL; 78 } 79 80 static int mlx5_sf_hw_table_id_alloc(struct mlx5_core_dev *dev, 81 struct mlx5_sf_hw_table *table, 82 u32 controller, 83 u32 usr_sfnum) 84 { 85 struct mlx5_sf_hwc_table *hwc; 86 int free_idx = -1; 87 int i; 88 89 hwc = mlx5_sf_controller_to_hwc(dev, controller); 90 if (!hwc->sfs) 91 return -ENOSPC; 92 93 for (i = 0; i < hwc->max_fn; i++) { 94 if (!hwc->sfs[i].allocated && free_idx == -1) { 95 free_idx = i; 96 continue; 97 } 98 99 if (hwc->sfs[i].allocated && hwc->sfs[i].usr_sfnum == usr_sfnum) 100 return -EEXIST; 101 } 102 103 if (free_idx == -1) 104 return -ENOSPC; 105 106 hwc->sfs[free_idx].usr_sfnum = usr_sfnum; 107 hwc->sfs[free_idx].allocated = true; 108 return free_idx; 109 } 110 111 static void mlx5_sf_hw_table_id_free(struct mlx5_core_dev *dev, 112 struct mlx5_sf_hw_table *table, 113 u32 controller, int id) 114 { 115 struct mlx5_sf_hwc_table *hwc; 116 117 hwc = mlx5_sf_controller_to_hwc(dev, controller); 118 hwc->sfs[id].allocated = false; 119 hwc->sfs[id].pending_delete = false; 120 } 121 122 int mlx5_sf_hw_table_sf_alloc(struct mlx5_core_dev *dev, u32 controller, u32 usr_sfnum) 123 { 124 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; 125 u16 hw_fn_id; 126 int sw_id; 127 int err; 128 129 if (!table) 130 return -EOPNOTSUPP; 131 132 mutex_lock(&table->table_lock); 133 sw_id = mlx5_sf_hw_table_id_alloc(dev, table, controller, usr_sfnum); 134 if (sw_id < 0) { 135 err = sw_id; 136 goto exist_err; 137 } 138 139 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, sw_id); 140 err = mlx5_cmd_alloc_sf(dev, hw_fn_id); 141 if (err) 142 goto err; 143 144 err = mlx5_modify_vhca_sw_id(dev, hw_fn_id, usr_sfnum); 145 if (err) 146 goto vhca_err; 147 148 if (controller) { 149 /* If this SF is for external controller, SF manager 150 * needs to arm firmware to receive the events. 151 */ 152 err = mlx5_vhca_event_arm(dev, hw_fn_id); 153 if (err) 154 goto vhca_err; 155 } 156 157 trace_mlx5_sf_hwc_alloc(dev, controller, hw_fn_id, usr_sfnum); 158 mutex_unlock(&table->table_lock); 159 return sw_id; 160 161 vhca_err: 162 mlx5_cmd_dealloc_sf(dev, hw_fn_id); 163 err: 164 mlx5_sf_hw_table_id_free(dev, table, controller, sw_id); 165 exist_err: 166 mutex_unlock(&table->table_lock); 167 return err; 168 } 169 170 void mlx5_sf_hw_table_sf_free(struct mlx5_core_dev *dev, u32 controller, u16 id) 171 { 172 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; 173 u16 hw_fn_id; 174 175 mutex_lock(&table->table_lock); 176 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, id); 177 mlx5_cmd_dealloc_sf(dev, hw_fn_id); 178 mlx5_sf_hw_table_id_free(dev, table, controller, id); 179 mutex_unlock(&table->table_lock); 180 } 181 182 static void mlx5_sf_hw_table_hwc_sf_free(struct mlx5_core_dev *dev, 183 struct mlx5_sf_hwc_table *hwc, int idx) 184 { 185 mlx5_cmd_dealloc_sf(dev, hwc->start_fn_id + idx); 186 hwc->sfs[idx].allocated = false; 187 hwc->sfs[idx].pending_delete = false; 188 trace_mlx5_sf_hwc_free(dev, hwc->start_fn_id + idx); 189 } 190 191 void mlx5_sf_hw_table_sf_deferred_free(struct mlx5_core_dev *dev, u32 controller, u16 id) 192 { 193 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; 194 u32 out[MLX5_ST_SZ_DW(query_vhca_state_out)] = {}; 195 struct mlx5_sf_hwc_table *hwc; 196 u16 hw_fn_id; 197 u8 state; 198 int err; 199 200 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, id); 201 hwc = mlx5_sf_controller_to_hwc(dev, controller); 202 mutex_lock(&table->table_lock); 203 err = mlx5_cmd_query_vhca_state(dev, hw_fn_id, out, sizeof(out)); 204 if (err) 205 goto err; 206 state = MLX5_GET(query_vhca_state_out, out, vhca_state_context.vhca_state); 207 if (state == MLX5_VHCA_STATE_ALLOCATED) { 208 mlx5_cmd_dealloc_sf(dev, hw_fn_id); 209 hwc->sfs[id].allocated = false; 210 } else { 211 hwc->sfs[id].pending_delete = true; 212 trace_mlx5_sf_hwc_deferred_free(dev, hw_fn_id); 213 } 214 err: 215 mutex_unlock(&table->table_lock); 216 } 217 218 static void mlx5_sf_hw_table_hwc_dealloc_all(struct mlx5_core_dev *dev, 219 struct mlx5_sf_hwc_table *hwc) 220 { 221 int i; 222 223 for (i = 0; i < hwc->max_fn; i++) { 224 if (hwc->sfs[i].allocated) 225 mlx5_sf_hw_table_hwc_sf_free(dev, hwc, i); 226 } 227 } 228 229 static void mlx5_sf_hw_table_dealloc_all(struct mlx5_core_dev *dev, 230 struct mlx5_sf_hw_table *table) 231 { 232 int i; 233 234 for (i = 0; i < table->num_hwc; i++) 235 mlx5_sf_hw_table_hwc_dealloc_all(dev, &table->hwc[i]); 236 } 237 238 static int mlx5_sf_hw_table_hwc_init(struct mlx5_sf_hwc_table *hwc, u16 max_fn, u16 base_id) 239 { 240 struct mlx5_sf_hw *sfs; 241 242 if (!max_fn) 243 return 0; 244 245 sfs = kzalloc_objs(*sfs, max_fn); 246 if (!sfs) 247 return -ENOMEM; 248 249 hwc->sfs = sfs; 250 hwc->max_fn = max_fn; 251 hwc->start_fn_id = base_id; 252 return 0; 253 } 254 255 static void mlx5_sf_hw_table_hwc_cleanup(struct mlx5_sf_hwc_table *hwc) 256 { 257 kfree(hwc->sfs); 258 } 259 260 static void mlx5_sf_hw_table_res_unregister(struct mlx5_core_dev *dev) 261 { 262 devl_resources_unregister(priv_to_devlink(dev)); 263 } 264 265 static int mlx5_sf_hw_table_res_register(struct mlx5_core_dev *dev, u16 max_fn, 266 u16 max_ext_fn) 267 { 268 struct devlink_resource_size_params size_params; 269 struct devlink *devlink = priv_to_devlink(dev); 270 int err; 271 272 devlink_resource_size_params_init(&size_params, max_fn, max_fn, 1, 273 DEVLINK_RESOURCE_UNIT_ENTRY); 274 err = devl_resource_register(devlink, "max_local_SFs", max_fn, MLX5_DL_RES_MAX_LOCAL_SFS, 275 DEVLINK_RESOURCE_ID_PARENT_TOP, &size_params); 276 if (err) 277 return err; 278 279 devlink_resource_size_params_init(&size_params, max_ext_fn, max_ext_fn, 1, 280 DEVLINK_RESOURCE_UNIT_ENTRY); 281 return devl_resource_register(devlink, "max_external_SFs", max_ext_fn, 282 MLX5_DL_RES_MAX_EXTERNAL_SFS, DEVLINK_RESOURCE_ID_PARENT_TOP, 283 &size_params); 284 } 285 286 int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev) 287 { 288 struct mlx5_sf_hw_table *table; 289 int num_spfs, num_hwc; 290 u16 max_ext_fn = 0; 291 u16 ext_base_id = 0; 292 u16 base_id; 293 u16 max_fn; 294 int err; 295 int i; 296 297 if (!mlx5_vhca_event_supported(dev)) 298 return 0; 299 300 max_fn = mlx5_sf_max_functions(dev); 301 302 err = mlx5_esw_sf_max_hpf_functions(dev, &max_ext_fn, &ext_base_id); 303 if (err) 304 return err; 305 306 if (mlx5_sf_hw_table_res_register(dev, max_fn, max_ext_fn)) 307 mlx5_core_dbg(dev, "failed to register max SFs resources"); 308 309 if (!max_fn && !max_ext_fn && !mlx5_esw_has_spf_sfs(dev)) 310 return 0; 311 312 table = kzalloc_obj(*table); 313 if (!table) { 314 err = -ENOMEM; 315 goto alloc_err; 316 } 317 318 num_spfs = mlx5_esw_get_num_spfs(dev); 319 num_hwc = MLX5_SF_HWC_FIRST_SPF + num_spfs; 320 table->hwc = kcalloc(num_hwc, sizeof(*table->hwc), GFP_KERNEL); 321 if (!table->hwc) { 322 err = -ENOMEM; 323 goto hwc_alloc_err; 324 } 325 table->num_hwc = num_hwc; 326 327 mutex_init(&table->table_lock); 328 dev->priv.sf_hw_table = table; 329 330 table->hwc[MLX5_SF_HWC_LOCAL].controller = 0; 331 base_id = mlx5_sf_start_function_id(dev); 332 err = mlx5_sf_hw_table_hwc_init(&table->hwc[MLX5_SF_HWC_LOCAL], max_fn, base_id); 333 if (err) 334 goto hwc_init_err; 335 336 table->hwc[MLX5_SF_HWC_EXT_HOST].controller = 337 mlx5_esw_get_hpf_host_number(dev) + 1; 338 err = mlx5_sf_hw_table_hwc_init(&table->hwc[MLX5_SF_HWC_EXT_HOST], 339 max_ext_fn, ext_base_id); 340 if (err) 341 goto hwc_init_err; 342 343 for (i = 0; i < num_spfs; i++) { 344 u16 spf_max_sfs, spf_base_id, host_number; 345 int hwc_idx = MLX5_SF_HWC_FIRST_SPF + i; 346 347 err = mlx5_esw_spf_get_host_number(dev, i, &host_number); 348 if (err) 349 goto hwc_init_err; 350 351 err = mlx5_esw_sf_max_spf_functions(dev, i, &spf_max_sfs, 352 &spf_base_id); 353 if (err) 354 goto hwc_init_err; 355 356 table->hwc[hwc_idx].controller = host_number + 1; 357 err = mlx5_sf_hw_table_hwc_init(&table->hwc[hwc_idx], 358 spf_max_sfs, spf_base_id); 359 if (err) 360 goto hwc_init_err; 361 } 362 363 mlx5_core_dbg(dev, "SF HW table: max sfs = %d, ext sfs = %d, num spfs = %d\n", 364 max_fn, max_ext_fn, num_spfs); 365 return 0; 366 367 hwc_init_err: 368 dev->priv.sf_hw_table = NULL; 369 for (i = 0; i < num_hwc; i++) 370 mlx5_sf_hw_table_hwc_cleanup(&table->hwc[i]); 371 mutex_destroy(&table->table_lock); 372 kfree(table->hwc); 373 hwc_alloc_err: 374 kfree(table); 375 alloc_err: 376 mlx5_sf_hw_table_res_unregister(dev); 377 return err; 378 } 379 380 void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev) 381 { 382 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; 383 int i; 384 385 if (!table) 386 goto res_unregister; 387 388 for (i = 0; i < table->num_hwc; i++) 389 mlx5_sf_hw_table_hwc_cleanup(&table->hwc[i]); 390 mutex_destroy(&table->table_lock); 391 kfree(table->hwc); 392 kfree(table); 393 dev->priv.sf_hw_table = NULL; 394 res_unregister: 395 mlx5_sf_hw_table_res_unregister(dev); 396 } 397 398 static int mlx5_sf_hw_vhca_event(struct notifier_block *nb, unsigned long opcode, void *data) 399 { 400 struct mlx5_core_dev *dev = container_of(nb, struct mlx5_core_dev, 401 priv.sf_hw_table_vhca_nb); 402 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; 403 const struct mlx5_vhca_state_event *event = data; 404 struct mlx5_sf_hwc_table *hwc; 405 struct mlx5_sf_hw *sf_hw; 406 u16 sw_id; 407 408 if (!table || event->new_vhca_state != MLX5_VHCA_STATE_ALLOCATED) 409 return 0; 410 411 hwc = mlx5_sf_table_fn_to_hwc(table, event->function_id); 412 if (!hwc) 413 return 0; 414 415 sw_id = mlx5_sf_hw_to_sw_id(hwc, event->function_id); 416 sf_hw = &hwc->sfs[sw_id]; 417 418 mutex_lock(&table->table_lock); 419 /* SF driver notified through firmware that SF is finally detached. 420 * Hence recycle the sf hardware id for reuse. 421 */ 422 if (sf_hw->allocated && sf_hw->pending_delete) 423 mlx5_sf_hw_table_hwc_sf_free(dev, hwc, sw_id); 424 mutex_unlock(&table->table_lock); 425 return 0; 426 } 427 428 int mlx5_sf_hw_notifier_init(struct mlx5_core_dev *dev) 429 { 430 if (mlx5_core_is_sf(dev)) 431 return 0; 432 433 dev->priv.sf_hw_table_vhca_nb.notifier_call = mlx5_sf_hw_vhca_event; 434 return mlx5_vhca_event_notifier_register(dev, 435 &dev->priv.sf_hw_table_vhca_nb); 436 } 437 438 void mlx5_sf_hw_notifier_cleanup(struct mlx5_core_dev *dev) 439 { 440 if (mlx5_core_is_sf(dev)) 441 return; 442 443 mlx5_vhca_event_notifier_unregister(dev, 444 &dev->priv.sf_hw_table_vhca_nb); 445 } 446 447 void mlx5_sf_hw_table_destroy(struct mlx5_core_dev *dev) 448 { 449 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; 450 451 if (!table) 452 return; 453 454 /* Dealloc SFs whose firmware event has been missed. */ 455 mlx5_sf_hw_table_dealloc_all(dev, table); 456 } 457 458 bool mlx5_sf_hw_table_supported(const struct mlx5_core_dev *dev) 459 { 460 return !!dev->priv.sf_hw_table; 461 } 462