1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd */
3 #include <linux/mlx5/driver.h>
4 #include "vhca_event.h"
5 #include "priv.h"
6 #include "sf.h"
7 #include "mlx5_ifc_vhca_event.h"
8 #include "ecpf.h"
9 #include "mlx5_core.h"
10 #include "eswitch.h"
11 #include "diag/sf_tracepoint.h"
12 #include "devlink.h"
13
14 struct mlx5_sf_hw {
15 u32 usr_sfnum;
16 u8 allocated: 1;
17 u8 pending_delete: 1;
18 };
19
20 struct mlx5_sf_hwc_table {
21 struct mlx5_sf_hw *sfs;
22 int max_fn;
23 u16 start_fn_id;
24 u32 controller;
25 };
26
27 enum {
28 MLX5_SF_HWC_LOCAL,
29 MLX5_SF_HWC_EXT_HOST,
30 MLX5_SF_HWC_FIRST_SPF,
31 };
32
33 struct mlx5_sf_hw_table {
34 struct mutex table_lock; /* Serializes sf deletion and vhca state change handler. */
35 struct mlx5_sf_hwc_table *hwc;
36 int num_hwc;
37 };
38
39 static struct mlx5_sf_hwc_table *
mlx5_sf_controller_to_hwc(struct mlx5_core_dev * dev,u32 controller)40 mlx5_sf_controller_to_hwc(struct mlx5_core_dev *dev, u32 controller)
41 {
42 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
43 int i;
44
45 for (i = MLX5_SF_HWC_FIRST_SPF; i < table->num_hwc; i++) {
46 if (table->hwc[i].controller == controller)
47 return &table->hwc[i];
48 }
49
50 return &table->hwc[!!controller];
51 }
52
mlx5_sf_sw_to_hw_id(struct mlx5_core_dev * dev,u32 controller,u16 sw_id)53 u16 mlx5_sf_sw_to_hw_id(struct mlx5_core_dev *dev, u32 controller, u16 sw_id)
54 {
55 struct mlx5_sf_hwc_table *hwc;
56
57 hwc = mlx5_sf_controller_to_hwc(dev, controller);
58 return hwc->start_fn_id + sw_id;
59 }
60
mlx5_sf_hw_to_sw_id(struct mlx5_sf_hwc_table * hwc,u16 hw_id)61 static u16 mlx5_sf_hw_to_sw_id(struct mlx5_sf_hwc_table *hwc, u16 hw_id)
62 {
63 return hw_id - hwc->start_fn_id;
64 }
65
66 static struct mlx5_sf_hwc_table *
mlx5_sf_table_fn_to_hwc(struct mlx5_sf_hw_table * table,u16 fn_id)67 mlx5_sf_table_fn_to_hwc(struct mlx5_sf_hw_table *table, u16 fn_id)
68 {
69 int i;
70
71 for (i = 0; i < table->num_hwc; i++) {
72 if (table->hwc[i].max_fn &&
73 fn_id >= table->hwc[i].start_fn_id &&
74 fn_id < (table->hwc[i].start_fn_id + table->hwc[i].max_fn))
75 return &table->hwc[i];
76 }
77 return NULL;
78 }
79
mlx5_sf_hw_table_id_alloc(struct mlx5_core_dev * dev,struct mlx5_sf_hw_table * table,u32 controller,u32 usr_sfnum)80 static int mlx5_sf_hw_table_id_alloc(struct mlx5_core_dev *dev,
81 struct mlx5_sf_hw_table *table,
82 u32 controller,
83 u32 usr_sfnum)
84 {
85 struct mlx5_sf_hwc_table *hwc;
86 int free_idx = -1;
87 int i;
88
89 hwc = mlx5_sf_controller_to_hwc(dev, controller);
90 if (!hwc->sfs)
91 return -ENOSPC;
92
93 for (i = 0; i < hwc->max_fn; i++) {
94 if (!hwc->sfs[i].allocated && free_idx == -1) {
95 free_idx = i;
96 continue;
97 }
98
99 if (hwc->sfs[i].allocated && hwc->sfs[i].usr_sfnum == usr_sfnum)
100 return -EEXIST;
101 }
102
103 if (free_idx == -1)
104 return -ENOSPC;
105
106 hwc->sfs[free_idx].usr_sfnum = usr_sfnum;
107 hwc->sfs[free_idx].allocated = true;
108 return free_idx;
109 }
110
mlx5_sf_hw_table_id_free(struct mlx5_core_dev * dev,struct mlx5_sf_hw_table * table,u32 controller,int id)111 static void mlx5_sf_hw_table_id_free(struct mlx5_core_dev *dev,
112 struct mlx5_sf_hw_table *table,
113 u32 controller, int id)
114 {
115 struct mlx5_sf_hwc_table *hwc;
116
117 hwc = mlx5_sf_controller_to_hwc(dev, controller);
118 hwc->sfs[id].allocated = false;
119 hwc->sfs[id].pending_delete = false;
120 }
121
mlx5_sf_hw_table_sf_alloc(struct mlx5_core_dev * dev,u32 controller,u32 usr_sfnum)122 int mlx5_sf_hw_table_sf_alloc(struct mlx5_core_dev *dev, u32 controller, u32 usr_sfnum)
123 {
124 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
125 u16 hw_fn_id;
126 int sw_id;
127 int err;
128
129 if (!table)
130 return -EOPNOTSUPP;
131
132 mutex_lock(&table->table_lock);
133 sw_id = mlx5_sf_hw_table_id_alloc(dev, table, controller, usr_sfnum);
134 if (sw_id < 0) {
135 err = sw_id;
136 goto exist_err;
137 }
138
139 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, sw_id);
140 err = mlx5_cmd_alloc_sf(dev, hw_fn_id);
141 if (err)
142 goto err;
143
144 err = mlx5_modify_vhca_sw_id(dev, hw_fn_id, usr_sfnum);
145 if (err)
146 goto vhca_err;
147
148 if (controller) {
149 /* If this SF is for external controller, SF manager
150 * needs to arm firmware to receive the events.
151 */
152 err = mlx5_vhca_event_arm(dev, hw_fn_id);
153 if (err)
154 goto vhca_err;
155 }
156
157 trace_mlx5_sf_hwc_alloc(dev, controller, hw_fn_id, usr_sfnum);
158 mutex_unlock(&table->table_lock);
159 return sw_id;
160
161 vhca_err:
162 mlx5_cmd_dealloc_sf(dev, hw_fn_id);
163 err:
164 mlx5_sf_hw_table_id_free(dev, table, controller, sw_id);
165 exist_err:
166 mutex_unlock(&table->table_lock);
167 return err;
168 }
169
mlx5_sf_hw_table_sf_free(struct mlx5_core_dev * dev,u32 controller,u16 id)170 void mlx5_sf_hw_table_sf_free(struct mlx5_core_dev *dev, u32 controller, u16 id)
171 {
172 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
173 u16 hw_fn_id;
174
175 mutex_lock(&table->table_lock);
176 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, id);
177 mlx5_cmd_dealloc_sf(dev, hw_fn_id);
178 mlx5_sf_hw_table_id_free(dev, table, controller, id);
179 mutex_unlock(&table->table_lock);
180 }
181
mlx5_sf_hw_table_hwc_sf_free(struct mlx5_core_dev * dev,struct mlx5_sf_hwc_table * hwc,int idx)182 static void mlx5_sf_hw_table_hwc_sf_free(struct mlx5_core_dev *dev,
183 struct mlx5_sf_hwc_table *hwc, int idx)
184 {
185 mlx5_cmd_dealloc_sf(dev, hwc->start_fn_id + idx);
186 hwc->sfs[idx].allocated = false;
187 hwc->sfs[idx].pending_delete = false;
188 trace_mlx5_sf_hwc_free(dev, hwc->start_fn_id + idx);
189 }
190
mlx5_sf_hw_table_sf_deferred_free(struct mlx5_core_dev * dev,u32 controller,u16 id)191 void mlx5_sf_hw_table_sf_deferred_free(struct mlx5_core_dev *dev, u32 controller, u16 id)
192 {
193 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
194 u32 out[MLX5_ST_SZ_DW(query_vhca_state_out)] = {};
195 struct mlx5_sf_hwc_table *hwc;
196 u16 hw_fn_id;
197 u8 state;
198 int err;
199
200 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, id);
201 hwc = mlx5_sf_controller_to_hwc(dev, controller);
202 mutex_lock(&table->table_lock);
203 err = mlx5_cmd_query_vhca_state(dev, hw_fn_id, out, sizeof(out));
204 if (err)
205 goto err;
206 state = MLX5_GET(query_vhca_state_out, out, vhca_state_context.vhca_state);
207 if (state == MLX5_VHCA_STATE_ALLOCATED) {
208 mlx5_cmd_dealloc_sf(dev, hw_fn_id);
209 hwc->sfs[id].allocated = false;
210 } else {
211 hwc->sfs[id].pending_delete = true;
212 trace_mlx5_sf_hwc_deferred_free(dev, hw_fn_id);
213 }
214 err:
215 mutex_unlock(&table->table_lock);
216 }
217
mlx5_sf_hw_table_hwc_dealloc_all(struct mlx5_core_dev * dev,struct mlx5_sf_hwc_table * hwc)218 static void mlx5_sf_hw_table_hwc_dealloc_all(struct mlx5_core_dev *dev,
219 struct mlx5_sf_hwc_table *hwc)
220 {
221 int i;
222
223 for (i = 0; i < hwc->max_fn; i++) {
224 if (hwc->sfs[i].allocated)
225 mlx5_sf_hw_table_hwc_sf_free(dev, hwc, i);
226 }
227 }
228
mlx5_sf_hw_table_dealloc_all(struct mlx5_core_dev * dev,struct mlx5_sf_hw_table * table)229 static void mlx5_sf_hw_table_dealloc_all(struct mlx5_core_dev *dev,
230 struct mlx5_sf_hw_table *table)
231 {
232 int i;
233
234 for (i = 0; i < table->num_hwc; i++)
235 mlx5_sf_hw_table_hwc_dealloc_all(dev, &table->hwc[i]);
236 }
237
mlx5_sf_hw_table_hwc_init(struct mlx5_sf_hwc_table * hwc,u16 max_fn,u16 base_id)238 static int mlx5_sf_hw_table_hwc_init(struct mlx5_sf_hwc_table *hwc, u16 max_fn, u16 base_id)
239 {
240 struct mlx5_sf_hw *sfs;
241
242 if (!max_fn)
243 return 0;
244
245 sfs = kzalloc_objs(*sfs, max_fn);
246 if (!sfs)
247 return -ENOMEM;
248
249 hwc->sfs = sfs;
250 hwc->max_fn = max_fn;
251 hwc->start_fn_id = base_id;
252 return 0;
253 }
254
mlx5_sf_hw_table_hwc_cleanup(struct mlx5_sf_hwc_table * hwc)255 static void mlx5_sf_hw_table_hwc_cleanup(struct mlx5_sf_hwc_table *hwc)
256 {
257 kfree(hwc->sfs);
258 }
259
mlx5_sf_hw_table_res_unregister(struct mlx5_core_dev * dev)260 static void mlx5_sf_hw_table_res_unregister(struct mlx5_core_dev *dev)
261 {
262 devl_resources_unregister(priv_to_devlink(dev));
263 }
264
mlx5_sf_hw_table_res_register(struct mlx5_core_dev * dev,u16 max_fn,u16 max_ext_fn)265 static int mlx5_sf_hw_table_res_register(struct mlx5_core_dev *dev, u16 max_fn,
266 u16 max_ext_fn)
267 {
268 struct devlink_resource_size_params size_params;
269 struct devlink *devlink = priv_to_devlink(dev);
270 int err;
271
272 devlink_resource_size_params_init(&size_params, max_fn, max_fn, 1,
273 DEVLINK_RESOURCE_UNIT_ENTRY);
274 err = devl_resource_register(devlink, "max_local_SFs", max_fn, MLX5_DL_RES_MAX_LOCAL_SFS,
275 DEVLINK_RESOURCE_ID_PARENT_TOP, &size_params);
276 if (err)
277 return err;
278
279 devlink_resource_size_params_init(&size_params, max_ext_fn, max_ext_fn, 1,
280 DEVLINK_RESOURCE_UNIT_ENTRY);
281 return devl_resource_register(devlink, "max_external_SFs", max_ext_fn,
282 MLX5_DL_RES_MAX_EXTERNAL_SFS, DEVLINK_RESOURCE_ID_PARENT_TOP,
283 &size_params);
284 }
285
mlx5_sf_hw_table_init(struct mlx5_core_dev * dev)286 int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev)
287 {
288 struct mlx5_sf_hw_table *table;
289 int num_spfs, num_hwc;
290 u16 max_ext_fn = 0;
291 u16 ext_base_id = 0;
292 u16 base_id;
293 u16 max_fn;
294 int err;
295 int i;
296
297 if (!mlx5_vhca_event_supported(dev))
298 return 0;
299
300 max_fn = mlx5_sf_max_functions(dev);
301
302 err = mlx5_esw_sf_max_hpf_functions(dev, &max_ext_fn, &ext_base_id);
303 if (err)
304 return err;
305
306 if (mlx5_sf_hw_table_res_register(dev, max_fn, max_ext_fn))
307 mlx5_core_dbg(dev, "failed to register max SFs resources");
308
309 if (!max_fn && !max_ext_fn && !mlx5_esw_has_spf_sfs(dev))
310 return 0;
311
312 table = kzalloc_obj(*table);
313 if (!table) {
314 err = -ENOMEM;
315 goto alloc_err;
316 }
317
318 num_spfs = mlx5_esw_get_num_spfs(dev);
319 num_hwc = MLX5_SF_HWC_FIRST_SPF + num_spfs;
320 table->hwc = kzalloc_objs(*table->hwc, num_hwc);
321 if (!table->hwc) {
322 err = -ENOMEM;
323 goto hwc_alloc_err;
324 }
325 table->num_hwc = num_hwc;
326
327 mutex_init(&table->table_lock);
328 dev->priv.sf_hw_table = table;
329
330 table->hwc[MLX5_SF_HWC_LOCAL].controller = 0;
331 base_id = mlx5_sf_start_function_id(dev);
332 err = mlx5_sf_hw_table_hwc_init(&table->hwc[MLX5_SF_HWC_LOCAL], max_fn, base_id);
333 if (err)
334 goto hwc_init_err;
335
336 table->hwc[MLX5_SF_HWC_EXT_HOST].controller =
337 mlx5_esw_get_hpf_host_number(dev) + 1;
338 err = mlx5_sf_hw_table_hwc_init(&table->hwc[MLX5_SF_HWC_EXT_HOST],
339 max_ext_fn, ext_base_id);
340 if (err)
341 goto hwc_init_err;
342
343 for (i = 0; i < num_spfs; i++) {
344 u16 spf_max_sfs, spf_base_id, host_number;
345 int hwc_idx = MLX5_SF_HWC_FIRST_SPF + i;
346
347 err = mlx5_esw_spf_get_host_number(dev, i, &host_number);
348 if (err)
349 goto hwc_init_err;
350
351 err = mlx5_esw_sf_max_spf_functions(dev, i, &spf_max_sfs,
352 &spf_base_id);
353 if (err)
354 goto hwc_init_err;
355
356 table->hwc[hwc_idx].controller = host_number + 1;
357 err = mlx5_sf_hw_table_hwc_init(&table->hwc[hwc_idx],
358 spf_max_sfs, spf_base_id);
359 if (err)
360 goto hwc_init_err;
361 }
362
363 mlx5_core_dbg(dev, "SF HW table: max sfs = %d, ext sfs = %d, num spfs = %d\n",
364 max_fn, max_ext_fn, num_spfs);
365 return 0;
366
367 hwc_init_err:
368 dev->priv.sf_hw_table = NULL;
369 for (i = 0; i < num_hwc; i++)
370 mlx5_sf_hw_table_hwc_cleanup(&table->hwc[i]);
371 mutex_destroy(&table->table_lock);
372 kfree(table->hwc);
373 hwc_alloc_err:
374 kfree(table);
375 alloc_err:
376 mlx5_sf_hw_table_res_unregister(dev);
377 return err;
378 }
379
mlx5_sf_hw_table_cleanup(struct mlx5_core_dev * dev)380 void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev)
381 {
382 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
383 int i;
384
385 if (!table)
386 goto res_unregister;
387
388 for (i = 0; i < table->num_hwc; i++)
389 mlx5_sf_hw_table_hwc_cleanup(&table->hwc[i]);
390 mutex_destroy(&table->table_lock);
391 kfree(table->hwc);
392 kfree(table);
393 dev->priv.sf_hw_table = NULL;
394 res_unregister:
395 mlx5_sf_hw_table_res_unregister(dev);
396 }
397
mlx5_sf_hw_vhca_event(struct notifier_block * nb,unsigned long opcode,void * data)398 static int mlx5_sf_hw_vhca_event(struct notifier_block *nb, unsigned long opcode, void *data)
399 {
400 struct mlx5_core_dev *dev = container_of(nb, struct mlx5_core_dev,
401 priv.sf_hw_table_vhca_nb);
402 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
403 const struct mlx5_vhca_state_event *event = data;
404 struct mlx5_sf_hwc_table *hwc;
405 struct mlx5_sf_hw *sf_hw;
406 u16 sw_id;
407
408 if (!table || event->new_vhca_state != MLX5_VHCA_STATE_ALLOCATED)
409 return 0;
410
411 hwc = mlx5_sf_table_fn_to_hwc(table, event->function_id);
412 if (!hwc)
413 return 0;
414
415 sw_id = mlx5_sf_hw_to_sw_id(hwc, event->function_id);
416 sf_hw = &hwc->sfs[sw_id];
417
418 mutex_lock(&table->table_lock);
419 /* SF driver notified through firmware that SF is finally detached.
420 * Hence recycle the sf hardware id for reuse.
421 */
422 if (sf_hw->allocated && sf_hw->pending_delete)
423 mlx5_sf_hw_table_hwc_sf_free(dev, hwc, sw_id);
424 mutex_unlock(&table->table_lock);
425 return 0;
426 }
427
mlx5_sf_hw_notifier_init(struct mlx5_core_dev * dev)428 int mlx5_sf_hw_notifier_init(struct mlx5_core_dev *dev)
429 {
430 if (mlx5_core_is_sf(dev))
431 return 0;
432
433 dev->priv.sf_hw_table_vhca_nb.notifier_call = mlx5_sf_hw_vhca_event;
434 return mlx5_vhca_event_notifier_register(dev,
435 &dev->priv.sf_hw_table_vhca_nb);
436 }
437
mlx5_sf_hw_notifier_cleanup(struct mlx5_core_dev * dev)438 void mlx5_sf_hw_notifier_cleanup(struct mlx5_core_dev *dev)
439 {
440 if (mlx5_core_is_sf(dev))
441 return;
442
443 mlx5_vhca_event_notifier_unregister(dev,
444 &dev->priv.sf_hw_table_vhca_nb);
445 }
446
mlx5_sf_hw_table_destroy(struct mlx5_core_dev * dev)447 void mlx5_sf_hw_table_destroy(struct mlx5_core_dev *dev)
448 {
449 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
450
451 if (!table)
452 return;
453
454 /* Dealloc SFs whose firmware event has been missed. */
455 mlx5_sf_hw_table_dealloc_all(dev, table);
456 }
457
mlx5_sf_hw_table_supported(const struct mlx5_core_dev * dev)458 bool mlx5_sf_hw_table_supported(const struct mlx5_core_dev *dev)
459 {
460 return !!dev->priv.sf_hw_table;
461 }
462
mlx5_sf_hw_table_esw_changed_event_handler(struct mlx5_core_dev * dev)463 void mlx5_sf_hw_table_esw_changed_event_handler(struct mlx5_core_dev *dev)
464 {
465 struct mlx5_sf_hw_table *table;
466 struct mlx5_sf_hwc_table *hwc;
467 int i;
468
469 table = dev->priv.sf_hw_table;
470 if (!table)
471 return;
472
473 mutex_lock(&table->table_lock);
474 hwc = &table->hwc[MLX5_SF_HWC_EXT_HOST];
475 for (i = 0; i < hwc->max_fn; i++) {
476 struct mlx5_sf_hw *sf_hw;
477
478 sf_hw = &hwc->sfs[i];
479 if (sf_hw->allocated && sf_hw->pending_delete)
480 mlx5_sf_hw_table_hwc_sf_free(dev, hwc, i);
481 }
482 mutex_unlock(&table->table_lock);
483 }
484