1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd */
3 #include <linux/mlx5/driver.h>
4 #include "vhca_event.h"
5 #include "priv.h"
6 #include "sf.h"
7 #include "mlx5_ifc_vhca_event.h"
8 #include "ecpf.h"
9 #include "mlx5_core.h"
10 #include "eswitch.h"
11 #include "diag/sf_tracepoint.h"
12 #include "devlink.h"
13
14 struct mlx5_sf_hw {
15 u32 usr_sfnum;
16 u8 allocated: 1;
17 u8 pending_delete: 1;
18 };
19
20 struct mlx5_sf_hwc_table {
21 struct mlx5_sf_hw *sfs;
22 int max_fn;
23 u16 start_fn_id;
24 };
25
26 enum mlx5_sf_hwc_index {
27 MLX5_SF_HWC_LOCAL,
28 MLX5_SF_HWC_EXTERNAL,
29 MLX5_SF_HWC_MAX,
30 };
31
32 struct mlx5_sf_hw_table {
33 struct mutex table_lock; /* Serializes sf deletion and vhca state change handler. */
34 struct mlx5_sf_hwc_table hwc[MLX5_SF_HWC_MAX];
35 };
36
37 static struct mlx5_sf_hwc_table *
mlx5_sf_controller_to_hwc(struct mlx5_core_dev * dev,u32 controller)38 mlx5_sf_controller_to_hwc(struct mlx5_core_dev *dev, u32 controller)
39 {
40 int idx = !!controller;
41
42 return &dev->priv.sf_hw_table->hwc[idx];
43 }
44
mlx5_sf_sw_to_hw_id(struct mlx5_core_dev * dev,u32 controller,u16 sw_id)45 u16 mlx5_sf_sw_to_hw_id(struct mlx5_core_dev *dev, u32 controller, u16 sw_id)
46 {
47 struct mlx5_sf_hwc_table *hwc;
48
49 hwc = mlx5_sf_controller_to_hwc(dev, controller);
50 return hwc->start_fn_id + sw_id;
51 }
52
mlx5_sf_hw_to_sw_id(struct mlx5_sf_hwc_table * hwc,u16 hw_id)53 static u16 mlx5_sf_hw_to_sw_id(struct mlx5_sf_hwc_table *hwc, u16 hw_id)
54 {
55 return hw_id - hwc->start_fn_id;
56 }
57
58 static struct mlx5_sf_hwc_table *
mlx5_sf_table_fn_to_hwc(struct mlx5_sf_hw_table * table,u16 fn_id)59 mlx5_sf_table_fn_to_hwc(struct mlx5_sf_hw_table *table, u16 fn_id)
60 {
61 int i;
62
63 for (i = 0; i < ARRAY_SIZE(table->hwc); i++) {
64 if (table->hwc[i].max_fn &&
65 fn_id >= table->hwc[i].start_fn_id &&
66 fn_id < (table->hwc[i].start_fn_id + table->hwc[i].max_fn))
67 return &table->hwc[i];
68 }
69 return NULL;
70 }
71
mlx5_sf_hw_table_id_alloc(struct mlx5_core_dev * dev,struct mlx5_sf_hw_table * table,u32 controller,u32 usr_sfnum)72 static int mlx5_sf_hw_table_id_alloc(struct mlx5_core_dev *dev,
73 struct mlx5_sf_hw_table *table,
74 u32 controller,
75 u32 usr_sfnum)
76 {
77 struct mlx5_sf_hwc_table *hwc;
78 int free_idx = -1;
79 int i;
80
81 hwc = mlx5_sf_controller_to_hwc(dev, controller);
82 if (!hwc->sfs)
83 return -ENOSPC;
84
85 for (i = 0; i < hwc->max_fn; i++) {
86 if (!hwc->sfs[i].allocated && free_idx == -1) {
87 free_idx = i;
88 continue;
89 }
90
91 if (hwc->sfs[i].allocated && hwc->sfs[i].usr_sfnum == usr_sfnum)
92 return -EEXIST;
93 }
94
95 if (free_idx == -1)
96 return -ENOSPC;
97
98 hwc->sfs[free_idx].usr_sfnum = usr_sfnum;
99 hwc->sfs[free_idx].allocated = true;
100 return free_idx;
101 }
102
mlx5_sf_hw_table_id_free(struct mlx5_core_dev * dev,struct mlx5_sf_hw_table * table,u32 controller,int id)103 static void mlx5_sf_hw_table_id_free(struct mlx5_core_dev *dev,
104 struct mlx5_sf_hw_table *table,
105 u32 controller, int id)
106 {
107 struct mlx5_sf_hwc_table *hwc;
108
109 hwc = mlx5_sf_controller_to_hwc(dev, controller);
110 hwc->sfs[id].allocated = false;
111 hwc->sfs[id].pending_delete = false;
112 }
113
mlx5_sf_hw_table_sf_alloc(struct mlx5_core_dev * dev,u32 controller,u32 usr_sfnum)114 int mlx5_sf_hw_table_sf_alloc(struct mlx5_core_dev *dev, u32 controller, u32 usr_sfnum)
115 {
116 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
117 u16 hw_fn_id;
118 int sw_id;
119 int err;
120
121 if (!table)
122 return -EOPNOTSUPP;
123
124 mutex_lock(&table->table_lock);
125 sw_id = mlx5_sf_hw_table_id_alloc(dev, table, controller, usr_sfnum);
126 if (sw_id < 0) {
127 err = sw_id;
128 goto exist_err;
129 }
130
131 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, sw_id);
132 err = mlx5_cmd_alloc_sf(dev, hw_fn_id);
133 if (err)
134 goto err;
135
136 err = mlx5_modify_vhca_sw_id(dev, hw_fn_id, usr_sfnum);
137 if (err)
138 goto vhca_err;
139
140 if (controller) {
141 /* If this SF is for external controller, SF manager
142 * needs to arm firmware to receive the events.
143 */
144 err = mlx5_vhca_event_arm(dev, hw_fn_id);
145 if (err)
146 goto vhca_err;
147 }
148
149 trace_mlx5_sf_hwc_alloc(dev, controller, hw_fn_id, usr_sfnum);
150 mutex_unlock(&table->table_lock);
151 return sw_id;
152
153 vhca_err:
154 mlx5_cmd_dealloc_sf(dev, hw_fn_id);
155 err:
156 mlx5_sf_hw_table_id_free(dev, table, controller, sw_id);
157 exist_err:
158 mutex_unlock(&table->table_lock);
159 return err;
160 }
161
mlx5_sf_hw_table_sf_free(struct mlx5_core_dev * dev,u32 controller,u16 id)162 void mlx5_sf_hw_table_sf_free(struct mlx5_core_dev *dev, u32 controller, u16 id)
163 {
164 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
165 u16 hw_fn_id;
166
167 mutex_lock(&table->table_lock);
168 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, id);
169 mlx5_cmd_dealloc_sf(dev, hw_fn_id);
170 mlx5_sf_hw_table_id_free(dev, table, controller, id);
171 mutex_unlock(&table->table_lock);
172 }
173
mlx5_sf_hw_table_hwc_sf_free(struct mlx5_core_dev * dev,struct mlx5_sf_hwc_table * hwc,int idx)174 static void mlx5_sf_hw_table_hwc_sf_free(struct mlx5_core_dev *dev,
175 struct mlx5_sf_hwc_table *hwc, int idx)
176 {
177 mlx5_cmd_dealloc_sf(dev, hwc->start_fn_id + idx);
178 hwc->sfs[idx].allocated = false;
179 hwc->sfs[idx].pending_delete = false;
180 trace_mlx5_sf_hwc_free(dev, hwc->start_fn_id + idx);
181 }
182
mlx5_sf_hw_table_sf_deferred_free(struct mlx5_core_dev * dev,u32 controller,u16 id)183 void mlx5_sf_hw_table_sf_deferred_free(struct mlx5_core_dev *dev, u32 controller, u16 id)
184 {
185 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
186 u32 out[MLX5_ST_SZ_DW(query_vhca_state_out)] = {};
187 struct mlx5_sf_hwc_table *hwc;
188 u16 hw_fn_id;
189 u8 state;
190 int err;
191
192 hw_fn_id = mlx5_sf_sw_to_hw_id(dev, controller, id);
193 hwc = mlx5_sf_controller_to_hwc(dev, controller);
194 mutex_lock(&table->table_lock);
195 err = mlx5_cmd_query_vhca_state(dev, hw_fn_id, out, sizeof(out));
196 if (err)
197 goto err;
198 state = MLX5_GET(query_vhca_state_out, out, vhca_state_context.vhca_state);
199 if (state == MLX5_VHCA_STATE_ALLOCATED) {
200 mlx5_cmd_dealloc_sf(dev, hw_fn_id);
201 hwc->sfs[id].allocated = false;
202 } else {
203 hwc->sfs[id].pending_delete = true;
204 trace_mlx5_sf_hwc_deferred_free(dev, hw_fn_id);
205 }
206 err:
207 mutex_unlock(&table->table_lock);
208 }
209
mlx5_sf_hw_table_hwc_dealloc_all(struct mlx5_core_dev * dev,struct mlx5_sf_hwc_table * hwc)210 static void mlx5_sf_hw_table_hwc_dealloc_all(struct mlx5_core_dev *dev,
211 struct mlx5_sf_hwc_table *hwc)
212 {
213 int i;
214
215 for (i = 0; i < hwc->max_fn; i++) {
216 if (hwc->sfs[i].allocated)
217 mlx5_sf_hw_table_hwc_sf_free(dev, hwc, i);
218 }
219 }
220
mlx5_sf_hw_table_dealloc_all(struct mlx5_core_dev * dev,struct mlx5_sf_hw_table * table)221 static void mlx5_sf_hw_table_dealloc_all(struct mlx5_core_dev *dev,
222 struct mlx5_sf_hw_table *table)
223 {
224 mlx5_sf_hw_table_hwc_dealloc_all(dev,
225 &table->hwc[MLX5_SF_HWC_EXTERNAL]);
226 mlx5_sf_hw_table_hwc_dealloc_all(dev, &table->hwc[MLX5_SF_HWC_LOCAL]);
227 }
228
mlx5_sf_hw_table_hwc_init(struct mlx5_sf_hwc_table * hwc,u16 max_fn,u16 base_id)229 static int mlx5_sf_hw_table_hwc_init(struct mlx5_sf_hwc_table *hwc, u16 max_fn, u16 base_id)
230 {
231 struct mlx5_sf_hw *sfs;
232
233 if (!max_fn)
234 return 0;
235
236 sfs = kzalloc_objs(*sfs, max_fn);
237 if (!sfs)
238 return -ENOMEM;
239
240 hwc->sfs = sfs;
241 hwc->max_fn = max_fn;
242 hwc->start_fn_id = base_id;
243 return 0;
244 }
245
mlx5_sf_hw_table_hwc_cleanup(struct mlx5_sf_hwc_table * hwc)246 static void mlx5_sf_hw_table_hwc_cleanup(struct mlx5_sf_hwc_table *hwc)
247 {
248 kfree(hwc->sfs);
249 }
250
mlx5_sf_hw_table_res_unregister(struct mlx5_core_dev * dev)251 static void mlx5_sf_hw_table_res_unregister(struct mlx5_core_dev *dev)
252 {
253 devl_resources_unregister(priv_to_devlink(dev));
254 }
255
mlx5_sf_hw_table_res_register(struct mlx5_core_dev * dev,u16 max_fn,u16 max_ext_fn)256 static int mlx5_sf_hw_table_res_register(struct mlx5_core_dev *dev, u16 max_fn,
257 u16 max_ext_fn)
258 {
259 struct devlink_resource_size_params size_params;
260 struct devlink *devlink = priv_to_devlink(dev);
261 int err;
262
263 devlink_resource_size_params_init(&size_params, max_fn, max_fn, 1,
264 DEVLINK_RESOURCE_UNIT_ENTRY);
265 err = devl_resource_register(devlink, "max_local_SFs", max_fn, MLX5_DL_RES_MAX_LOCAL_SFS,
266 DEVLINK_RESOURCE_ID_PARENT_TOP, &size_params);
267 if (err)
268 return err;
269
270 devlink_resource_size_params_init(&size_params, max_ext_fn, max_ext_fn, 1,
271 DEVLINK_RESOURCE_UNIT_ENTRY);
272 return devl_resource_register(devlink, "max_external_SFs", max_ext_fn,
273 MLX5_DL_RES_MAX_EXTERNAL_SFS, DEVLINK_RESOURCE_ID_PARENT_TOP,
274 &size_params);
275 }
276
mlx5_sf_hw_table_init(struct mlx5_core_dev * dev)277 int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev)
278 {
279 struct mlx5_sf_hw_table *table;
280 u16 max_ext_fn = 0;
281 u16 ext_base_id = 0;
282 u16 base_id;
283 u16 max_fn;
284 int err;
285
286 if (!mlx5_vhca_event_supported(dev))
287 return 0;
288
289 max_fn = mlx5_sf_max_functions(dev);
290
291 err = mlx5_esw_sf_max_hpf_functions(dev, &max_ext_fn, &ext_base_id);
292 if (err)
293 return err;
294
295 if (mlx5_sf_hw_table_res_register(dev, max_fn, max_ext_fn))
296 mlx5_core_dbg(dev, "failed to register max SFs resources");
297
298 if (!max_fn && !max_ext_fn)
299 return 0;
300
301 table = kzalloc_obj(*table);
302 if (!table) {
303 err = -ENOMEM;
304 goto alloc_err;
305 }
306
307 mutex_init(&table->table_lock);
308 dev->priv.sf_hw_table = table;
309
310 base_id = mlx5_sf_start_function_id(dev);
311 err = mlx5_sf_hw_table_hwc_init(&table->hwc[MLX5_SF_HWC_LOCAL], max_fn, base_id);
312 if (err)
313 goto table_err;
314
315 err = mlx5_sf_hw_table_hwc_init(&table->hwc[MLX5_SF_HWC_EXTERNAL],
316 max_ext_fn, ext_base_id);
317 if (err)
318 goto ext_err;
319
320 mlx5_core_dbg(dev, "SF HW table: max sfs = %d, ext sfs = %d\n", max_fn, max_ext_fn);
321 return 0;
322
323 ext_err:
324 mlx5_sf_hw_table_hwc_cleanup(&table->hwc[MLX5_SF_HWC_LOCAL]);
325 table_err:
326 mutex_destroy(&table->table_lock);
327 kfree(table);
328 alloc_err:
329 mlx5_sf_hw_table_res_unregister(dev);
330 return err;
331 }
332
mlx5_sf_hw_table_cleanup(struct mlx5_core_dev * dev)333 void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev)
334 {
335 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
336
337 if (!table)
338 goto res_unregister;
339
340 mlx5_sf_hw_table_hwc_cleanup(&table->hwc[MLX5_SF_HWC_EXTERNAL]);
341 mlx5_sf_hw_table_hwc_cleanup(&table->hwc[MLX5_SF_HWC_LOCAL]);
342 mutex_destroy(&table->table_lock);
343 kfree(table);
344 dev->priv.sf_hw_table = NULL;
345 res_unregister:
346 mlx5_sf_hw_table_res_unregister(dev);
347 }
348
mlx5_sf_hw_vhca_event(struct notifier_block * nb,unsigned long opcode,void * data)349 static int mlx5_sf_hw_vhca_event(struct notifier_block *nb, unsigned long opcode, void *data)
350 {
351 struct mlx5_core_dev *dev = container_of(nb, struct mlx5_core_dev,
352 priv.sf_hw_table_vhca_nb);
353 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
354 const struct mlx5_vhca_state_event *event = data;
355 struct mlx5_sf_hwc_table *hwc;
356 struct mlx5_sf_hw *sf_hw;
357 u16 sw_id;
358
359 if (!table || event->new_vhca_state != MLX5_VHCA_STATE_ALLOCATED)
360 return 0;
361
362 hwc = mlx5_sf_table_fn_to_hwc(table, event->function_id);
363 if (!hwc)
364 return 0;
365
366 sw_id = mlx5_sf_hw_to_sw_id(hwc, event->function_id);
367 sf_hw = &hwc->sfs[sw_id];
368
369 mutex_lock(&table->table_lock);
370 /* SF driver notified through firmware that SF is finally detached.
371 * Hence recycle the sf hardware id for reuse.
372 */
373 if (sf_hw->allocated && sf_hw->pending_delete)
374 mlx5_sf_hw_table_hwc_sf_free(dev, hwc, sw_id);
375 mutex_unlock(&table->table_lock);
376 return 0;
377 }
378
mlx5_sf_hw_notifier_init(struct mlx5_core_dev * dev)379 int mlx5_sf_hw_notifier_init(struct mlx5_core_dev *dev)
380 {
381 if (mlx5_core_is_sf(dev))
382 return 0;
383
384 dev->priv.sf_hw_table_vhca_nb.notifier_call = mlx5_sf_hw_vhca_event;
385 return mlx5_vhca_event_notifier_register(dev,
386 &dev->priv.sf_hw_table_vhca_nb);
387 }
388
mlx5_sf_hw_notifier_cleanup(struct mlx5_core_dev * dev)389 void mlx5_sf_hw_notifier_cleanup(struct mlx5_core_dev *dev)
390 {
391 if (mlx5_core_is_sf(dev))
392 return;
393
394 mlx5_vhca_event_notifier_unregister(dev,
395 &dev->priv.sf_hw_table_vhca_nb);
396 }
397
mlx5_sf_hw_table_destroy(struct mlx5_core_dev * dev)398 void mlx5_sf_hw_table_destroy(struct mlx5_core_dev *dev)
399 {
400 struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table;
401
402 if (!table)
403 return;
404
405 /* Dealloc SFs whose firmware event has been missed. */
406 mlx5_sf_hw_table_dealloc_all(dev, table);
407 }
408
mlx5_sf_hw_table_supported(const struct mlx5_core_dev * dev)409 bool mlx5_sf_hw_table_supported(const struct mlx5_core_dev *dev)
410 {
411 return !!dev->priv.sf_hw_table;
412 }
413