1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2018 Mellanox Technologies */
3
4 #include <linux/mlx5/vport.h>
5 #include <linux/list.h>
6 #include <linux/lockdep.h>
7 #include "lib/devcom.h"
8 #include "lib/mlx5.h"
9 #include "mlx5_core.h"
10
11 static LIST_HEAD(devcom_dev_list);
12 static LIST_HEAD(devcom_comp_list);
13 /* protect device list */
14 static DEFINE_MUTEX(dev_list_lock);
15 /* protect component list */
16 static DEFINE_MUTEX(comp_list_lock);
17
18 #define devcom_for_each_component(iter) \
19 list_for_each_entry(iter, &devcom_comp_list, comp_list)
20
21 struct mlx5_devcom_dev {
22 struct list_head list;
23 struct mlx5_core_dev *dev;
24 struct kref ref;
25 };
26
27 struct mlx5_devcom_key {
28 u32 flags;
29 union mlx5_devcom_match_key key;
30 possible_net_t net;
31 };
32
33 struct mlx5_devcom_comp {
34 struct list_head comp_list;
35 enum mlx5_devcom_component id;
36 struct list_head comp_dev_list_head;
37 struct mlx5_devcom_key key;
38 mlx5_devcom_event_handler_t handler;
39 struct kref ref;
40 int nr_devs;
41 bool ready;
42 struct rw_semaphore sem;
43 struct lock_class_key lock_key;
44 };
45
46 struct mlx5_devcom_comp_dev {
47 struct list_head list;
48 struct mlx5_devcom_comp *comp;
49 struct mlx5_devcom_dev *devc;
50 void __rcu *data;
51 };
52
devcom_dev_exists(struct mlx5_core_dev * dev)53 static bool devcom_dev_exists(struct mlx5_core_dev *dev)
54 {
55 struct mlx5_devcom_dev *iter;
56
57 list_for_each_entry(iter, &devcom_dev_list, list)
58 if (iter->dev == dev)
59 return true;
60
61 return false;
62 }
63
64 static struct mlx5_devcom_dev *
mlx5_devcom_dev_alloc(struct mlx5_core_dev * dev)65 mlx5_devcom_dev_alloc(struct mlx5_core_dev *dev)
66 {
67 struct mlx5_devcom_dev *devc;
68
69 devc = kzalloc_obj(*devc);
70 if (!devc)
71 return NULL;
72
73 devc->dev = dev;
74 kref_init(&devc->ref);
75 return devc;
76 }
77
78 struct mlx5_devcom_dev *
mlx5_devcom_register_device(struct mlx5_core_dev * dev)79 mlx5_devcom_register_device(struct mlx5_core_dev *dev)
80 {
81 struct mlx5_devcom_dev *devc = NULL;
82
83 mutex_lock(&dev_list_lock);
84
85 if (devcom_dev_exists(dev)) {
86 mlx5_core_err(dev, "devcom device already exists");
87 goto out;
88 }
89
90 devc = mlx5_devcom_dev_alloc(dev);
91 if (!devc)
92 goto out;
93
94 list_add_tail(&devc->list, &devcom_dev_list);
95 out:
96 mutex_unlock(&dev_list_lock);
97 return devc;
98 }
99
100 static void
mlx5_devcom_dev_release(struct kref * ref)101 mlx5_devcom_dev_release(struct kref *ref)
102 {
103 struct mlx5_devcom_dev *devc = container_of(ref, struct mlx5_devcom_dev, ref);
104
105 mutex_lock(&dev_list_lock);
106 list_del(&devc->list);
107 mutex_unlock(&dev_list_lock);
108 kfree(devc);
109 }
110
mlx5_devcom_unregister_device(struct mlx5_devcom_dev * devc)111 void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc)
112 {
113 if (!devc)
114 return;
115
116 kref_put(&devc->ref, mlx5_devcom_dev_release);
117 }
118
119 static struct mlx5_devcom_comp *
mlx5_devcom_comp_alloc(u64 id,const struct mlx5_devcom_match_attr * attr,mlx5_devcom_event_handler_t handler)120 mlx5_devcom_comp_alloc(u64 id, const struct mlx5_devcom_match_attr *attr,
121 mlx5_devcom_event_handler_t handler)
122 {
123 struct mlx5_devcom_comp *comp;
124
125 comp = kzalloc_obj(*comp);
126 if (!comp)
127 return NULL;
128
129 comp->id = id;
130 comp->key.key = attr->key;
131 comp->key.flags = attr->flags;
132 if (attr->flags & MLX5_DEVCOM_MATCH_FLAGS_NS)
133 write_pnet(&comp->key.net, attr->net);
134 comp->handler = handler;
135 init_rwsem(&comp->sem);
136 lockdep_register_key(&comp->lock_key);
137 lockdep_set_class(&comp->sem, &comp->lock_key);
138 kref_init(&comp->ref);
139 INIT_LIST_HEAD(&comp->comp_dev_list_head);
140
141 return comp;
142 }
143
144 static void
mlx5_devcom_comp_release(struct kref * ref)145 mlx5_devcom_comp_release(struct kref *ref)
146 {
147 struct mlx5_devcom_comp *comp = container_of(ref, struct mlx5_devcom_comp, ref);
148
149 mutex_lock(&comp_list_lock);
150 list_del(&comp->comp_list);
151 mutex_unlock(&comp_list_lock);
152 lockdep_unregister_key(&comp->lock_key);
153 kfree(comp);
154 }
155
156 static struct mlx5_devcom_comp_dev *
devcom_alloc_comp_dev(struct mlx5_devcom_dev * devc,struct mlx5_devcom_comp * comp,void * data)157 devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc,
158 struct mlx5_devcom_comp *comp,
159 void *data)
160 {
161 struct mlx5_devcom_comp_dev *devcom;
162
163 devcom = kzalloc_obj(*devcom);
164 if (!devcom)
165 return NULL;
166
167 kref_get(&devc->ref);
168 devcom->devc = devc;
169 devcom->comp = comp;
170 rcu_assign_pointer(devcom->data, data);
171
172 down_write(&comp->sem);
173 list_add_tail(&devcom->list, &comp->comp_dev_list_head);
174 WRITE_ONCE(comp->nr_devs, comp->nr_devs + 1);
175 up_write(&comp->sem);
176
177 return devcom;
178 }
179
180 static void
devcom_free_comp_dev(struct mlx5_devcom_comp_dev * devcom)181 devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom)
182 {
183 struct mlx5_devcom_comp *comp = devcom->comp;
184
185 down_write(&comp->sem);
186 list_del(&devcom->list);
187 WRITE_ONCE(comp->nr_devs, comp->nr_devs - 1);
188 up_write(&comp->sem);
189
190 kref_put(&devcom->devc->ref, mlx5_devcom_dev_release);
191 kfree(devcom);
192 kref_put(&comp->ref, mlx5_devcom_comp_release);
193 }
194
195 static bool
devcom_component_equal(struct mlx5_devcom_comp * devcom,enum mlx5_devcom_component id,const struct mlx5_devcom_match_attr * attr)196 devcom_component_equal(struct mlx5_devcom_comp *devcom,
197 enum mlx5_devcom_component id,
198 const struct mlx5_devcom_match_attr *attr)
199 {
200 if (devcom->id != id)
201 return false;
202
203 if (devcom->key.flags != attr->flags)
204 return false;
205
206 if (memcmp(&devcom->key.key, &attr->key, sizeof(devcom->key.key)))
207 return false;
208
209 if (devcom->key.flags & MLX5_DEVCOM_MATCH_FLAGS_NS &&
210 !net_eq(read_pnet(&devcom->key.net), attr->net))
211 return false;
212
213 return true;
214 }
215
216 static struct mlx5_devcom_comp *
devcom_component_get(struct mlx5_devcom_dev * devc,enum mlx5_devcom_component id,const struct mlx5_devcom_match_attr * attr,mlx5_devcom_event_handler_t handler)217 devcom_component_get(struct mlx5_devcom_dev *devc,
218 enum mlx5_devcom_component id,
219 const struct mlx5_devcom_match_attr *attr,
220 mlx5_devcom_event_handler_t handler)
221 {
222 struct mlx5_devcom_comp *comp;
223
224 devcom_for_each_component(comp) {
225 if (devcom_component_equal(comp, id, attr)) {
226 if (handler == comp->handler) {
227 kref_get(&comp->ref);
228 return comp;
229 }
230
231 mlx5_core_err(devc->dev,
232 "Cannot register existing devcom component with different handler\n");
233 return ERR_PTR(-EINVAL);
234 }
235 }
236
237 return NULL;
238 }
239
240 struct mlx5_devcom_comp_dev *
mlx5_devcom_register_component(struct mlx5_devcom_dev * devc,enum mlx5_devcom_component id,const struct mlx5_devcom_match_attr * attr,mlx5_devcom_event_handler_t handler,void * data)241 mlx5_devcom_register_component(struct mlx5_devcom_dev *devc,
242 enum mlx5_devcom_component id,
243 const struct mlx5_devcom_match_attr *attr,
244 mlx5_devcom_event_handler_t handler,
245 void *data)
246 {
247 struct mlx5_devcom_comp_dev *devcom = NULL;
248 struct mlx5_devcom_comp *comp;
249
250 if (!devc)
251 return NULL;
252
253 mutex_lock(&comp_list_lock);
254 comp = devcom_component_get(devc, id, attr, handler);
255 if (IS_ERR(comp))
256 goto out_unlock;
257
258 if (!comp) {
259 comp = mlx5_devcom_comp_alloc(id, attr, handler);
260 if (!comp)
261 goto out_unlock;
262
263 list_add_tail(&comp->comp_list, &devcom_comp_list);
264 }
265 mutex_unlock(&comp_list_lock);
266
267 devcom = devcom_alloc_comp_dev(devc, comp, data);
268 if (!devcom)
269 kref_put(&comp->ref, mlx5_devcom_comp_release);
270
271 return devcom;
272
273 out_unlock:
274 mutex_unlock(&comp_list_lock);
275 return devcom;
276 }
277
mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev * devcom)278 void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom)
279 {
280 if (!devcom)
281 return;
282
283 devcom_free_comp_dev(devcom);
284 }
285
mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev * devcom)286 int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom)
287 {
288 struct mlx5_devcom_comp *comp = devcom->comp;
289
290 return READ_ONCE(comp->nr_devs);
291 }
292
mlx5_devcom_locked_send_event(struct mlx5_devcom_comp_dev * devcom,int event,int rollback_event,void * event_data)293 int mlx5_devcom_locked_send_event(struct mlx5_devcom_comp_dev *devcom,
294 int event, int rollback_event,
295 void *event_data)
296 {
297 struct mlx5_devcom_comp_dev *pos;
298 struct mlx5_devcom_comp *comp;
299 int err = 0;
300 void *data;
301
302 if (!devcom)
303 return -ENODEV;
304
305 lockdep_assert_held_write(&devcom->comp->sem);
306 comp = devcom->comp;
307 list_for_each_entry(pos, &comp->comp_dev_list_head, list) {
308 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem));
309
310 if (pos != devcom && data) {
311 err = comp->handler(event, data, event_data);
312 if (err && rollback_event != DEVCOM_CANT_FAIL) {
313 goto rollback;
314 } else if (err && rollback_event == DEVCOM_CANT_FAIL) {
315 WARN_ONCE(1, "devcom component %d event %d failed: %d\n",
316 comp->id, event, err);
317 return err;
318 }
319 }
320 }
321
322 return 0;
323
324 rollback:
325 if (list_entry_is_head(pos, &comp->comp_dev_list_head, list))
326 return err;
327 pos = list_prev_entry(pos, list);
328 list_for_each_entry_from_reverse(pos, &comp->comp_dev_list_head, list) {
329 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem));
330
331 if (pos != devcom && data)
332 comp->handler(rollback_event, data, event_data);
333 }
334 return err;
335 }
336
mlx5_devcom_send_event(struct mlx5_devcom_comp_dev * devcom,int event,int rollback_event,void * event_data)337 int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom,
338 int event, int rollback_event,
339 void *event_data)
340 {
341 struct mlx5_devcom_comp *comp;
342 int err;
343
344 if (!devcom)
345 return -ENODEV;
346
347 comp = devcom->comp;
348 down_write(&comp->sem);
349 err = mlx5_devcom_locked_send_event(devcom, event, rollback_event,
350 event_data);
351 up_write(&comp->sem);
352 return err;
353 }
354
mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev * devcom,bool ready)355 void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready)
356 {
357 WARN_ON(!rwsem_is_locked(&devcom->comp->sem));
358
359 WRITE_ONCE(devcom->comp->ready, ready);
360 }
361
mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev * devcom)362 bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom)
363 {
364 if (!devcom)
365 return false;
366
367 return READ_ONCE(devcom->comp->ready);
368 }
369
mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev * devcom)370 bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom)
371 {
372 struct mlx5_devcom_comp *comp;
373
374 if (!devcom)
375 return false;
376
377 comp = devcom->comp;
378 down_read(&comp->sem);
379 if (!READ_ONCE(comp->ready)) {
380 up_read(&comp->sem);
381 return false;
382 }
383
384 return true;
385 }
386
mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev * devcom)387 void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom)
388 {
389 up_read(&devcom->comp->sem);
390 }
391
mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev * devcom,struct mlx5_devcom_comp_dev ** pos)392 void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom,
393 struct mlx5_devcom_comp_dev **pos)
394 {
395 struct mlx5_devcom_comp *comp = devcom->comp;
396 struct mlx5_devcom_comp_dev *tmp;
397 void *data;
398
399 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list);
400
401 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) {
402 if (tmp != devcom) {
403 data = rcu_dereference_protected(tmp->data, lockdep_is_held(&comp->sem));
404 if (data)
405 break;
406 }
407 }
408
409 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list))
410 return NULL;
411
412 *pos = tmp;
413 return data;
414 }
415
mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev * devcom,struct mlx5_devcom_comp_dev ** pos)416 void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom,
417 struct mlx5_devcom_comp_dev **pos)
418 {
419 struct mlx5_devcom_comp *comp = devcom->comp;
420 struct mlx5_devcom_comp_dev *tmp;
421 void *data;
422
423 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list);
424
425 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) {
426 if (tmp != devcom) {
427 /* This can change concurrently, however 'data' pointer will remain
428 * valid for the duration of RCU read section.
429 */
430 if (!READ_ONCE(comp->ready))
431 return NULL;
432 data = rcu_dereference(tmp->data);
433 if (data)
434 break;
435 }
436 }
437
438 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list))
439 return NULL;
440
441 *pos = tmp;
442 return data;
443 }
444
mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev * devcom)445 void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom)
446 {
447 if (!devcom)
448 return;
449 down_write(&devcom->comp->sem);
450 }
451
mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev * devcom)452 void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom)
453 {
454 if (!devcom)
455 return;
456 up_write(&devcom->comp->sem);
457 }
458
mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev * devcom)459 int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom)
460 {
461 if (!devcom)
462 return 0;
463 return down_write_trylock(&devcom->comp->sem);
464 }
465
mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev * devcom)466 void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom)
467 {
468 if (!devcom)
469 return;
470 lockdep_assert_held_write(&devcom->comp->sem);
471 }
472