1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/netdevice.h>
34 #include <net/bonding.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/eswitch.h>
37 #include <linux/mlx5/vport.h>
38 #include "lib/devcom.h"
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/acl/ofld.h"
42 #include "lag.h"
43 #include "mp.h"
44 #include "mpesw.h"
45
46
47 /* General purpose, use for short periods of time.
48 * Beware of lock dependencies (preferably, no locks should be acquired
49 * under it).
50 */
51 static DEFINE_SPINLOCK(lag_lock);
52
get_port_sel_mode(enum mlx5_lag_mode mode,unsigned long flags)53 static int get_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags)
54 {
55 if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags))
56 return MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT;
57
58 if (mode == MLX5_LAG_MODE_MPESW)
59 return MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_MPESW;
60
61 return MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY;
62 }
63
lag_active_port_bits(struct mlx5_lag * ldev)64 static u8 lag_active_port_bits(struct mlx5_lag *ldev)
65 {
66 u8 enabled_ports[MLX5_MAX_PORTS] = {};
67 u8 active_port = 0;
68 int num_enabled;
69 int idx;
70
71 mlx5_infer_tx_enabled(&ldev->tracker, ldev, enabled_ports,
72 &num_enabled);
73 for (idx = 0; idx < num_enabled; idx++)
74 active_port |= BIT_MASK(enabled_ports[idx]);
75
76 return active_port;
77 }
78
mlx5_cmd_create_lag(struct mlx5_core_dev * dev,struct mlx5_lag * ldev,int mode,unsigned long flags)79 static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev,
80 int mode, unsigned long flags)
81 {
82 bool fdb_sel_mode = test_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE,
83 &flags);
84 int port_sel_mode = get_port_sel_mode(mode, flags);
85 u32 in[MLX5_ST_SZ_DW(create_lag_in)] = {};
86 u8 *ports = ldev->v2p_map;
87 int idx0, idx1;
88 void *lag_ctx;
89
90 lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx);
91 MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG);
92 MLX5_SET(lagc, lag_ctx, fdb_selection_mode, fdb_sel_mode);
93 idx0 = mlx5_lag_get_dev_index_by_seq(ldev, 0);
94 idx1 = mlx5_lag_get_dev_index_by_seq(ldev, 1);
95
96 if (idx0 < 0 || idx1 < 0)
97 return -EINVAL;
98
99 switch (port_sel_mode) {
100 case MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY:
101 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, ports[idx0]);
102 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, ports[idx1]);
103 break;
104 case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT:
105 if (!MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass))
106 break;
107
108 MLX5_SET(lagc, lag_ctx, active_port,
109 lag_active_port_bits(mlx5_lag_dev(dev)));
110 break;
111 default:
112 break;
113 }
114 MLX5_SET(lagc, lag_ctx, port_select_mode, port_sel_mode);
115
116 return mlx5_cmd_exec_in(dev, create_lag, in);
117 }
118
mlx5_cmd_modify_lag(struct mlx5_core_dev * dev,struct mlx5_lag * ldev,u8 * ports)119 static int mlx5_cmd_modify_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev,
120 u8 *ports)
121 {
122 u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {};
123 void *lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
124 int idx0, idx1;
125
126 idx0 = mlx5_lag_get_dev_index_by_seq(ldev, 0);
127 idx1 = mlx5_lag_get_dev_index_by_seq(ldev, 1);
128 if (idx0 < 0 || idx1 < 0)
129 return -EINVAL;
130
131 MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
132 MLX5_SET(modify_lag_in, in, field_select, 0x1);
133
134 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, ports[idx0]);
135 MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, ports[idx1]);
136
137 return mlx5_cmd_exec_in(dev, modify_lag, in);
138 }
139
mlx5_cmd_create_vport_lag(struct mlx5_core_dev * dev)140 int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev)
141 {
142 u32 in[MLX5_ST_SZ_DW(create_vport_lag_in)] = {};
143
144 MLX5_SET(create_vport_lag_in, in, opcode, MLX5_CMD_OP_CREATE_VPORT_LAG);
145
146 return mlx5_cmd_exec_in(dev, create_vport_lag, in);
147 }
148 EXPORT_SYMBOL(mlx5_cmd_create_vport_lag);
149
mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev * dev)150 int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev)
151 {
152 u32 in[MLX5_ST_SZ_DW(destroy_vport_lag_in)] = {};
153
154 MLX5_SET(destroy_vport_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_VPORT_LAG);
155
156 return mlx5_cmd_exec_in(dev, destroy_vport_lag, in);
157 }
158 EXPORT_SYMBOL(mlx5_cmd_destroy_vport_lag);
159
mlx5_infer_tx_disabled(struct lag_tracker * tracker,struct mlx5_lag * ldev,u8 * ports,int * num_disabled)160 static void mlx5_infer_tx_disabled(struct lag_tracker *tracker, struct mlx5_lag *ldev,
161 u8 *ports, int *num_disabled)
162 {
163 int i;
164
165 *num_disabled = 0;
166 mlx5_ldev_for_each(i, 0, ldev)
167 if (!tracker->netdev_state[i].tx_enabled ||
168 !tracker->netdev_state[i].link_up)
169 ports[(*num_disabled)++] = i;
170 }
171
mlx5_infer_tx_enabled(struct lag_tracker * tracker,struct mlx5_lag * ldev,u8 * ports,int * num_enabled)172 void mlx5_infer_tx_enabled(struct lag_tracker *tracker, struct mlx5_lag *ldev,
173 u8 *ports, int *num_enabled)
174 {
175 int i;
176
177 *num_enabled = 0;
178 mlx5_ldev_for_each(i, 0, ldev)
179 if (tracker->netdev_state[i].tx_enabled &&
180 tracker->netdev_state[i].link_up)
181 ports[(*num_enabled)++] = i;
182
183 if (*num_enabled == 0)
184 mlx5_infer_tx_disabled(tracker, ldev, ports, num_enabled);
185 }
186
mlx5_lag_print_mapping(struct mlx5_core_dev * dev,struct mlx5_lag * ldev,struct lag_tracker * tracker,unsigned long flags)187 static void mlx5_lag_print_mapping(struct mlx5_core_dev *dev,
188 struct mlx5_lag *ldev,
189 struct lag_tracker *tracker,
190 unsigned long flags)
191 {
192 char buf[MLX5_MAX_PORTS * 10 + 1] = {};
193 u8 enabled_ports[MLX5_MAX_PORTS] = {};
194 int written = 0;
195 int num_enabled;
196 int idx;
197 int err;
198 int i;
199 int j;
200
201 if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
202 mlx5_infer_tx_enabled(tracker, ldev, enabled_ports,
203 &num_enabled);
204 for (i = 0; i < num_enabled; i++) {
205 err = scnprintf(buf + written, 4, "%d, ", enabled_ports[i] + 1);
206 if (err != 3)
207 return;
208 written += err;
209 }
210 buf[written - 2] = 0;
211 mlx5_core_info(dev, "lag map active ports: %s\n", buf);
212 } else {
213 mlx5_ldev_for_each(i, 0, ldev) {
214 for (j = 0; j < ldev->buckets; j++) {
215 idx = i * ldev->buckets + j;
216 err = scnprintf(buf + written, 10,
217 " port %d:%d", i + 1, ldev->v2p_map[idx]);
218 if (err != 9)
219 return;
220 written += err;
221 }
222 }
223 mlx5_core_info(dev, "lag map:%s\n", buf);
224 }
225 }
226
227 static int mlx5_lag_netdev_event(struct notifier_block *this,
228 unsigned long event, void *ptr);
229 static void mlx5_do_bond_work(struct work_struct *work);
230
mlx5_ldev_free(struct kref * ref)231 static void mlx5_ldev_free(struct kref *ref)
232 {
233 struct mlx5_lag *ldev = container_of(ref, struct mlx5_lag, ref);
234
235 if (ldev->nb.notifier_call)
236 unregister_netdevice_notifier_net(&init_net, &ldev->nb);
237 mlx5_lag_mp_cleanup(ldev);
238 cancel_delayed_work_sync(&ldev->bond_work);
239 destroy_workqueue(ldev->wq);
240 mutex_destroy(&ldev->lock);
241 kfree(ldev);
242 }
243
mlx5_ldev_put(struct mlx5_lag * ldev)244 static void mlx5_ldev_put(struct mlx5_lag *ldev)
245 {
246 kref_put(&ldev->ref, mlx5_ldev_free);
247 }
248
mlx5_ldev_get(struct mlx5_lag * ldev)249 static void mlx5_ldev_get(struct mlx5_lag *ldev)
250 {
251 kref_get(&ldev->ref);
252 }
253
mlx5_lag_dev_alloc(struct mlx5_core_dev * dev)254 static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev)
255 {
256 struct mlx5_lag *ldev;
257 int err;
258
259 ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
260 if (!ldev)
261 return NULL;
262
263 ldev->wq = create_singlethread_workqueue("mlx5_lag");
264 if (!ldev->wq) {
265 kfree(ldev);
266 return NULL;
267 }
268
269 kref_init(&ldev->ref);
270 mutex_init(&ldev->lock);
271 INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work);
272
273 ldev->nb.notifier_call = mlx5_lag_netdev_event;
274 if (register_netdevice_notifier_net(&init_net, &ldev->nb)) {
275 ldev->nb.notifier_call = NULL;
276 mlx5_core_err(dev, "Failed to register LAG netdev notifier\n");
277 }
278 ldev->mode = MLX5_LAG_MODE_NONE;
279
280 err = mlx5_lag_mp_init(ldev);
281 if (err)
282 mlx5_core_err(dev, "Failed to init multipath lag err=%d\n",
283 err);
284
285 ldev->ports = MLX5_CAP_GEN(dev, num_lag_ports);
286 ldev->buckets = 1;
287
288 return ldev;
289 }
290
mlx5_lag_dev_get_netdev_idx(struct mlx5_lag * ldev,struct net_device * ndev)291 int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
292 struct net_device *ndev)
293 {
294 int i;
295
296 mlx5_ldev_for_each(i, 0, ldev)
297 if (ldev->pf[i].netdev == ndev)
298 return i;
299
300 return -ENOENT;
301 }
302
mlx5_lag_get_dev_index_by_seq(struct mlx5_lag * ldev,int seq)303 int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq)
304 {
305 int i, num = 0;
306
307 if (!ldev)
308 return -ENOENT;
309
310 mlx5_ldev_for_each(i, 0, ldev) {
311 if (num == seq)
312 return i;
313 num++;
314 }
315 return -ENOENT;
316 }
317
mlx5_lag_num_devs(struct mlx5_lag * ldev)318 int mlx5_lag_num_devs(struct mlx5_lag *ldev)
319 {
320 int i, num = 0;
321
322 if (!ldev)
323 return 0;
324
325 mlx5_ldev_for_each(i, 0, ldev) {
326 (void)i;
327 num++;
328 }
329 return num;
330 }
331
mlx5_lag_num_netdevs(struct mlx5_lag * ldev)332 int mlx5_lag_num_netdevs(struct mlx5_lag *ldev)
333 {
334 int i, num = 0;
335
336 if (!ldev)
337 return 0;
338
339 mlx5_ldev_for_each(i, 0, ldev)
340 if (ldev->pf[i].netdev)
341 num++;
342 return num;
343 }
344
__mlx5_lag_is_roce(struct mlx5_lag * ldev)345 static bool __mlx5_lag_is_roce(struct mlx5_lag *ldev)
346 {
347 return ldev->mode == MLX5_LAG_MODE_ROCE;
348 }
349
__mlx5_lag_is_sriov(struct mlx5_lag * ldev)350 static bool __mlx5_lag_is_sriov(struct mlx5_lag *ldev)
351 {
352 return ldev->mode == MLX5_LAG_MODE_SRIOV;
353 }
354
355 /* Create a mapping between steering slots and active ports.
356 * As we have ldev->buckets slots per port first assume the native
357 * mapping should be used.
358 * If there are ports that are disabled fill the relevant slots
359 * with mapping that points to active ports.
360 */
mlx5_infer_tx_affinity_mapping(struct lag_tracker * tracker,struct mlx5_lag * ldev,u8 buckets,u8 * ports)361 static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
362 struct mlx5_lag *ldev,
363 u8 buckets,
364 u8 *ports)
365 {
366 int disabled[MLX5_MAX_PORTS] = {};
367 int enabled[MLX5_MAX_PORTS] = {};
368 int disabled_ports_num = 0;
369 int enabled_ports_num = 0;
370 int idx;
371 u32 rand;
372 int i;
373 int j;
374
375 mlx5_ldev_for_each(i, 0, ldev) {
376 if (tracker->netdev_state[i].tx_enabled &&
377 tracker->netdev_state[i].link_up)
378 enabled[enabled_ports_num++] = i;
379 else
380 disabled[disabled_ports_num++] = i;
381 }
382
383 /* Use native mapping by default where each port's buckets
384 * point the native port: 1 1 1 .. 1 2 2 2 ... 2 3 3 3 ... 3 etc
385 */
386 mlx5_ldev_for_each(i, 0, ldev) {
387 for (j = 0; j < buckets; j++) {
388 idx = i * buckets + j;
389 ports[idx] = i + 1;
390 }
391 }
392
393 /* If all ports are disabled/enabled keep native mapping */
394 if (enabled_ports_num == ldev->ports ||
395 disabled_ports_num == ldev->ports)
396 return;
397
398 /* Go over the disabled ports and for each assign a random active port */
399 for (i = 0; i < disabled_ports_num; i++) {
400 for (j = 0; j < buckets; j++) {
401 get_random_bytes(&rand, 4);
402 ports[disabled[i] * buckets + j] = enabled[rand % enabled_ports_num] + 1;
403 }
404 }
405 }
406
mlx5_lag_has_drop_rule(struct mlx5_lag * ldev)407 static bool mlx5_lag_has_drop_rule(struct mlx5_lag *ldev)
408 {
409 int i;
410
411 mlx5_ldev_for_each(i, 0, ldev)
412 if (ldev->pf[i].has_drop)
413 return true;
414 return false;
415 }
416
mlx5_lag_drop_rule_cleanup(struct mlx5_lag * ldev)417 static void mlx5_lag_drop_rule_cleanup(struct mlx5_lag *ldev)
418 {
419 int i;
420
421 mlx5_ldev_for_each(i, 0, ldev) {
422 if (!ldev->pf[i].has_drop)
423 continue;
424
425 mlx5_esw_acl_ingress_vport_drop_rule_destroy(ldev->pf[i].dev->priv.eswitch,
426 MLX5_VPORT_UPLINK);
427 ldev->pf[i].has_drop = false;
428 }
429 }
430
mlx5_lag_drop_rule_setup(struct mlx5_lag * ldev,struct lag_tracker * tracker)431 static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
432 struct lag_tracker *tracker)
433 {
434 u8 disabled_ports[MLX5_MAX_PORTS] = {};
435 struct mlx5_core_dev *dev;
436 int disabled_index;
437 int num_disabled;
438 int err;
439 int i;
440
441 /* First delete the current drop rule so there won't be any dropped
442 * packets
443 */
444 mlx5_lag_drop_rule_cleanup(ldev);
445
446 if (!ldev->tracker.has_inactive)
447 return;
448
449 mlx5_infer_tx_disabled(tracker, ldev, disabled_ports, &num_disabled);
450
451 for (i = 0; i < num_disabled; i++) {
452 disabled_index = disabled_ports[i];
453 dev = ldev->pf[disabled_index].dev;
454 err = mlx5_esw_acl_ingress_vport_drop_rule_create(dev->priv.eswitch,
455 MLX5_VPORT_UPLINK);
456 if (!err)
457 ldev->pf[disabled_index].has_drop = true;
458 else
459 mlx5_core_err(dev,
460 "Failed to create lag drop rule, error: %d", err);
461 }
462 }
463
mlx5_cmd_modify_active_port(struct mlx5_core_dev * dev,u8 ports)464 static int mlx5_cmd_modify_active_port(struct mlx5_core_dev *dev, u8 ports)
465 {
466 u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {};
467 void *lag_ctx;
468
469 lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
470
471 MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
472 MLX5_SET(modify_lag_in, in, field_select, 0x2);
473
474 MLX5_SET(lagc, lag_ctx, active_port, ports);
475
476 return mlx5_cmd_exec_in(dev, modify_lag, in);
477 }
478
_mlx5_modify_lag(struct mlx5_lag * ldev,u8 * ports)479 static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports)
480 {
481 int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
482 struct mlx5_core_dev *dev0;
483 u8 active_ports;
484 int ret;
485
486 if (idx < 0)
487 return -EINVAL;
488
489 dev0 = ldev->pf[idx].dev;
490 if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) {
491 ret = mlx5_lag_port_sel_modify(ldev, ports);
492 if (ret ||
493 !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table_bypass))
494 return ret;
495
496 active_ports = lag_active_port_bits(ldev);
497
498 return mlx5_cmd_modify_active_port(dev0, active_ports);
499 }
500 return mlx5_cmd_modify_lag(dev0, ldev, ports);
501 }
502
mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev * dev)503 static struct net_device *mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev *dev)
504 {
505 struct net_device *ndev = NULL;
506 struct mlx5_lag *ldev;
507 unsigned long flags;
508 int i, last_idx;
509
510 spin_lock_irqsave(&lag_lock, flags);
511 ldev = mlx5_lag_dev(dev);
512
513 if (!ldev)
514 goto unlock;
515
516 mlx5_ldev_for_each(i, 0, ldev)
517 if (ldev->tracker.netdev_state[i].tx_enabled)
518 ndev = ldev->pf[i].netdev;
519 if (!ndev) {
520 last_idx = mlx5_lag_get_dev_index_by_seq(ldev, ldev->ports - 1);
521 if (last_idx < 0)
522 goto unlock;
523 ndev = ldev->pf[last_idx].netdev;
524 }
525
526 dev_hold(ndev);
527
528 unlock:
529 spin_unlock_irqrestore(&lag_lock, flags);
530
531 return ndev;
532 }
533
mlx5_modify_lag(struct mlx5_lag * ldev,struct lag_tracker * tracker)534 void mlx5_modify_lag(struct mlx5_lag *ldev,
535 struct lag_tracker *tracker)
536 {
537 int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
538 u8 ports[MLX5_MAX_PORTS * MLX5_LAG_MAX_HASH_BUCKETS] = {};
539 struct mlx5_core_dev *dev0;
540 int idx;
541 int err;
542 int i;
543 int j;
544
545 if (first_idx < 0)
546 return;
547
548 dev0 = ldev->pf[first_idx].dev;
549 mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ports);
550
551 mlx5_ldev_for_each(i, 0, ldev) {
552 for (j = 0; j < ldev->buckets; j++) {
553 idx = i * ldev->buckets + j;
554 if (ports[idx] == ldev->v2p_map[idx])
555 continue;
556 err = _mlx5_modify_lag(ldev, ports);
557 if (err) {
558 mlx5_core_err(dev0,
559 "Failed to modify LAG (%d)\n",
560 err);
561 return;
562 }
563 memcpy(ldev->v2p_map, ports, sizeof(ports));
564
565 mlx5_lag_print_mapping(dev0, ldev, tracker,
566 ldev->mode_flags);
567 break;
568 }
569 }
570
571 if (tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
572 struct net_device *ndev = mlx5_lag_active_backup_get_netdev(dev0);
573
574 if(!(ldev->mode == MLX5_LAG_MODE_ROCE))
575 mlx5_lag_drop_rule_setup(ldev, tracker);
576 /** Only sriov and roce lag should have tracker->tx_type set so
577 * no need to check the mode
578 */
579 blocking_notifier_call_chain(&dev0->priv.lag_nh,
580 MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE,
581 ndev);
582 dev_put(ndev);
583 }
584 }
585
mlx5_lag_set_port_sel_mode(struct mlx5_lag * ldev,enum mlx5_lag_mode mode,unsigned long * flags)586 static int mlx5_lag_set_port_sel_mode(struct mlx5_lag *ldev,
587 enum mlx5_lag_mode mode,
588 unsigned long *flags)
589 {
590 int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
591 struct mlx5_core_dev *dev0;
592
593 if (first_idx < 0)
594 return -EINVAL;
595
596 if (mode == MLX5_LAG_MODE_MPESW ||
597 mode == MLX5_LAG_MODE_MULTIPATH)
598 return 0;
599
600 dev0 = ldev->pf[first_idx].dev;
601
602 if (!MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table)) {
603 if (ldev->ports > 2)
604 return -EINVAL;
605 return 0;
606 }
607
608 if (ldev->ports > 2)
609 ldev->buckets = MLX5_LAG_MAX_HASH_BUCKETS;
610
611 set_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, flags);
612
613 return 0;
614 }
615
mlx5_lag_set_flags(struct mlx5_lag * ldev,enum mlx5_lag_mode mode,struct lag_tracker * tracker,bool shared_fdb,unsigned long * flags)616 static int mlx5_lag_set_flags(struct mlx5_lag *ldev, enum mlx5_lag_mode mode,
617 struct lag_tracker *tracker, bool shared_fdb,
618 unsigned long *flags)
619 {
620 *flags = 0;
621 if (shared_fdb) {
622 set_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, flags);
623 set_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE, flags);
624 }
625
626 if (mode == MLX5_LAG_MODE_MPESW)
627 set_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE, flags);
628
629 return mlx5_lag_set_port_sel_mode(ldev, mode, flags);
630 }
631
mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode,unsigned long flags)632 char *mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags)
633 {
634 int port_sel_mode = get_port_sel_mode(mode, flags);
635
636 switch (port_sel_mode) {
637 case MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY: return "queue_affinity";
638 case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT: return "hash";
639 case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_MPESW: return "mpesw";
640 default: return "invalid";
641 }
642 }
643
mlx5_lag_create_single_fdb(struct mlx5_lag * ldev)644 static int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev)
645 {
646 int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
647 struct mlx5_eswitch *master_esw;
648 struct mlx5_core_dev *dev0;
649 int i, j;
650 int err;
651
652 if (first_idx < 0)
653 return -EINVAL;
654
655 dev0 = ldev->pf[first_idx].dev;
656 master_esw = dev0->priv.eswitch;
657 mlx5_ldev_for_each(i, first_idx + 1, ldev) {
658 struct mlx5_eswitch *slave_esw = ldev->pf[i].dev->priv.eswitch;
659
660 err = mlx5_eswitch_offloads_single_fdb_add_one(master_esw,
661 slave_esw, ldev->ports);
662 if (err)
663 goto err;
664 }
665 return 0;
666 err:
667 mlx5_ldev_for_each_reverse(j, i, first_idx + 1, ldev)
668 mlx5_eswitch_offloads_single_fdb_del_one(master_esw,
669 ldev->pf[j].dev->priv.eswitch);
670 return err;
671 }
672
mlx5_create_lag(struct mlx5_lag * ldev,struct lag_tracker * tracker,enum mlx5_lag_mode mode,unsigned long flags)673 static int mlx5_create_lag(struct mlx5_lag *ldev,
674 struct lag_tracker *tracker,
675 enum mlx5_lag_mode mode,
676 unsigned long flags)
677 {
678 bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags);
679 int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
680 u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {};
681 struct mlx5_core_dev *dev0;
682 int err;
683
684 if (first_idx < 0)
685 return -EINVAL;
686
687 dev0 = ldev->pf[first_idx].dev;
688 if (tracker)
689 mlx5_lag_print_mapping(dev0, ldev, tracker, flags);
690 mlx5_core_info(dev0, "shared_fdb:%d mode:%s\n",
691 shared_fdb, mlx5_get_str_port_sel_mode(mode, flags));
692
693 err = mlx5_cmd_create_lag(dev0, ldev, mode, flags);
694 if (err) {
695 mlx5_core_err(dev0,
696 "Failed to create LAG (%d)\n",
697 err);
698 return err;
699 }
700
701 if (shared_fdb) {
702 err = mlx5_lag_create_single_fdb(ldev);
703 if (err)
704 mlx5_core_err(dev0, "Can't enable single FDB mode\n");
705 else
706 mlx5_core_info(dev0, "Operation mode is single FDB\n");
707 }
708
709 if (err) {
710 MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);
711 if (mlx5_cmd_exec_in(dev0, destroy_lag, in))
712 mlx5_core_err(dev0,
713 "Failed to deactivate RoCE LAG; driver restart required\n");
714 }
715 BLOCKING_INIT_NOTIFIER_HEAD(&dev0->priv.lag_nh);
716
717 return err;
718 }
719
mlx5_activate_lag(struct mlx5_lag * ldev,struct lag_tracker * tracker,enum mlx5_lag_mode mode,bool shared_fdb)720 int mlx5_activate_lag(struct mlx5_lag *ldev,
721 struct lag_tracker *tracker,
722 enum mlx5_lag_mode mode,
723 bool shared_fdb)
724 {
725 int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
726 bool roce_lag = mode == MLX5_LAG_MODE_ROCE;
727 struct mlx5_core_dev *dev0;
728 unsigned long flags = 0;
729 int err;
730
731 if (first_idx < 0)
732 return -EINVAL;
733
734 dev0 = ldev->pf[first_idx].dev;
735 err = mlx5_lag_set_flags(ldev, mode, tracker, shared_fdb, &flags);
736 if (err)
737 return err;
738
739 if (mode != MLX5_LAG_MODE_MPESW) {
740 mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ldev->v2p_map);
741 if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
742 err = mlx5_lag_port_sel_create(ldev, tracker->hash_type,
743 ldev->v2p_map);
744 if (err) {
745 mlx5_core_err(dev0,
746 "Failed to create LAG port selection(%d)\n",
747 err);
748 return err;
749 }
750 }
751 }
752
753 err = mlx5_create_lag(ldev, tracker, mode, flags);
754 if (err) {
755 if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags))
756 mlx5_lag_port_sel_destroy(ldev);
757 if (roce_lag)
758 mlx5_core_err(dev0,
759 "Failed to activate RoCE LAG\n");
760 else
761 mlx5_core_err(dev0,
762 "Failed to activate VF LAG\n"
763 "Make sure all VFs are unbound prior to VF LAG activation or deactivation\n");
764 return err;
765 }
766
767 if (tracker && tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP &&
768 !roce_lag)
769 mlx5_lag_drop_rule_setup(ldev, tracker);
770
771 ldev->mode = mode;
772 ldev->mode_flags = flags;
773 return 0;
774 }
775
mlx5_deactivate_lag(struct mlx5_lag * ldev)776 int mlx5_deactivate_lag(struct mlx5_lag *ldev)
777 {
778 int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
779 u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {};
780 bool roce_lag = __mlx5_lag_is_roce(ldev);
781 unsigned long flags = ldev->mode_flags;
782 struct mlx5_eswitch *master_esw;
783 struct mlx5_core_dev *dev0;
784 int err;
785 int i;
786
787 if (first_idx < 0)
788 return -EINVAL;
789
790 dev0 = ldev->pf[first_idx].dev;
791 master_esw = dev0->priv.eswitch;
792 ldev->mode = MLX5_LAG_MODE_NONE;
793 ldev->mode_flags = 0;
794 mlx5_lag_mp_reset(ldev);
795
796 if (test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags)) {
797 mlx5_ldev_for_each(i, first_idx + 1, ldev)
798 mlx5_eswitch_offloads_single_fdb_del_one(master_esw,
799 ldev->pf[i].dev->priv.eswitch);
800 clear_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags);
801 }
802
803 MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);
804 err = mlx5_cmd_exec_in(dev0, destroy_lag, in);
805 if (err) {
806 if (roce_lag) {
807 mlx5_core_err(dev0,
808 "Failed to deactivate RoCE LAG; driver restart required\n");
809 } else {
810 mlx5_core_err(dev0,
811 "Failed to deactivate VF LAG; driver restart required\n"
812 "Make sure all VFs are unbound prior to VF LAG activation or deactivation\n");
813 }
814 return err;
815 }
816
817 if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
818 mlx5_lag_port_sel_destroy(ldev);
819 ldev->buckets = 1;
820 }
821 if (mlx5_lag_has_drop_rule(ldev))
822 mlx5_lag_drop_rule_cleanup(ldev);
823
824 return 0;
825 }
826
mlx5_lag_check_prereq(struct mlx5_lag * ldev)827 bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
828 {
829 int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
830 #ifdef CONFIG_MLX5_ESWITCH
831 struct mlx5_core_dev *dev;
832 u8 mode;
833 #endif
834 bool roce_support;
835 int i;
836
837 if (first_idx < 0 || mlx5_lag_num_devs(ldev) != ldev->ports)
838 return false;
839
840 #ifdef CONFIG_MLX5_ESWITCH
841 mlx5_ldev_for_each(i, 0, ldev) {
842 dev = ldev->pf[i].dev;
843 if (mlx5_eswitch_num_vfs(dev->priv.eswitch) && !is_mdev_switchdev_mode(dev))
844 return false;
845 }
846
847 dev = ldev->pf[first_idx].dev;
848 mode = mlx5_eswitch_mode(dev);
849 mlx5_ldev_for_each(i, 0, ldev)
850 if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode)
851 return false;
852
853 #else
854 mlx5_ldev_for_each(i, 0, ldev)
855 if (mlx5_sriov_is_enabled(ldev->pf[i].dev))
856 return false;
857 #endif
858 roce_support = mlx5_get_roce_state(ldev->pf[first_idx].dev);
859 mlx5_ldev_for_each(i, first_idx + 1, ldev)
860 if (mlx5_get_roce_state(ldev->pf[i].dev) != roce_support)
861 return false;
862
863 return true;
864 }
865
mlx5_lag_add_devices(struct mlx5_lag * ldev)866 void mlx5_lag_add_devices(struct mlx5_lag *ldev)
867 {
868 int i;
869
870 mlx5_ldev_for_each(i, 0, ldev) {
871 if (ldev->pf[i].dev->priv.flags &
872 MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
873 continue;
874
875 ldev->pf[i].dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
876 mlx5_rescan_drivers_locked(ldev->pf[i].dev);
877 }
878 }
879
mlx5_lag_remove_devices(struct mlx5_lag * ldev)880 void mlx5_lag_remove_devices(struct mlx5_lag *ldev)
881 {
882 int i;
883
884 mlx5_ldev_for_each(i, 0, ldev) {
885 if (ldev->pf[i].dev->priv.flags &
886 MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
887 continue;
888
889 ldev->pf[i].dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
890 mlx5_rescan_drivers_locked(ldev->pf[i].dev);
891 }
892 }
893
mlx5_disable_lag(struct mlx5_lag * ldev)894 void mlx5_disable_lag(struct mlx5_lag *ldev)
895 {
896 bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags);
897 int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
898 struct mlx5_core_dev *dev0;
899 bool roce_lag;
900 int err;
901 int i;
902
903 if (idx < 0)
904 return;
905
906 dev0 = ldev->pf[idx].dev;
907 roce_lag = __mlx5_lag_is_roce(ldev);
908
909 if (shared_fdb) {
910 mlx5_lag_remove_devices(ldev);
911 } else if (roce_lag) {
912 if (!(dev0->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) {
913 dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
914 mlx5_rescan_drivers_locked(dev0);
915 }
916 mlx5_ldev_for_each(i, idx + 1, ldev)
917 mlx5_nic_vport_disable_roce(ldev->pf[i].dev);
918 }
919
920 err = mlx5_deactivate_lag(ldev);
921 if (err)
922 return;
923
924 if (shared_fdb || roce_lag)
925 mlx5_lag_add_devices(ldev);
926
927 if (shared_fdb)
928 mlx5_ldev_for_each(i, 0, ldev)
929 if (!(ldev->pf[i].dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV))
930 mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
931 }
932
mlx5_lag_shared_fdb_supported(struct mlx5_lag * ldev)933 bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev)
934 {
935 int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
936 struct mlx5_core_dev *dev;
937 int i;
938
939 if (idx < 0)
940 return false;
941
942 mlx5_ldev_for_each(i, idx + 1, ldev) {
943 dev = ldev->pf[i].dev;
944 if (is_mdev_switchdev_mode(dev) &&
945 mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) &&
946 MLX5_CAP_GEN(dev, lag_native_fdb_selection) &&
947 MLX5_CAP_ESW(dev, root_ft_on_other_esw) &&
948 mlx5_eswitch_get_npeers(dev->priv.eswitch) ==
949 MLX5_CAP_GEN(dev, num_lag_ports) - 1)
950 continue;
951 return false;
952 }
953
954 dev = ldev->pf[idx].dev;
955 if (is_mdev_switchdev_mode(dev) &&
956 mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) &&
957 mlx5_esw_offloads_devcom_is_ready(dev->priv.eswitch) &&
958 MLX5_CAP_ESW(dev, esw_shared_ingress_acl) &&
959 mlx5_eswitch_get_npeers(dev->priv.eswitch) == MLX5_CAP_GEN(dev, num_lag_ports) - 1)
960 return true;
961
962 return false;
963 }
964
mlx5_lag_is_roce_lag(struct mlx5_lag * ldev)965 static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev)
966 {
967 bool roce_lag = true;
968 int i;
969
970 mlx5_ldev_for_each(i, 0, ldev)
971 roce_lag = roce_lag && !mlx5_sriov_is_enabled(ldev->pf[i].dev);
972
973 #ifdef CONFIG_MLX5_ESWITCH
974 mlx5_ldev_for_each(i, 0, ldev)
975 roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev);
976 #endif
977
978 return roce_lag;
979 }
980
mlx5_lag_should_modify_lag(struct mlx5_lag * ldev,bool do_bond)981 static bool mlx5_lag_should_modify_lag(struct mlx5_lag *ldev, bool do_bond)
982 {
983 return do_bond && __mlx5_lag_is_active(ldev) &&
984 ldev->mode != MLX5_LAG_MODE_MPESW;
985 }
986
mlx5_lag_should_disable_lag(struct mlx5_lag * ldev,bool do_bond)987 static bool mlx5_lag_should_disable_lag(struct mlx5_lag *ldev, bool do_bond)
988 {
989 return !do_bond && __mlx5_lag_is_active(ldev) &&
990 ldev->mode != MLX5_LAG_MODE_MPESW;
991 }
992
mlx5_do_bond(struct mlx5_lag * ldev)993 static void mlx5_do_bond(struct mlx5_lag *ldev)
994 {
995 int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
996 struct lag_tracker tracker = { };
997 struct mlx5_core_dev *dev0;
998 struct net_device *ndev;
999 bool do_bond, roce_lag;
1000 int err;
1001 int i;
1002
1003 if (idx < 0)
1004 return;
1005
1006 dev0 = ldev->pf[idx].dev;
1007 if (!mlx5_lag_is_ready(ldev)) {
1008 do_bond = false;
1009 } else {
1010 /* VF LAG is in multipath mode, ignore bond change requests */
1011 if (mlx5_lag_is_multipath(dev0))
1012 return;
1013
1014 tracker = ldev->tracker;
1015
1016 do_bond = tracker.is_bonded && mlx5_lag_check_prereq(ldev);
1017 }
1018
1019 if (do_bond && !__mlx5_lag_is_active(ldev)) {
1020 bool shared_fdb = mlx5_lag_shared_fdb_supported(ldev);
1021
1022 roce_lag = mlx5_lag_is_roce_lag(ldev);
1023
1024 if (shared_fdb || roce_lag)
1025 mlx5_lag_remove_devices(ldev);
1026
1027 err = mlx5_activate_lag(ldev, &tracker,
1028 roce_lag ? MLX5_LAG_MODE_ROCE :
1029 MLX5_LAG_MODE_SRIOV,
1030 shared_fdb);
1031 if (err) {
1032 if (shared_fdb || roce_lag)
1033 mlx5_lag_add_devices(ldev);
1034 if (shared_fdb) {
1035 mlx5_ldev_for_each(i, 0, ldev)
1036 mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
1037 }
1038
1039 return;
1040 } else if (roce_lag) {
1041 dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
1042 mlx5_rescan_drivers_locked(dev0);
1043 mlx5_ldev_for_each(i, idx + 1, ldev) {
1044 if (mlx5_get_roce_state(ldev->pf[i].dev))
1045 mlx5_nic_vport_enable_roce(ldev->pf[i].dev);
1046 }
1047 } else if (shared_fdb) {
1048 int i;
1049
1050 dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
1051 mlx5_rescan_drivers_locked(dev0);
1052
1053 mlx5_ldev_for_each(i, 0, ldev) {
1054 err = mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
1055 if (err)
1056 break;
1057 }
1058
1059 if (err) {
1060 dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
1061 mlx5_rescan_drivers_locked(dev0);
1062 mlx5_deactivate_lag(ldev);
1063 mlx5_lag_add_devices(ldev);
1064 mlx5_ldev_for_each(i, 0, ldev)
1065 mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
1066 mlx5_core_err(dev0, "Failed to enable lag\n");
1067 return;
1068 }
1069 }
1070 if (tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
1071 ndev = mlx5_lag_active_backup_get_netdev(dev0);
1072 /** Only sriov and roce lag should have tracker->TX_type
1073 * set so no need to check the mode
1074 */
1075 blocking_notifier_call_chain(&dev0->priv.lag_nh,
1076 MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE,
1077 ndev);
1078 dev_put(ndev);
1079 }
1080 } else if (mlx5_lag_should_modify_lag(ldev, do_bond)) {
1081 mlx5_modify_lag(ldev, &tracker);
1082 } else if (mlx5_lag_should_disable_lag(ldev, do_bond)) {
1083 mlx5_disable_lag(ldev);
1084 }
1085 }
1086
1087 /* The last mdev to unregister will destroy the workqueue before removing the
1088 * devcom component, and as all the mdevs use the same devcom component we are
1089 * guaranteed that the devcom is valid while the calling work is running.
1090 */
mlx5_lag_get_devcom_comp(struct mlx5_lag * ldev)1091 struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev)
1092 {
1093 struct mlx5_devcom_comp_dev *devcom = NULL;
1094 int i;
1095
1096 mutex_lock(&ldev->lock);
1097 i = mlx5_get_next_ldev_func(ldev, 0);
1098 if (i < MLX5_MAX_PORTS)
1099 devcom = ldev->pf[i].dev->priv.hca_devcom_comp;
1100 mutex_unlock(&ldev->lock);
1101 return devcom;
1102 }
1103
mlx5_queue_bond_work(struct mlx5_lag * ldev,unsigned long delay)1104 static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay)
1105 {
1106 queue_delayed_work(ldev->wq, &ldev->bond_work, delay);
1107 }
1108
mlx5_do_bond_work(struct work_struct * work)1109 static void mlx5_do_bond_work(struct work_struct *work)
1110 {
1111 struct delayed_work *delayed_work = to_delayed_work(work);
1112 struct mlx5_lag *ldev = container_of(delayed_work, struct mlx5_lag,
1113 bond_work);
1114 struct mlx5_devcom_comp_dev *devcom;
1115 int status;
1116
1117 devcom = mlx5_lag_get_devcom_comp(ldev);
1118 if (!devcom)
1119 return;
1120
1121 status = mlx5_devcom_comp_trylock(devcom);
1122 if (!status) {
1123 mlx5_queue_bond_work(ldev, HZ);
1124 return;
1125 }
1126
1127 mutex_lock(&ldev->lock);
1128 if (ldev->mode_changes_in_progress) {
1129 mutex_unlock(&ldev->lock);
1130 mlx5_devcom_comp_unlock(devcom);
1131 mlx5_queue_bond_work(ldev, HZ);
1132 return;
1133 }
1134
1135 mlx5_do_bond(ldev);
1136 mutex_unlock(&ldev->lock);
1137 mlx5_devcom_comp_unlock(devcom);
1138 }
1139
mlx5_handle_changeupper_event(struct mlx5_lag * ldev,struct lag_tracker * tracker,struct netdev_notifier_changeupper_info * info)1140 static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
1141 struct lag_tracker *tracker,
1142 struct netdev_notifier_changeupper_info *info)
1143 {
1144 struct net_device *upper = info->upper_dev, *ndev_tmp;
1145 struct netdev_lag_upper_info *lag_upper_info = NULL;
1146 bool is_bonded, is_in_lag, mode_supported;
1147 bool has_inactive = 0;
1148 struct slave *slave;
1149 u8 bond_status = 0;
1150 int num_slaves = 0;
1151 int changed = 0;
1152 int i, idx = -1;
1153
1154 if (!netif_is_lag_master(upper))
1155 return 0;
1156
1157 if (info->linking)
1158 lag_upper_info = info->upper_info;
1159
1160 /* The event may still be of interest if the slave does not belong to
1161 * us, but is enslaved to a master which has one or more of our netdevs
1162 * as slaves (e.g., if a new slave is added to a master that bonds two
1163 * of our netdevs, we should unbond).
1164 */
1165 rcu_read_lock();
1166 for_each_netdev_in_bond_rcu(upper, ndev_tmp) {
1167 mlx5_ldev_for_each(i, 0, ldev) {
1168 if (ldev->pf[i].netdev == ndev_tmp) {
1169 idx++;
1170 break;
1171 }
1172 }
1173 if (i < MLX5_MAX_PORTS) {
1174 slave = bond_slave_get_rcu(ndev_tmp);
1175 if (slave)
1176 has_inactive |= bond_is_slave_inactive(slave);
1177 bond_status |= (1 << idx);
1178 }
1179
1180 num_slaves++;
1181 }
1182 rcu_read_unlock();
1183
1184 /* None of this lagdev's netdevs are slaves of this master. */
1185 if (!(bond_status & GENMASK(ldev->ports - 1, 0)))
1186 return 0;
1187
1188 if (lag_upper_info) {
1189 tracker->tx_type = lag_upper_info->tx_type;
1190 tracker->hash_type = lag_upper_info->hash_type;
1191 }
1192
1193 tracker->has_inactive = has_inactive;
1194 /* Determine bonding status:
1195 * A device is considered bonded if both its physical ports are slaves
1196 * of the same lag master, and only them.
1197 */
1198 is_in_lag = num_slaves == ldev->ports &&
1199 bond_status == GENMASK(ldev->ports - 1, 0);
1200
1201 /* Lag mode must be activebackup or hash. */
1202 mode_supported = tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP ||
1203 tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH;
1204
1205 is_bonded = is_in_lag && mode_supported;
1206 if (tracker->is_bonded != is_bonded) {
1207 tracker->is_bonded = is_bonded;
1208 changed = 1;
1209 }
1210
1211 if (!is_in_lag)
1212 return changed;
1213
1214 if (!mlx5_lag_is_ready(ldev))
1215 NL_SET_ERR_MSG_MOD(info->info.extack,
1216 "Can't activate LAG offload, PF is configured with more than 64 VFs");
1217 else if (!mode_supported)
1218 NL_SET_ERR_MSG_MOD(info->info.extack,
1219 "Can't activate LAG offload, TX type isn't supported");
1220
1221 return changed;
1222 }
1223
mlx5_handle_changelowerstate_event(struct mlx5_lag * ldev,struct lag_tracker * tracker,struct net_device * ndev,struct netdev_notifier_changelowerstate_info * info)1224 static int mlx5_handle_changelowerstate_event(struct mlx5_lag *ldev,
1225 struct lag_tracker *tracker,
1226 struct net_device *ndev,
1227 struct netdev_notifier_changelowerstate_info *info)
1228 {
1229 struct netdev_lag_lower_state_info *lag_lower_info;
1230 int idx;
1231
1232 if (!netif_is_lag_port(ndev))
1233 return 0;
1234
1235 idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev);
1236 if (idx < 0)
1237 return 0;
1238
1239 /* This information is used to determine virtual to physical
1240 * port mapping.
1241 */
1242 lag_lower_info = info->lower_state_info;
1243 if (!lag_lower_info)
1244 return 0;
1245
1246 tracker->netdev_state[idx] = *lag_lower_info;
1247
1248 return 1;
1249 }
1250
mlx5_handle_changeinfodata_event(struct mlx5_lag * ldev,struct lag_tracker * tracker,struct net_device * ndev)1251 static int mlx5_handle_changeinfodata_event(struct mlx5_lag *ldev,
1252 struct lag_tracker *tracker,
1253 struct net_device *ndev)
1254 {
1255 struct net_device *ndev_tmp;
1256 struct slave *slave;
1257 bool has_inactive = 0;
1258 int idx;
1259
1260 if (!netif_is_lag_master(ndev))
1261 return 0;
1262
1263 rcu_read_lock();
1264 for_each_netdev_in_bond_rcu(ndev, ndev_tmp) {
1265 idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev_tmp);
1266 if (idx < 0)
1267 continue;
1268
1269 slave = bond_slave_get_rcu(ndev_tmp);
1270 if (slave)
1271 has_inactive |= bond_is_slave_inactive(slave);
1272 }
1273 rcu_read_unlock();
1274
1275 if (tracker->has_inactive == has_inactive)
1276 return 0;
1277
1278 tracker->has_inactive = has_inactive;
1279
1280 return 1;
1281 }
1282
1283 /* this handler is always registered to netdev events */
mlx5_lag_netdev_event(struct notifier_block * this,unsigned long event,void * ptr)1284 static int mlx5_lag_netdev_event(struct notifier_block *this,
1285 unsigned long event, void *ptr)
1286 {
1287 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
1288 struct lag_tracker tracker;
1289 struct mlx5_lag *ldev;
1290 int changed = 0;
1291
1292 if (event != NETDEV_CHANGEUPPER &&
1293 event != NETDEV_CHANGELOWERSTATE &&
1294 event != NETDEV_CHANGEINFODATA)
1295 return NOTIFY_DONE;
1296
1297 ldev = container_of(this, struct mlx5_lag, nb);
1298
1299 tracker = ldev->tracker;
1300
1301 switch (event) {
1302 case NETDEV_CHANGEUPPER:
1303 changed = mlx5_handle_changeupper_event(ldev, &tracker, ptr);
1304 break;
1305 case NETDEV_CHANGELOWERSTATE:
1306 changed = mlx5_handle_changelowerstate_event(ldev, &tracker,
1307 ndev, ptr);
1308 break;
1309 case NETDEV_CHANGEINFODATA:
1310 changed = mlx5_handle_changeinfodata_event(ldev, &tracker, ndev);
1311 break;
1312 }
1313
1314 ldev->tracker = tracker;
1315
1316 if (changed)
1317 mlx5_queue_bond_work(ldev, 0);
1318
1319 return NOTIFY_DONE;
1320 }
1321
mlx5_ldev_add_netdev(struct mlx5_lag * ldev,struct mlx5_core_dev * dev,struct net_device * netdev)1322 static void mlx5_ldev_add_netdev(struct mlx5_lag *ldev,
1323 struct mlx5_core_dev *dev,
1324 struct net_device *netdev)
1325 {
1326 unsigned int fn = mlx5_get_dev_index(dev);
1327 unsigned long flags;
1328
1329 spin_lock_irqsave(&lag_lock, flags);
1330 ldev->pf[fn].netdev = netdev;
1331 ldev->tracker.netdev_state[fn].link_up = 0;
1332 ldev->tracker.netdev_state[fn].tx_enabled = 0;
1333 spin_unlock_irqrestore(&lag_lock, flags);
1334 }
1335
mlx5_ldev_remove_netdev(struct mlx5_lag * ldev,struct net_device * netdev)1336 static void mlx5_ldev_remove_netdev(struct mlx5_lag *ldev,
1337 struct net_device *netdev)
1338 {
1339 unsigned long flags;
1340 int i;
1341
1342 spin_lock_irqsave(&lag_lock, flags);
1343 mlx5_ldev_for_each(i, 0, ldev) {
1344 if (ldev->pf[i].netdev == netdev) {
1345 ldev->pf[i].netdev = NULL;
1346 break;
1347 }
1348 }
1349 spin_unlock_irqrestore(&lag_lock, flags);
1350 }
1351
mlx5_ldev_add_mdev(struct mlx5_lag * ldev,struct mlx5_core_dev * dev)1352 static void mlx5_ldev_add_mdev(struct mlx5_lag *ldev,
1353 struct mlx5_core_dev *dev)
1354 {
1355 unsigned int fn = mlx5_get_dev_index(dev);
1356
1357 ldev->pf[fn].dev = dev;
1358 dev->priv.lag = ldev;
1359 }
1360
mlx5_ldev_remove_mdev(struct mlx5_lag * ldev,struct mlx5_core_dev * dev)1361 static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev,
1362 struct mlx5_core_dev *dev)
1363 {
1364 int fn;
1365
1366 fn = mlx5_get_dev_index(dev);
1367 if (ldev->pf[fn].dev != dev)
1368 return;
1369
1370 ldev->pf[fn].dev = NULL;
1371 dev->priv.lag = NULL;
1372 }
1373
1374 /* Must be called with HCA devcom component lock held */
__mlx5_lag_dev_add_mdev(struct mlx5_core_dev * dev)1375 static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)
1376 {
1377 struct mlx5_devcom_comp_dev *pos = NULL;
1378 struct mlx5_lag *ldev = NULL;
1379 struct mlx5_core_dev *tmp_dev;
1380
1381 tmp_dev = mlx5_devcom_get_next_peer_data(dev->priv.hca_devcom_comp, &pos);
1382 if (tmp_dev)
1383 ldev = mlx5_lag_dev(tmp_dev);
1384
1385 if (!ldev) {
1386 ldev = mlx5_lag_dev_alloc(dev);
1387 if (!ldev) {
1388 mlx5_core_err(dev, "Failed to alloc lag dev\n");
1389 return 0;
1390 }
1391 mlx5_ldev_add_mdev(ldev, dev);
1392 return 0;
1393 }
1394
1395 mutex_lock(&ldev->lock);
1396 if (ldev->mode_changes_in_progress) {
1397 mutex_unlock(&ldev->lock);
1398 return -EAGAIN;
1399 }
1400 mlx5_ldev_get(ldev);
1401 mlx5_ldev_add_mdev(ldev, dev);
1402 mutex_unlock(&ldev->lock);
1403
1404 return 0;
1405 }
1406
mlx5_lag_remove_mdev(struct mlx5_core_dev * dev)1407 void mlx5_lag_remove_mdev(struct mlx5_core_dev *dev)
1408 {
1409 struct mlx5_lag *ldev;
1410
1411 ldev = mlx5_lag_dev(dev);
1412 if (!ldev)
1413 return;
1414
1415 /* mdev is being removed, might as well remove debugfs
1416 * as early as possible.
1417 */
1418 mlx5_ldev_remove_debugfs(dev->priv.dbg.lag_debugfs);
1419 recheck:
1420 mutex_lock(&ldev->lock);
1421 if (ldev->mode_changes_in_progress) {
1422 mutex_unlock(&ldev->lock);
1423 msleep(100);
1424 goto recheck;
1425 }
1426 mlx5_ldev_remove_mdev(ldev, dev);
1427 mutex_unlock(&ldev->lock);
1428 mlx5_ldev_put(ldev);
1429 }
1430
mlx5_lag_add_mdev(struct mlx5_core_dev * dev)1431 void mlx5_lag_add_mdev(struct mlx5_core_dev *dev)
1432 {
1433 int err;
1434
1435 if (!mlx5_lag_is_supported(dev))
1436 return;
1437
1438 if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp))
1439 return;
1440
1441 recheck:
1442 mlx5_devcom_comp_lock(dev->priv.hca_devcom_comp);
1443 err = __mlx5_lag_dev_add_mdev(dev);
1444 mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp);
1445
1446 if (err) {
1447 msleep(100);
1448 goto recheck;
1449 }
1450 mlx5_ldev_add_debugfs(dev);
1451 }
1452
mlx5_lag_remove_netdev(struct mlx5_core_dev * dev,struct net_device * netdev)1453 void mlx5_lag_remove_netdev(struct mlx5_core_dev *dev,
1454 struct net_device *netdev)
1455 {
1456 struct mlx5_lag *ldev;
1457 bool lag_is_active;
1458
1459 ldev = mlx5_lag_dev(dev);
1460 if (!ldev)
1461 return;
1462
1463 mutex_lock(&ldev->lock);
1464 mlx5_ldev_remove_netdev(ldev, netdev);
1465 clear_bit(MLX5_LAG_FLAG_NDEVS_READY, &ldev->state_flags);
1466
1467 lag_is_active = __mlx5_lag_is_active(ldev);
1468 mutex_unlock(&ldev->lock);
1469
1470 if (lag_is_active)
1471 mlx5_queue_bond_work(ldev, 0);
1472 }
1473
mlx5_lag_add_netdev(struct mlx5_core_dev * dev,struct net_device * netdev)1474 void mlx5_lag_add_netdev(struct mlx5_core_dev *dev,
1475 struct net_device *netdev)
1476 {
1477 struct mlx5_lag *ldev;
1478 int num = 0;
1479
1480 ldev = mlx5_lag_dev(dev);
1481 if (!ldev)
1482 return;
1483
1484 mutex_lock(&ldev->lock);
1485 mlx5_ldev_add_netdev(ldev, dev, netdev);
1486 num = mlx5_lag_num_netdevs(ldev);
1487 if (num >= ldev->ports)
1488 set_bit(MLX5_LAG_FLAG_NDEVS_READY, &ldev->state_flags);
1489 mutex_unlock(&ldev->lock);
1490 mlx5_queue_bond_work(ldev, 0);
1491 }
1492
mlx5_get_pre_ldev_func(struct mlx5_lag * ldev,int start_idx,int end_idx)1493 int mlx5_get_pre_ldev_func(struct mlx5_lag *ldev, int start_idx, int end_idx)
1494 {
1495 int i;
1496
1497 for (i = start_idx; i >= end_idx; i--)
1498 if (ldev->pf[i].dev)
1499 return i;
1500 return -1;
1501 }
1502
mlx5_get_next_ldev_func(struct mlx5_lag * ldev,int start_idx)1503 int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx)
1504 {
1505 int i;
1506
1507 for (i = start_idx; i < MLX5_MAX_PORTS; i++)
1508 if (ldev->pf[i].dev)
1509 return i;
1510 return MLX5_MAX_PORTS;
1511 }
1512
mlx5_lag_is_roce(struct mlx5_core_dev * dev)1513 bool mlx5_lag_is_roce(struct mlx5_core_dev *dev)
1514 {
1515 struct mlx5_lag *ldev;
1516 unsigned long flags;
1517 bool res;
1518
1519 spin_lock_irqsave(&lag_lock, flags);
1520 ldev = mlx5_lag_dev(dev);
1521 res = ldev && __mlx5_lag_is_roce(ldev);
1522 spin_unlock_irqrestore(&lag_lock, flags);
1523
1524 return res;
1525 }
1526 EXPORT_SYMBOL(mlx5_lag_is_roce);
1527
mlx5_lag_is_active(struct mlx5_core_dev * dev)1528 bool mlx5_lag_is_active(struct mlx5_core_dev *dev)
1529 {
1530 struct mlx5_lag *ldev;
1531 unsigned long flags;
1532 bool res;
1533
1534 spin_lock_irqsave(&lag_lock, flags);
1535 ldev = mlx5_lag_dev(dev);
1536 res = ldev && __mlx5_lag_is_active(ldev);
1537 spin_unlock_irqrestore(&lag_lock, flags);
1538
1539 return res;
1540 }
1541 EXPORT_SYMBOL(mlx5_lag_is_active);
1542
mlx5_lag_mode_is_hash(struct mlx5_core_dev * dev)1543 bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev)
1544 {
1545 struct mlx5_lag *ldev;
1546 unsigned long flags;
1547 bool res = 0;
1548
1549 spin_lock_irqsave(&lag_lock, flags);
1550 ldev = mlx5_lag_dev(dev);
1551 if (ldev)
1552 res = test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags);
1553 spin_unlock_irqrestore(&lag_lock, flags);
1554
1555 return res;
1556 }
1557 EXPORT_SYMBOL(mlx5_lag_mode_is_hash);
1558
mlx5_lag_is_master(struct mlx5_core_dev * dev)1559 bool mlx5_lag_is_master(struct mlx5_core_dev *dev)
1560 {
1561 struct mlx5_lag *ldev;
1562 unsigned long flags;
1563 bool res = false;
1564 int idx;
1565
1566 spin_lock_irqsave(&lag_lock, flags);
1567 ldev = mlx5_lag_dev(dev);
1568 idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
1569 res = ldev && __mlx5_lag_is_active(ldev) && idx >= 0 && dev == ldev->pf[idx].dev;
1570 spin_unlock_irqrestore(&lag_lock, flags);
1571
1572 return res;
1573 }
1574 EXPORT_SYMBOL(mlx5_lag_is_master);
1575
mlx5_lag_is_sriov(struct mlx5_core_dev * dev)1576 bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev)
1577 {
1578 struct mlx5_lag *ldev;
1579 unsigned long flags;
1580 bool res;
1581
1582 spin_lock_irqsave(&lag_lock, flags);
1583 ldev = mlx5_lag_dev(dev);
1584 res = ldev && __mlx5_lag_is_sriov(ldev);
1585 spin_unlock_irqrestore(&lag_lock, flags);
1586
1587 return res;
1588 }
1589 EXPORT_SYMBOL(mlx5_lag_is_sriov);
1590
mlx5_lag_is_shared_fdb(struct mlx5_core_dev * dev)1591 bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev)
1592 {
1593 struct mlx5_lag *ldev;
1594 unsigned long flags;
1595 bool res;
1596
1597 spin_lock_irqsave(&lag_lock, flags);
1598 ldev = mlx5_lag_dev(dev);
1599 res = ldev && test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags);
1600 spin_unlock_irqrestore(&lag_lock, flags);
1601
1602 return res;
1603 }
1604 EXPORT_SYMBOL(mlx5_lag_is_shared_fdb);
1605
mlx5_lag_disable_change(struct mlx5_core_dev * dev)1606 void mlx5_lag_disable_change(struct mlx5_core_dev *dev)
1607 {
1608 struct mlx5_lag *ldev;
1609
1610 ldev = mlx5_lag_dev(dev);
1611 if (!ldev)
1612 return;
1613
1614 mlx5_devcom_comp_lock(dev->priv.hca_devcom_comp);
1615 mutex_lock(&ldev->lock);
1616
1617 ldev->mode_changes_in_progress++;
1618 if (__mlx5_lag_is_active(ldev))
1619 mlx5_disable_lag(ldev);
1620
1621 mutex_unlock(&ldev->lock);
1622 mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp);
1623 }
1624
mlx5_lag_enable_change(struct mlx5_core_dev * dev)1625 void mlx5_lag_enable_change(struct mlx5_core_dev *dev)
1626 {
1627 struct mlx5_lag *ldev;
1628
1629 ldev = mlx5_lag_dev(dev);
1630 if (!ldev)
1631 return;
1632
1633 mutex_lock(&ldev->lock);
1634 ldev->mode_changes_in_progress--;
1635 mutex_unlock(&ldev->lock);
1636 mlx5_queue_bond_work(ldev, 0);
1637 }
1638
mlx5_lag_get_slave_port(struct mlx5_core_dev * dev,struct net_device * slave)1639 u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev,
1640 struct net_device *slave)
1641 {
1642 struct mlx5_lag *ldev;
1643 unsigned long flags;
1644 u8 port = 0;
1645 int i;
1646
1647 spin_lock_irqsave(&lag_lock, flags);
1648 ldev = mlx5_lag_dev(dev);
1649 if (!(ldev && __mlx5_lag_is_roce(ldev)))
1650 goto unlock;
1651
1652 mlx5_ldev_for_each(i, 0, ldev) {
1653 if (ldev->pf[i].netdev == slave) {
1654 port = i;
1655 break;
1656 }
1657 }
1658
1659 port = ldev->v2p_map[port * ldev->buckets];
1660
1661 unlock:
1662 spin_unlock_irqrestore(&lag_lock, flags);
1663 return port;
1664 }
1665 EXPORT_SYMBOL(mlx5_lag_get_slave_port);
1666
mlx5_lag_get_num_ports(struct mlx5_core_dev * dev)1667 u8 mlx5_lag_get_num_ports(struct mlx5_core_dev *dev)
1668 {
1669 struct mlx5_lag *ldev;
1670
1671 ldev = mlx5_lag_dev(dev);
1672 if (!ldev)
1673 return 0;
1674
1675 return ldev->ports;
1676 }
1677 EXPORT_SYMBOL(mlx5_lag_get_num_ports);
1678
mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev * dev,int * i)1679 struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int *i)
1680 {
1681 struct mlx5_core_dev *peer_dev = NULL;
1682 struct mlx5_lag *ldev;
1683 unsigned long flags;
1684 int idx;
1685
1686 spin_lock_irqsave(&lag_lock, flags);
1687 ldev = mlx5_lag_dev(dev);
1688 if (!ldev)
1689 goto unlock;
1690
1691 if (*i == MLX5_MAX_PORTS)
1692 goto unlock;
1693 mlx5_ldev_for_each(idx, *i, ldev)
1694 if (ldev->pf[idx].dev != dev)
1695 break;
1696
1697 if (idx == MLX5_MAX_PORTS) {
1698 *i = idx;
1699 goto unlock;
1700 }
1701 *i = idx + 1;
1702
1703 peer_dev = ldev->pf[idx].dev;
1704
1705 unlock:
1706 spin_unlock_irqrestore(&lag_lock, flags);
1707 return peer_dev;
1708 }
1709 EXPORT_SYMBOL(mlx5_lag_get_next_peer_mdev);
1710
mlx5_lag_query_cong_counters(struct mlx5_core_dev * dev,u64 * values,int num_counters,size_t * offsets)1711 int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
1712 u64 *values,
1713 int num_counters,
1714 size_t *offsets)
1715 {
1716 int outlen = MLX5_ST_SZ_BYTES(query_cong_statistics_out);
1717 struct mlx5_core_dev **mdev;
1718 int ret = 0, i, j, idx = 0;
1719 struct mlx5_lag *ldev;
1720 unsigned long flags;
1721 int num_ports;
1722 void *out;
1723
1724 out = kvzalloc(outlen, GFP_KERNEL);
1725 if (!out)
1726 return -ENOMEM;
1727
1728 mdev = kvzalloc(sizeof(mdev[0]) * MLX5_MAX_PORTS, GFP_KERNEL);
1729 if (!mdev) {
1730 ret = -ENOMEM;
1731 goto free_out;
1732 }
1733
1734 memset(values, 0, sizeof(*values) * num_counters);
1735
1736 spin_lock_irqsave(&lag_lock, flags);
1737 ldev = mlx5_lag_dev(dev);
1738 if (ldev && __mlx5_lag_is_active(ldev)) {
1739 num_ports = ldev->ports;
1740 mlx5_ldev_for_each(i, 0, ldev)
1741 mdev[idx++] = ldev->pf[i].dev;
1742 } else {
1743 num_ports = 1;
1744 mdev[MLX5_LAG_P1] = dev;
1745 }
1746 spin_unlock_irqrestore(&lag_lock, flags);
1747
1748 for (i = 0; i < num_ports; ++i) {
1749 u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = {};
1750
1751 MLX5_SET(query_cong_statistics_in, in, opcode,
1752 MLX5_CMD_OP_QUERY_CONG_STATISTICS);
1753 ret = mlx5_cmd_exec_inout(mdev[i], query_cong_statistics, in,
1754 out);
1755 if (ret)
1756 goto free_mdev;
1757
1758 for (j = 0; j < num_counters; ++j)
1759 values[j] += be64_to_cpup((__be64 *)(out + offsets[j]));
1760 }
1761
1762 free_mdev:
1763 kvfree(mdev);
1764 free_out:
1765 kvfree(out);
1766 return ret;
1767 }
1768 EXPORT_SYMBOL(mlx5_lag_query_cong_counters);
1769