xref: /linux/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c (revision 1a9239bb4253f9076b5b4b2a1a4e8d7defd77a95)
1 /*
2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <linux/netdevice.h>
34 #include <net/bonding.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/eswitch.h>
37 #include <linux/mlx5/vport.h>
38 #include "lib/devcom.h"
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/acl/ofld.h"
42 #include "lag.h"
43 #include "mp.h"
44 #include "mpesw.h"
45 
46 
47 /* General purpose, use for short periods of time.
48  * Beware of lock dependencies (preferably, no locks should be acquired
49  * under it).
50  */
51 static DEFINE_SPINLOCK(lag_lock);
52 
get_port_sel_mode(enum mlx5_lag_mode mode,unsigned long flags)53 static int get_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags)
54 {
55 	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags))
56 		return MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT;
57 
58 	if (mode == MLX5_LAG_MODE_MPESW)
59 		return MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_MPESW;
60 
61 	return MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY;
62 }
63 
lag_active_port_bits(struct mlx5_lag * ldev)64 static u8 lag_active_port_bits(struct mlx5_lag *ldev)
65 {
66 	u8 enabled_ports[MLX5_MAX_PORTS] = {};
67 	u8 active_port = 0;
68 	int num_enabled;
69 	int idx;
70 
71 	mlx5_infer_tx_enabled(&ldev->tracker, ldev, enabled_ports,
72 			      &num_enabled);
73 	for (idx = 0; idx < num_enabled; idx++)
74 		active_port |= BIT_MASK(enabled_ports[idx]);
75 
76 	return active_port;
77 }
78 
mlx5_cmd_create_lag(struct mlx5_core_dev * dev,struct mlx5_lag * ldev,int mode,unsigned long flags)79 static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev,
80 			       int mode, unsigned long flags)
81 {
82 	bool fdb_sel_mode = test_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE,
83 				     &flags);
84 	int port_sel_mode = get_port_sel_mode(mode, flags);
85 	u32 in[MLX5_ST_SZ_DW(create_lag_in)] = {};
86 	u8 *ports = ldev->v2p_map;
87 	int idx0, idx1;
88 	void *lag_ctx;
89 
90 	lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx);
91 	MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG);
92 	MLX5_SET(lagc, lag_ctx, fdb_selection_mode, fdb_sel_mode);
93 	idx0 = mlx5_lag_get_dev_index_by_seq(ldev, 0);
94 	idx1 = mlx5_lag_get_dev_index_by_seq(ldev, 1);
95 
96 	if (idx0 < 0 || idx1 < 0)
97 		return -EINVAL;
98 
99 	switch (port_sel_mode) {
100 	case MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY:
101 		MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, ports[idx0]);
102 		MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, ports[idx1]);
103 		break;
104 	case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT:
105 		if (!MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass))
106 			break;
107 
108 		MLX5_SET(lagc, lag_ctx, active_port,
109 			 lag_active_port_bits(mlx5_lag_dev(dev)));
110 		break;
111 	default:
112 		break;
113 	}
114 	MLX5_SET(lagc, lag_ctx, port_select_mode, port_sel_mode);
115 
116 	return mlx5_cmd_exec_in(dev, create_lag, in);
117 }
118 
mlx5_cmd_modify_lag(struct mlx5_core_dev * dev,struct mlx5_lag * ldev,u8 * ports)119 static int mlx5_cmd_modify_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev,
120 			       u8 *ports)
121 {
122 	u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {};
123 	void *lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
124 	int idx0, idx1;
125 
126 	idx0 = mlx5_lag_get_dev_index_by_seq(ldev, 0);
127 	idx1 = mlx5_lag_get_dev_index_by_seq(ldev, 1);
128 	if (idx0 < 0 || idx1 < 0)
129 		return -EINVAL;
130 
131 	MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
132 	MLX5_SET(modify_lag_in, in, field_select, 0x1);
133 
134 	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, ports[idx0]);
135 	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, ports[idx1]);
136 
137 	return mlx5_cmd_exec_in(dev, modify_lag, in);
138 }
139 
mlx5_cmd_create_vport_lag(struct mlx5_core_dev * dev)140 int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev)
141 {
142 	u32 in[MLX5_ST_SZ_DW(create_vport_lag_in)] = {};
143 
144 	MLX5_SET(create_vport_lag_in, in, opcode, MLX5_CMD_OP_CREATE_VPORT_LAG);
145 
146 	return mlx5_cmd_exec_in(dev, create_vport_lag, in);
147 }
148 EXPORT_SYMBOL(mlx5_cmd_create_vport_lag);
149 
mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev * dev)150 int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev)
151 {
152 	u32 in[MLX5_ST_SZ_DW(destroy_vport_lag_in)] = {};
153 
154 	MLX5_SET(destroy_vport_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_VPORT_LAG);
155 
156 	return mlx5_cmd_exec_in(dev, destroy_vport_lag, in);
157 }
158 EXPORT_SYMBOL(mlx5_cmd_destroy_vport_lag);
159 
mlx5_infer_tx_disabled(struct lag_tracker * tracker,struct mlx5_lag * ldev,u8 * ports,int * num_disabled)160 static void mlx5_infer_tx_disabled(struct lag_tracker *tracker, struct mlx5_lag *ldev,
161 				   u8 *ports, int *num_disabled)
162 {
163 	int i;
164 
165 	*num_disabled = 0;
166 	mlx5_ldev_for_each(i, 0, ldev)
167 		if (!tracker->netdev_state[i].tx_enabled ||
168 		    !tracker->netdev_state[i].link_up)
169 			ports[(*num_disabled)++] = i;
170 }
171 
mlx5_infer_tx_enabled(struct lag_tracker * tracker,struct mlx5_lag * ldev,u8 * ports,int * num_enabled)172 void mlx5_infer_tx_enabled(struct lag_tracker *tracker, struct mlx5_lag *ldev,
173 			   u8 *ports, int *num_enabled)
174 {
175 	int i;
176 
177 	*num_enabled = 0;
178 	mlx5_ldev_for_each(i, 0, ldev)
179 		if (tracker->netdev_state[i].tx_enabled &&
180 		    tracker->netdev_state[i].link_up)
181 			ports[(*num_enabled)++] = i;
182 
183 	if (*num_enabled == 0)
184 		mlx5_infer_tx_disabled(tracker, ldev, ports, num_enabled);
185 }
186 
mlx5_lag_print_mapping(struct mlx5_core_dev * dev,struct mlx5_lag * ldev,struct lag_tracker * tracker,unsigned long flags)187 static void mlx5_lag_print_mapping(struct mlx5_core_dev *dev,
188 				   struct mlx5_lag *ldev,
189 				   struct lag_tracker *tracker,
190 				   unsigned long flags)
191 {
192 	char buf[MLX5_MAX_PORTS * 10 + 1] = {};
193 	u8 enabled_ports[MLX5_MAX_PORTS] = {};
194 	int written = 0;
195 	int num_enabled;
196 	int idx;
197 	int err;
198 	int i;
199 	int j;
200 
201 	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
202 		mlx5_infer_tx_enabled(tracker, ldev, enabled_ports,
203 				      &num_enabled);
204 		for (i = 0; i < num_enabled; i++) {
205 			err = scnprintf(buf + written, 4, "%d, ", enabled_ports[i] + 1);
206 			if (err != 3)
207 				return;
208 			written += err;
209 		}
210 		buf[written - 2] = 0;
211 		mlx5_core_info(dev, "lag map active ports: %s\n", buf);
212 	} else {
213 		mlx5_ldev_for_each(i, 0, ldev) {
214 			for (j  = 0; j < ldev->buckets; j++) {
215 				idx = i * ldev->buckets + j;
216 				err = scnprintf(buf + written, 10,
217 						" port %d:%d", i + 1, ldev->v2p_map[idx]);
218 				if (err != 9)
219 					return;
220 				written += err;
221 			}
222 		}
223 		mlx5_core_info(dev, "lag map:%s\n", buf);
224 	}
225 }
226 
227 static int mlx5_lag_netdev_event(struct notifier_block *this,
228 				 unsigned long event, void *ptr);
229 static void mlx5_do_bond_work(struct work_struct *work);
230 
mlx5_ldev_free(struct kref * ref)231 static void mlx5_ldev_free(struct kref *ref)
232 {
233 	struct mlx5_lag *ldev = container_of(ref, struct mlx5_lag, ref);
234 
235 	if (ldev->nb.notifier_call)
236 		unregister_netdevice_notifier_net(&init_net, &ldev->nb);
237 	mlx5_lag_mp_cleanup(ldev);
238 	cancel_delayed_work_sync(&ldev->bond_work);
239 	destroy_workqueue(ldev->wq);
240 	mutex_destroy(&ldev->lock);
241 	kfree(ldev);
242 }
243 
mlx5_ldev_put(struct mlx5_lag * ldev)244 static void mlx5_ldev_put(struct mlx5_lag *ldev)
245 {
246 	kref_put(&ldev->ref, mlx5_ldev_free);
247 }
248 
mlx5_ldev_get(struct mlx5_lag * ldev)249 static void mlx5_ldev_get(struct mlx5_lag *ldev)
250 {
251 	kref_get(&ldev->ref);
252 }
253 
mlx5_lag_dev_alloc(struct mlx5_core_dev * dev)254 static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev)
255 {
256 	struct mlx5_lag *ldev;
257 	int err;
258 
259 	ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
260 	if (!ldev)
261 		return NULL;
262 
263 	ldev->wq = create_singlethread_workqueue("mlx5_lag");
264 	if (!ldev->wq) {
265 		kfree(ldev);
266 		return NULL;
267 	}
268 
269 	kref_init(&ldev->ref);
270 	mutex_init(&ldev->lock);
271 	INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work);
272 
273 	ldev->nb.notifier_call = mlx5_lag_netdev_event;
274 	if (register_netdevice_notifier_net(&init_net, &ldev->nb)) {
275 		ldev->nb.notifier_call = NULL;
276 		mlx5_core_err(dev, "Failed to register LAG netdev notifier\n");
277 	}
278 	ldev->mode = MLX5_LAG_MODE_NONE;
279 
280 	err = mlx5_lag_mp_init(ldev);
281 	if (err)
282 		mlx5_core_err(dev, "Failed to init multipath lag err=%d\n",
283 			      err);
284 
285 	ldev->ports = MLX5_CAP_GEN(dev, num_lag_ports);
286 	ldev->buckets = 1;
287 
288 	return ldev;
289 }
290 
mlx5_lag_dev_get_netdev_idx(struct mlx5_lag * ldev,struct net_device * ndev)291 int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
292 				struct net_device *ndev)
293 {
294 	int i;
295 
296 	mlx5_ldev_for_each(i, 0, ldev)
297 		if (ldev->pf[i].netdev == ndev)
298 			return i;
299 
300 	return -ENOENT;
301 }
302 
mlx5_lag_get_dev_index_by_seq(struct mlx5_lag * ldev,int seq)303 int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq)
304 {
305 	int i, num = 0;
306 
307 	if (!ldev)
308 		return -ENOENT;
309 
310 	mlx5_ldev_for_each(i, 0, ldev) {
311 		if (num == seq)
312 			return i;
313 		num++;
314 	}
315 	return -ENOENT;
316 }
317 
mlx5_lag_num_devs(struct mlx5_lag * ldev)318 int mlx5_lag_num_devs(struct mlx5_lag *ldev)
319 {
320 	int i, num = 0;
321 
322 	if (!ldev)
323 		return 0;
324 
325 	mlx5_ldev_for_each(i, 0, ldev) {
326 		(void)i;
327 		num++;
328 	}
329 	return num;
330 }
331 
mlx5_lag_num_netdevs(struct mlx5_lag * ldev)332 int mlx5_lag_num_netdevs(struct mlx5_lag *ldev)
333 {
334 	int i, num = 0;
335 
336 	if (!ldev)
337 		return 0;
338 
339 	mlx5_ldev_for_each(i, 0, ldev)
340 		if (ldev->pf[i].netdev)
341 			num++;
342 	return num;
343 }
344 
__mlx5_lag_is_roce(struct mlx5_lag * ldev)345 static bool __mlx5_lag_is_roce(struct mlx5_lag *ldev)
346 {
347 	return ldev->mode == MLX5_LAG_MODE_ROCE;
348 }
349 
__mlx5_lag_is_sriov(struct mlx5_lag * ldev)350 static bool __mlx5_lag_is_sriov(struct mlx5_lag *ldev)
351 {
352 	return ldev->mode == MLX5_LAG_MODE_SRIOV;
353 }
354 
355 /* Create a mapping between steering slots and active ports.
356  * As we have ldev->buckets slots per port first assume the native
357  * mapping should be used.
358  * If there are ports that are disabled fill the relevant slots
359  * with mapping that points to active ports.
360  */
mlx5_infer_tx_affinity_mapping(struct lag_tracker * tracker,struct mlx5_lag * ldev,u8 buckets,u8 * ports)361 static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
362 					   struct mlx5_lag *ldev,
363 					   u8 buckets,
364 					   u8 *ports)
365 {
366 	int disabled[MLX5_MAX_PORTS] = {};
367 	int enabled[MLX5_MAX_PORTS] = {};
368 	int disabled_ports_num = 0;
369 	int enabled_ports_num = 0;
370 	int idx;
371 	u32 rand;
372 	int i;
373 	int j;
374 
375 	mlx5_ldev_for_each(i, 0, ldev) {
376 		if (tracker->netdev_state[i].tx_enabled &&
377 		    tracker->netdev_state[i].link_up)
378 			enabled[enabled_ports_num++] = i;
379 		else
380 			disabled[disabled_ports_num++] = i;
381 	}
382 
383 	/* Use native mapping by default where each port's buckets
384 	 * point the native port: 1 1 1 .. 1 2 2 2 ... 2 3 3 3 ... 3 etc
385 	 */
386 	mlx5_ldev_for_each(i, 0, ldev) {
387 		for (j = 0; j < buckets; j++) {
388 			idx = i * buckets + j;
389 			ports[idx] = i + 1;
390 		}
391 	}
392 
393 	/* If all ports are disabled/enabled keep native mapping */
394 	if (enabled_ports_num == ldev->ports ||
395 	    disabled_ports_num == ldev->ports)
396 		return;
397 
398 	/* Go over the disabled ports and for each assign a random active port */
399 	for (i = 0; i < disabled_ports_num; i++) {
400 		for (j = 0; j < buckets; j++) {
401 			get_random_bytes(&rand, 4);
402 			ports[disabled[i] * buckets + j] = enabled[rand % enabled_ports_num] + 1;
403 		}
404 	}
405 }
406 
mlx5_lag_has_drop_rule(struct mlx5_lag * ldev)407 static bool mlx5_lag_has_drop_rule(struct mlx5_lag *ldev)
408 {
409 	int i;
410 
411 	mlx5_ldev_for_each(i, 0, ldev)
412 		if (ldev->pf[i].has_drop)
413 			return true;
414 	return false;
415 }
416 
mlx5_lag_drop_rule_cleanup(struct mlx5_lag * ldev)417 static void mlx5_lag_drop_rule_cleanup(struct mlx5_lag *ldev)
418 {
419 	int i;
420 
421 	mlx5_ldev_for_each(i, 0, ldev) {
422 		if (!ldev->pf[i].has_drop)
423 			continue;
424 
425 		mlx5_esw_acl_ingress_vport_drop_rule_destroy(ldev->pf[i].dev->priv.eswitch,
426 							     MLX5_VPORT_UPLINK);
427 		ldev->pf[i].has_drop = false;
428 	}
429 }
430 
mlx5_lag_drop_rule_setup(struct mlx5_lag * ldev,struct lag_tracker * tracker)431 static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
432 				     struct lag_tracker *tracker)
433 {
434 	u8 disabled_ports[MLX5_MAX_PORTS] = {};
435 	struct mlx5_core_dev *dev;
436 	int disabled_index;
437 	int num_disabled;
438 	int err;
439 	int i;
440 
441 	/* First delete the current drop rule so there won't be any dropped
442 	 * packets
443 	 */
444 	mlx5_lag_drop_rule_cleanup(ldev);
445 
446 	if (!ldev->tracker.has_inactive)
447 		return;
448 
449 	mlx5_infer_tx_disabled(tracker, ldev, disabled_ports, &num_disabled);
450 
451 	for (i = 0; i < num_disabled; i++) {
452 		disabled_index = disabled_ports[i];
453 		dev = ldev->pf[disabled_index].dev;
454 		err = mlx5_esw_acl_ingress_vport_drop_rule_create(dev->priv.eswitch,
455 								  MLX5_VPORT_UPLINK);
456 		if (!err)
457 			ldev->pf[disabled_index].has_drop = true;
458 		else
459 			mlx5_core_err(dev,
460 				      "Failed to create lag drop rule, error: %d", err);
461 	}
462 }
463 
mlx5_cmd_modify_active_port(struct mlx5_core_dev * dev,u8 ports)464 static int mlx5_cmd_modify_active_port(struct mlx5_core_dev *dev, u8 ports)
465 {
466 	u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {};
467 	void *lag_ctx;
468 
469 	lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
470 
471 	MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
472 	MLX5_SET(modify_lag_in, in, field_select, 0x2);
473 
474 	MLX5_SET(lagc, lag_ctx, active_port, ports);
475 
476 	return mlx5_cmd_exec_in(dev, modify_lag, in);
477 }
478 
_mlx5_modify_lag(struct mlx5_lag * ldev,u8 * ports)479 static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports)
480 {
481 	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
482 	struct mlx5_core_dev *dev0;
483 	u8 active_ports;
484 	int ret;
485 
486 	if (idx < 0)
487 		return -EINVAL;
488 
489 	dev0 = ldev->pf[idx].dev;
490 	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) {
491 		ret = mlx5_lag_port_sel_modify(ldev, ports);
492 		if (ret ||
493 		    !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table_bypass))
494 			return ret;
495 
496 		active_ports = lag_active_port_bits(ldev);
497 
498 		return mlx5_cmd_modify_active_port(dev0, active_ports);
499 	}
500 	return mlx5_cmd_modify_lag(dev0, ldev, ports);
501 }
502 
mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev * dev)503 static struct net_device *mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev *dev)
504 {
505 	struct net_device *ndev = NULL;
506 	struct mlx5_lag *ldev;
507 	unsigned long flags;
508 	int i, last_idx;
509 
510 	spin_lock_irqsave(&lag_lock, flags);
511 	ldev = mlx5_lag_dev(dev);
512 
513 	if (!ldev)
514 		goto unlock;
515 
516 	mlx5_ldev_for_each(i, 0, ldev)
517 		if (ldev->tracker.netdev_state[i].tx_enabled)
518 			ndev = ldev->pf[i].netdev;
519 	if (!ndev) {
520 		last_idx = mlx5_lag_get_dev_index_by_seq(ldev, ldev->ports - 1);
521 		if (last_idx < 0)
522 			goto unlock;
523 		ndev = ldev->pf[last_idx].netdev;
524 	}
525 
526 	dev_hold(ndev);
527 
528 unlock:
529 	spin_unlock_irqrestore(&lag_lock, flags);
530 
531 	return ndev;
532 }
533 
mlx5_modify_lag(struct mlx5_lag * ldev,struct lag_tracker * tracker)534 void mlx5_modify_lag(struct mlx5_lag *ldev,
535 		     struct lag_tracker *tracker)
536 {
537 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
538 	u8 ports[MLX5_MAX_PORTS * MLX5_LAG_MAX_HASH_BUCKETS] = {};
539 	struct mlx5_core_dev *dev0;
540 	int idx;
541 	int err;
542 	int i;
543 	int j;
544 
545 	if (first_idx < 0)
546 		return;
547 
548 	dev0 = ldev->pf[first_idx].dev;
549 	mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ports);
550 
551 	mlx5_ldev_for_each(i, 0, ldev) {
552 		for (j = 0; j < ldev->buckets; j++) {
553 			idx = i * ldev->buckets + j;
554 			if (ports[idx] == ldev->v2p_map[idx])
555 				continue;
556 			err = _mlx5_modify_lag(ldev, ports);
557 			if (err) {
558 				mlx5_core_err(dev0,
559 					      "Failed to modify LAG (%d)\n",
560 					      err);
561 				return;
562 			}
563 			memcpy(ldev->v2p_map, ports, sizeof(ports));
564 
565 			mlx5_lag_print_mapping(dev0, ldev, tracker,
566 					       ldev->mode_flags);
567 			break;
568 		}
569 	}
570 
571 	if (tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
572 		struct net_device *ndev = mlx5_lag_active_backup_get_netdev(dev0);
573 
574 		if(!(ldev->mode == MLX5_LAG_MODE_ROCE))
575 			mlx5_lag_drop_rule_setup(ldev, tracker);
576 		/** Only sriov and roce lag should have tracker->tx_type set so
577 		 *  no need to check the mode
578 		 */
579 		blocking_notifier_call_chain(&dev0->priv.lag_nh,
580 					     MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE,
581 					     ndev);
582 		dev_put(ndev);
583 	}
584 }
585 
mlx5_lag_set_port_sel_mode(struct mlx5_lag * ldev,enum mlx5_lag_mode mode,unsigned long * flags)586 static int mlx5_lag_set_port_sel_mode(struct mlx5_lag *ldev,
587 				      enum mlx5_lag_mode mode,
588 				      unsigned long *flags)
589 {
590 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
591 	struct mlx5_core_dev *dev0;
592 
593 	if (first_idx < 0)
594 		return -EINVAL;
595 
596 	if (mode == MLX5_LAG_MODE_MPESW ||
597 	    mode == MLX5_LAG_MODE_MULTIPATH)
598 		return 0;
599 
600 	dev0 = ldev->pf[first_idx].dev;
601 
602 	if (!MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table)) {
603 		if (ldev->ports > 2)
604 			return -EINVAL;
605 		return 0;
606 	}
607 
608 	if (ldev->ports > 2)
609 		ldev->buckets = MLX5_LAG_MAX_HASH_BUCKETS;
610 
611 	set_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, flags);
612 
613 	return 0;
614 }
615 
mlx5_lag_set_flags(struct mlx5_lag * ldev,enum mlx5_lag_mode mode,struct lag_tracker * tracker,bool shared_fdb,unsigned long * flags)616 static int mlx5_lag_set_flags(struct mlx5_lag *ldev, enum mlx5_lag_mode mode,
617 			      struct lag_tracker *tracker, bool shared_fdb,
618 			      unsigned long *flags)
619 {
620 	*flags = 0;
621 	if (shared_fdb) {
622 		set_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, flags);
623 		set_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE, flags);
624 	}
625 
626 	if (mode == MLX5_LAG_MODE_MPESW)
627 		set_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE, flags);
628 
629 	return mlx5_lag_set_port_sel_mode(ldev, mode, flags);
630 }
631 
mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode,unsigned long flags)632 char *mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags)
633 {
634 	int port_sel_mode = get_port_sel_mode(mode, flags);
635 
636 	switch (port_sel_mode) {
637 	case MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY: return "queue_affinity";
638 	case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT: return "hash";
639 	case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_MPESW: return "mpesw";
640 	default: return "invalid";
641 	}
642 }
643 
mlx5_lag_create_single_fdb(struct mlx5_lag * ldev)644 static int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev)
645 {
646 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
647 	struct mlx5_eswitch *master_esw;
648 	struct mlx5_core_dev *dev0;
649 	int i, j;
650 	int err;
651 
652 	if (first_idx < 0)
653 		return -EINVAL;
654 
655 	dev0 = ldev->pf[first_idx].dev;
656 	master_esw = dev0->priv.eswitch;
657 	mlx5_ldev_for_each(i, first_idx + 1, ldev) {
658 		struct mlx5_eswitch *slave_esw = ldev->pf[i].dev->priv.eswitch;
659 
660 		err = mlx5_eswitch_offloads_single_fdb_add_one(master_esw,
661 							       slave_esw, ldev->ports);
662 		if (err)
663 			goto err;
664 	}
665 	return 0;
666 err:
667 	mlx5_ldev_for_each_reverse(j, i, first_idx + 1, ldev)
668 		mlx5_eswitch_offloads_single_fdb_del_one(master_esw,
669 							 ldev->pf[j].dev->priv.eswitch);
670 	return err;
671 }
672 
mlx5_create_lag(struct mlx5_lag * ldev,struct lag_tracker * tracker,enum mlx5_lag_mode mode,unsigned long flags)673 static int mlx5_create_lag(struct mlx5_lag *ldev,
674 			   struct lag_tracker *tracker,
675 			   enum mlx5_lag_mode mode,
676 			   unsigned long flags)
677 {
678 	bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags);
679 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
680 	u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {};
681 	struct mlx5_core_dev *dev0;
682 	int err;
683 
684 	if (first_idx < 0)
685 		return -EINVAL;
686 
687 	dev0 = ldev->pf[first_idx].dev;
688 	if (tracker)
689 		mlx5_lag_print_mapping(dev0, ldev, tracker, flags);
690 	mlx5_core_info(dev0, "shared_fdb:%d mode:%s\n",
691 		       shared_fdb, mlx5_get_str_port_sel_mode(mode, flags));
692 
693 	err = mlx5_cmd_create_lag(dev0, ldev, mode, flags);
694 	if (err) {
695 		mlx5_core_err(dev0,
696 			      "Failed to create LAG (%d)\n",
697 			      err);
698 		return err;
699 	}
700 
701 	if (shared_fdb) {
702 		err = mlx5_lag_create_single_fdb(ldev);
703 		if (err)
704 			mlx5_core_err(dev0, "Can't enable single FDB mode\n");
705 		else
706 			mlx5_core_info(dev0, "Operation mode is single FDB\n");
707 	}
708 
709 	if (err) {
710 		MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);
711 		if (mlx5_cmd_exec_in(dev0, destroy_lag, in))
712 			mlx5_core_err(dev0,
713 				      "Failed to deactivate RoCE LAG; driver restart required\n");
714 	}
715 	BLOCKING_INIT_NOTIFIER_HEAD(&dev0->priv.lag_nh);
716 
717 	return err;
718 }
719 
mlx5_activate_lag(struct mlx5_lag * ldev,struct lag_tracker * tracker,enum mlx5_lag_mode mode,bool shared_fdb)720 int mlx5_activate_lag(struct mlx5_lag *ldev,
721 		      struct lag_tracker *tracker,
722 		      enum mlx5_lag_mode mode,
723 		      bool shared_fdb)
724 {
725 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
726 	bool roce_lag = mode == MLX5_LAG_MODE_ROCE;
727 	struct mlx5_core_dev *dev0;
728 	unsigned long flags = 0;
729 	int err;
730 
731 	if (first_idx < 0)
732 		return -EINVAL;
733 
734 	dev0 = ldev->pf[first_idx].dev;
735 	err = mlx5_lag_set_flags(ldev, mode, tracker, shared_fdb, &flags);
736 	if (err)
737 		return err;
738 
739 	if (mode != MLX5_LAG_MODE_MPESW) {
740 		mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ldev->v2p_map);
741 		if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
742 			err = mlx5_lag_port_sel_create(ldev, tracker->hash_type,
743 						       ldev->v2p_map);
744 			if (err) {
745 				mlx5_core_err(dev0,
746 					      "Failed to create LAG port selection(%d)\n",
747 					      err);
748 				return err;
749 			}
750 		}
751 	}
752 
753 	err = mlx5_create_lag(ldev, tracker, mode, flags);
754 	if (err) {
755 		if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags))
756 			mlx5_lag_port_sel_destroy(ldev);
757 		if (roce_lag)
758 			mlx5_core_err(dev0,
759 				      "Failed to activate RoCE LAG\n");
760 		else
761 			mlx5_core_err(dev0,
762 				      "Failed to activate VF LAG\n"
763 				      "Make sure all VFs are unbound prior to VF LAG activation or deactivation\n");
764 		return err;
765 	}
766 
767 	if (tracker && tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP &&
768 	    !roce_lag)
769 		mlx5_lag_drop_rule_setup(ldev, tracker);
770 
771 	ldev->mode = mode;
772 	ldev->mode_flags = flags;
773 	return 0;
774 }
775 
mlx5_deactivate_lag(struct mlx5_lag * ldev)776 int mlx5_deactivate_lag(struct mlx5_lag *ldev)
777 {
778 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
779 	u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {};
780 	bool roce_lag = __mlx5_lag_is_roce(ldev);
781 	unsigned long flags = ldev->mode_flags;
782 	struct mlx5_eswitch *master_esw;
783 	struct mlx5_core_dev *dev0;
784 	int err;
785 	int i;
786 
787 	if (first_idx < 0)
788 		return -EINVAL;
789 
790 	dev0 = ldev->pf[first_idx].dev;
791 	master_esw = dev0->priv.eswitch;
792 	ldev->mode = MLX5_LAG_MODE_NONE;
793 	ldev->mode_flags = 0;
794 	mlx5_lag_mp_reset(ldev);
795 
796 	if (test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags)) {
797 		mlx5_ldev_for_each(i, first_idx + 1, ldev)
798 			mlx5_eswitch_offloads_single_fdb_del_one(master_esw,
799 								 ldev->pf[i].dev->priv.eswitch);
800 		clear_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags);
801 	}
802 
803 	MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);
804 	err = mlx5_cmd_exec_in(dev0, destroy_lag, in);
805 	if (err) {
806 		if (roce_lag) {
807 			mlx5_core_err(dev0,
808 				      "Failed to deactivate RoCE LAG; driver restart required\n");
809 		} else {
810 			mlx5_core_err(dev0,
811 				      "Failed to deactivate VF LAG; driver restart required\n"
812 				      "Make sure all VFs are unbound prior to VF LAG activation or deactivation\n");
813 		}
814 		return err;
815 	}
816 
817 	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
818 		mlx5_lag_port_sel_destroy(ldev);
819 		ldev->buckets = 1;
820 	}
821 	if (mlx5_lag_has_drop_rule(ldev))
822 		mlx5_lag_drop_rule_cleanup(ldev);
823 
824 	return 0;
825 }
826 
mlx5_lag_check_prereq(struct mlx5_lag * ldev)827 bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
828 {
829 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
830 #ifdef CONFIG_MLX5_ESWITCH
831 	struct mlx5_core_dev *dev;
832 	u8 mode;
833 #endif
834 	bool roce_support;
835 	int i;
836 
837 	if (first_idx < 0 || mlx5_lag_num_devs(ldev) != ldev->ports)
838 		return false;
839 
840 #ifdef CONFIG_MLX5_ESWITCH
841 	mlx5_ldev_for_each(i, 0, ldev) {
842 		dev = ldev->pf[i].dev;
843 		if (mlx5_eswitch_num_vfs(dev->priv.eswitch) && !is_mdev_switchdev_mode(dev))
844 			return false;
845 	}
846 
847 	dev = ldev->pf[first_idx].dev;
848 	mode = mlx5_eswitch_mode(dev);
849 	mlx5_ldev_for_each(i, 0, ldev)
850 		if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode)
851 			return false;
852 
853 #else
854 	mlx5_ldev_for_each(i, 0, ldev)
855 		if (mlx5_sriov_is_enabled(ldev->pf[i].dev))
856 			return false;
857 #endif
858 	roce_support = mlx5_get_roce_state(ldev->pf[first_idx].dev);
859 	mlx5_ldev_for_each(i, first_idx + 1, ldev)
860 		if (mlx5_get_roce_state(ldev->pf[i].dev) != roce_support)
861 			return false;
862 
863 	return true;
864 }
865 
mlx5_lag_add_devices(struct mlx5_lag * ldev)866 void mlx5_lag_add_devices(struct mlx5_lag *ldev)
867 {
868 	int i;
869 
870 	mlx5_ldev_for_each(i, 0, ldev) {
871 		if (ldev->pf[i].dev->priv.flags &
872 		    MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
873 			continue;
874 
875 		ldev->pf[i].dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
876 		mlx5_rescan_drivers_locked(ldev->pf[i].dev);
877 	}
878 }
879 
mlx5_lag_remove_devices(struct mlx5_lag * ldev)880 void mlx5_lag_remove_devices(struct mlx5_lag *ldev)
881 {
882 	int i;
883 
884 	mlx5_ldev_for_each(i, 0, ldev) {
885 		if (ldev->pf[i].dev->priv.flags &
886 		    MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
887 			continue;
888 
889 		ldev->pf[i].dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
890 		mlx5_rescan_drivers_locked(ldev->pf[i].dev);
891 	}
892 }
893 
mlx5_disable_lag(struct mlx5_lag * ldev)894 void mlx5_disable_lag(struct mlx5_lag *ldev)
895 {
896 	bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags);
897 	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
898 	struct mlx5_core_dev *dev0;
899 	bool roce_lag;
900 	int err;
901 	int i;
902 
903 	if (idx < 0)
904 		return;
905 
906 	dev0 = ldev->pf[idx].dev;
907 	roce_lag = __mlx5_lag_is_roce(ldev);
908 
909 	if (shared_fdb) {
910 		mlx5_lag_remove_devices(ldev);
911 	} else if (roce_lag) {
912 		if (!(dev0->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) {
913 			dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
914 			mlx5_rescan_drivers_locked(dev0);
915 		}
916 		mlx5_ldev_for_each(i, idx + 1, ldev)
917 			mlx5_nic_vport_disable_roce(ldev->pf[i].dev);
918 	}
919 
920 	err = mlx5_deactivate_lag(ldev);
921 	if (err)
922 		return;
923 
924 	if (shared_fdb || roce_lag)
925 		mlx5_lag_add_devices(ldev);
926 
927 	if (shared_fdb)
928 		mlx5_ldev_for_each(i, 0, ldev)
929 			if (!(ldev->pf[i].dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV))
930 				mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
931 }
932 
mlx5_lag_shared_fdb_supported(struct mlx5_lag * ldev)933 bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev)
934 {
935 	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
936 	struct mlx5_core_dev *dev;
937 	int i;
938 
939 	if (idx < 0)
940 		return false;
941 
942 	mlx5_ldev_for_each(i, idx + 1, ldev) {
943 		dev = ldev->pf[i].dev;
944 		if (is_mdev_switchdev_mode(dev) &&
945 		    mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) &&
946 		    MLX5_CAP_GEN(dev, lag_native_fdb_selection) &&
947 		    MLX5_CAP_ESW(dev, root_ft_on_other_esw) &&
948 		    mlx5_eswitch_get_npeers(dev->priv.eswitch) ==
949 		    MLX5_CAP_GEN(dev, num_lag_ports) - 1)
950 			continue;
951 		return false;
952 	}
953 
954 	dev = ldev->pf[idx].dev;
955 	if (is_mdev_switchdev_mode(dev) &&
956 	    mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) &&
957 	    mlx5_esw_offloads_devcom_is_ready(dev->priv.eswitch) &&
958 	    MLX5_CAP_ESW(dev, esw_shared_ingress_acl) &&
959 	    mlx5_eswitch_get_npeers(dev->priv.eswitch) == MLX5_CAP_GEN(dev, num_lag_ports) - 1)
960 		return true;
961 
962 	return false;
963 }
964 
mlx5_lag_is_roce_lag(struct mlx5_lag * ldev)965 static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev)
966 {
967 	bool roce_lag = true;
968 	int i;
969 
970 	mlx5_ldev_for_each(i, 0, ldev)
971 		roce_lag = roce_lag && !mlx5_sriov_is_enabled(ldev->pf[i].dev);
972 
973 #ifdef CONFIG_MLX5_ESWITCH
974 	mlx5_ldev_for_each(i, 0, ldev)
975 		roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev);
976 #endif
977 
978 	return roce_lag;
979 }
980 
mlx5_lag_should_modify_lag(struct mlx5_lag * ldev,bool do_bond)981 static bool mlx5_lag_should_modify_lag(struct mlx5_lag *ldev, bool do_bond)
982 {
983 	return do_bond && __mlx5_lag_is_active(ldev) &&
984 	       ldev->mode != MLX5_LAG_MODE_MPESW;
985 }
986 
mlx5_lag_should_disable_lag(struct mlx5_lag * ldev,bool do_bond)987 static bool mlx5_lag_should_disable_lag(struct mlx5_lag *ldev, bool do_bond)
988 {
989 	return !do_bond && __mlx5_lag_is_active(ldev) &&
990 	       ldev->mode != MLX5_LAG_MODE_MPESW;
991 }
992 
mlx5_do_bond(struct mlx5_lag * ldev)993 static void mlx5_do_bond(struct mlx5_lag *ldev)
994 {
995 	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
996 	struct lag_tracker tracker = { };
997 	struct mlx5_core_dev *dev0;
998 	struct net_device *ndev;
999 	bool do_bond, roce_lag;
1000 	int err;
1001 	int i;
1002 
1003 	if (idx < 0)
1004 		return;
1005 
1006 	dev0 = ldev->pf[idx].dev;
1007 	if (!mlx5_lag_is_ready(ldev)) {
1008 		do_bond = false;
1009 	} else {
1010 		/* VF LAG is in multipath mode, ignore bond change requests */
1011 		if (mlx5_lag_is_multipath(dev0))
1012 			return;
1013 
1014 		tracker = ldev->tracker;
1015 
1016 		do_bond = tracker.is_bonded && mlx5_lag_check_prereq(ldev);
1017 	}
1018 
1019 	if (do_bond && !__mlx5_lag_is_active(ldev)) {
1020 		bool shared_fdb = mlx5_lag_shared_fdb_supported(ldev);
1021 
1022 		roce_lag = mlx5_lag_is_roce_lag(ldev);
1023 
1024 		if (shared_fdb || roce_lag)
1025 			mlx5_lag_remove_devices(ldev);
1026 
1027 		err = mlx5_activate_lag(ldev, &tracker,
1028 					roce_lag ? MLX5_LAG_MODE_ROCE :
1029 						   MLX5_LAG_MODE_SRIOV,
1030 					shared_fdb);
1031 		if (err) {
1032 			if (shared_fdb || roce_lag)
1033 				mlx5_lag_add_devices(ldev);
1034 			if (shared_fdb) {
1035 				mlx5_ldev_for_each(i, 0, ldev)
1036 					mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
1037 			}
1038 
1039 			return;
1040 		} else if (roce_lag) {
1041 			dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
1042 			mlx5_rescan_drivers_locked(dev0);
1043 			mlx5_ldev_for_each(i, idx + 1, ldev) {
1044 				if (mlx5_get_roce_state(ldev->pf[i].dev))
1045 					mlx5_nic_vport_enable_roce(ldev->pf[i].dev);
1046 			}
1047 		} else if (shared_fdb) {
1048 			int i;
1049 
1050 			dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
1051 			mlx5_rescan_drivers_locked(dev0);
1052 
1053 			mlx5_ldev_for_each(i, 0, ldev) {
1054 				err = mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
1055 				if (err)
1056 					break;
1057 			}
1058 
1059 			if (err) {
1060 				dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
1061 				mlx5_rescan_drivers_locked(dev0);
1062 				mlx5_deactivate_lag(ldev);
1063 				mlx5_lag_add_devices(ldev);
1064 				mlx5_ldev_for_each(i, 0, ldev)
1065 					mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
1066 				mlx5_core_err(dev0, "Failed to enable lag\n");
1067 				return;
1068 			}
1069 		}
1070 		if (tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
1071 			ndev = mlx5_lag_active_backup_get_netdev(dev0);
1072 			/** Only sriov and roce lag should have tracker->TX_type
1073 			 *  set so no need to check the mode
1074 			 */
1075 			blocking_notifier_call_chain(&dev0->priv.lag_nh,
1076 						     MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE,
1077 						     ndev);
1078 			dev_put(ndev);
1079 		}
1080 	} else if (mlx5_lag_should_modify_lag(ldev, do_bond)) {
1081 		mlx5_modify_lag(ldev, &tracker);
1082 	} else if (mlx5_lag_should_disable_lag(ldev, do_bond)) {
1083 		mlx5_disable_lag(ldev);
1084 	}
1085 }
1086 
1087 /* The last mdev to unregister will destroy the workqueue before removing the
1088  * devcom component, and as all the mdevs use the same devcom component we are
1089  * guaranteed that the devcom is valid while the calling work is running.
1090  */
mlx5_lag_get_devcom_comp(struct mlx5_lag * ldev)1091 struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev)
1092 {
1093 	struct mlx5_devcom_comp_dev *devcom = NULL;
1094 	int i;
1095 
1096 	mutex_lock(&ldev->lock);
1097 	i = mlx5_get_next_ldev_func(ldev, 0);
1098 	if (i < MLX5_MAX_PORTS)
1099 		devcom = ldev->pf[i].dev->priv.hca_devcom_comp;
1100 	mutex_unlock(&ldev->lock);
1101 	return devcom;
1102 }
1103 
mlx5_queue_bond_work(struct mlx5_lag * ldev,unsigned long delay)1104 static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay)
1105 {
1106 	queue_delayed_work(ldev->wq, &ldev->bond_work, delay);
1107 }
1108 
mlx5_do_bond_work(struct work_struct * work)1109 static void mlx5_do_bond_work(struct work_struct *work)
1110 {
1111 	struct delayed_work *delayed_work = to_delayed_work(work);
1112 	struct mlx5_lag *ldev = container_of(delayed_work, struct mlx5_lag,
1113 					     bond_work);
1114 	struct mlx5_devcom_comp_dev *devcom;
1115 	int status;
1116 
1117 	devcom = mlx5_lag_get_devcom_comp(ldev);
1118 	if (!devcom)
1119 		return;
1120 
1121 	status = mlx5_devcom_comp_trylock(devcom);
1122 	if (!status) {
1123 		mlx5_queue_bond_work(ldev, HZ);
1124 		return;
1125 	}
1126 
1127 	mutex_lock(&ldev->lock);
1128 	if (ldev->mode_changes_in_progress) {
1129 		mutex_unlock(&ldev->lock);
1130 		mlx5_devcom_comp_unlock(devcom);
1131 		mlx5_queue_bond_work(ldev, HZ);
1132 		return;
1133 	}
1134 
1135 	mlx5_do_bond(ldev);
1136 	mutex_unlock(&ldev->lock);
1137 	mlx5_devcom_comp_unlock(devcom);
1138 }
1139 
mlx5_handle_changeupper_event(struct mlx5_lag * ldev,struct lag_tracker * tracker,struct netdev_notifier_changeupper_info * info)1140 static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
1141 					 struct lag_tracker *tracker,
1142 					 struct netdev_notifier_changeupper_info *info)
1143 {
1144 	struct net_device *upper = info->upper_dev, *ndev_tmp;
1145 	struct netdev_lag_upper_info *lag_upper_info = NULL;
1146 	bool is_bonded, is_in_lag, mode_supported;
1147 	bool has_inactive = 0;
1148 	struct slave *slave;
1149 	u8 bond_status = 0;
1150 	int num_slaves = 0;
1151 	int changed = 0;
1152 	int i, idx = -1;
1153 
1154 	if (!netif_is_lag_master(upper))
1155 		return 0;
1156 
1157 	if (info->linking)
1158 		lag_upper_info = info->upper_info;
1159 
1160 	/* The event may still be of interest if the slave does not belong to
1161 	 * us, but is enslaved to a master which has one or more of our netdevs
1162 	 * as slaves (e.g., if a new slave is added to a master that bonds two
1163 	 * of our netdevs, we should unbond).
1164 	 */
1165 	rcu_read_lock();
1166 	for_each_netdev_in_bond_rcu(upper, ndev_tmp) {
1167 		mlx5_ldev_for_each(i, 0, ldev) {
1168 			if (ldev->pf[i].netdev == ndev_tmp) {
1169 				idx++;
1170 				break;
1171 			}
1172 		}
1173 		if (i < MLX5_MAX_PORTS) {
1174 			slave = bond_slave_get_rcu(ndev_tmp);
1175 			if (slave)
1176 				has_inactive |= bond_is_slave_inactive(slave);
1177 			bond_status |= (1 << idx);
1178 		}
1179 
1180 		num_slaves++;
1181 	}
1182 	rcu_read_unlock();
1183 
1184 	/* None of this lagdev's netdevs are slaves of this master. */
1185 	if (!(bond_status & GENMASK(ldev->ports - 1, 0)))
1186 		return 0;
1187 
1188 	if (lag_upper_info) {
1189 		tracker->tx_type = lag_upper_info->tx_type;
1190 		tracker->hash_type = lag_upper_info->hash_type;
1191 	}
1192 
1193 	tracker->has_inactive = has_inactive;
1194 	/* Determine bonding status:
1195 	 * A device is considered bonded if both its physical ports are slaves
1196 	 * of the same lag master, and only them.
1197 	 */
1198 	is_in_lag = num_slaves == ldev->ports &&
1199 		bond_status == GENMASK(ldev->ports - 1, 0);
1200 
1201 	/* Lag mode must be activebackup or hash. */
1202 	mode_supported = tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP ||
1203 			 tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH;
1204 
1205 	is_bonded = is_in_lag && mode_supported;
1206 	if (tracker->is_bonded != is_bonded) {
1207 		tracker->is_bonded = is_bonded;
1208 		changed = 1;
1209 	}
1210 
1211 	if (!is_in_lag)
1212 		return changed;
1213 
1214 	if (!mlx5_lag_is_ready(ldev))
1215 		NL_SET_ERR_MSG_MOD(info->info.extack,
1216 				   "Can't activate LAG offload, PF is configured with more than 64 VFs");
1217 	else if (!mode_supported)
1218 		NL_SET_ERR_MSG_MOD(info->info.extack,
1219 				   "Can't activate LAG offload, TX type isn't supported");
1220 
1221 	return changed;
1222 }
1223 
mlx5_handle_changelowerstate_event(struct mlx5_lag * ldev,struct lag_tracker * tracker,struct net_device * ndev,struct netdev_notifier_changelowerstate_info * info)1224 static int mlx5_handle_changelowerstate_event(struct mlx5_lag *ldev,
1225 					      struct lag_tracker *tracker,
1226 					      struct net_device *ndev,
1227 					      struct netdev_notifier_changelowerstate_info *info)
1228 {
1229 	struct netdev_lag_lower_state_info *lag_lower_info;
1230 	int idx;
1231 
1232 	if (!netif_is_lag_port(ndev))
1233 		return 0;
1234 
1235 	idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev);
1236 	if (idx < 0)
1237 		return 0;
1238 
1239 	/* This information is used to determine virtual to physical
1240 	 * port mapping.
1241 	 */
1242 	lag_lower_info = info->lower_state_info;
1243 	if (!lag_lower_info)
1244 		return 0;
1245 
1246 	tracker->netdev_state[idx] = *lag_lower_info;
1247 
1248 	return 1;
1249 }
1250 
mlx5_handle_changeinfodata_event(struct mlx5_lag * ldev,struct lag_tracker * tracker,struct net_device * ndev)1251 static int mlx5_handle_changeinfodata_event(struct mlx5_lag *ldev,
1252 					    struct lag_tracker *tracker,
1253 					    struct net_device *ndev)
1254 {
1255 	struct net_device *ndev_tmp;
1256 	struct slave *slave;
1257 	bool has_inactive = 0;
1258 	int idx;
1259 
1260 	if (!netif_is_lag_master(ndev))
1261 		return 0;
1262 
1263 	rcu_read_lock();
1264 	for_each_netdev_in_bond_rcu(ndev, ndev_tmp) {
1265 		idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev_tmp);
1266 		if (idx < 0)
1267 			continue;
1268 
1269 		slave = bond_slave_get_rcu(ndev_tmp);
1270 		if (slave)
1271 			has_inactive |= bond_is_slave_inactive(slave);
1272 	}
1273 	rcu_read_unlock();
1274 
1275 	if (tracker->has_inactive == has_inactive)
1276 		return 0;
1277 
1278 	tracker->has_inactive = has_inactive;
1279 
1280 	return 1;
1281 }
1282 
1283 /* this handler is always registered to netdev events */
mlx5_lag_netdev_event(struct notifier_block * this,unsigned long event,void * ptr)1284 static int mlx5_lag_netdev_event(struct notifier_block *this,
1285 				 unsigned long event, void *ptr)
1286 {
1287 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
1288 	struct lag_tracker tracker;
1289 	struct mlx5_lag *ldev;
1290 	int changed = 0;
1291 
1292 	if (event != NETDEV_CHANGEUPPER &&
1293 	    event != NETDEV_CHANGELOWERSTATE &&
1294 	    event != NETDEV_CHANGEINFODATA)
1295 		return NOTIFY_DONE;
1296 
1297 	ldev    = container_of(this, struct mlx5_lag, nb);
1298 
1299 	tracker = ldev->tracker;
1300 
1301 	switch (event) {
1302 	case NETDEV_CHANGEUPPER:
1303 		changed = mlx5_handle_changeupper_event(ldev, &tracker, ptr);
1304 		break;
1305 	case NETDEV_CHANGELOWERSTATE:
1306 		changed = mlx5_handle_changelowerstate_event(ldev, &tracker,
1307 							     ndev, ptr);
1308 		break;
1309 	case NETDEV_CHANGEINFODATA:
1310 		changed = mlx5_handle_changeinfodata_event(ldev, &tracker, ndev);
1311 		break;
1312 	}
1313 
1314 	ldev->tracker = tracker;
1315 
1316 	if (changed)
1317 		mlx5_queue_bond_work(ldev, 0);
1318 
1319 	return NOTIFY_DONE;
1320 }
1321 
mlx5_ldev_add_netdev(struct mlx5_lag * ldev,struct mlx5_core_dev * dev,struct net_device * netdev)1322 static void mlx5_ldev_add_netdev(struct mlx5_lag *ldev,
1323 				struct mlx5_core_dev *dev,
1324 				struct net_device *netdev)
1325 {
1326 	unsigned int fn = mlx5_get_dev_index(dev);
1327 	unsigned long flags;
1328 
1329 	spin_lock_irqsave(&lag_lock, flags);
1330 	ldev->pf[fn].netdev = netdev;
1331 	ldev->tracker.netdev_state[fn].link_up = 0;
1332 	ldev->tracker.netdev_state[fn].tx_enabled = 0;
1333 	spin_unlock_irqrestore(&lag_lock, flags);
1334 }
1335 
mlx5_ldev_remove_netdev(struct mlx5_lag * ldev,struct net_device * netdev)1336 static void mlx5_ldev_remove_netdev(struct mlx5_lag *ldev,
1337 				    struct net_device *netdev)
1338 {
1339 	unsigned long flags;
1340 	int i;
1341 
1342 	spin_lock_irqsave(&lag_lock, flags);
1343 	mlx5_ldev_for_each(i, 0, ldev) {
1344 		if (ldev->pf[i].netdev == netdev) {
1345 			ldev->pf[i].netdev = NULL;
1346 			break;
1347 		}
1348 	}
1349 	spin_unlock_irqrestore(&lag_lock, flags);
1350 }
1351 
mlx5_ldev_add_mdev(struct mlx5_lag * ldev,struct mlx5_core_dev * dev)1352 static void mlx5_ldev_add_mdev(struct mlx5_lag *ldev,
1353 			      struct mlx5_core_dev *dev)
1354 {
1355 	unsigned int fn = mlx5_get_dev_index(dev);
1356 
1357 	ldev->pf[fn].dev = dev;
1358 	dev->priv.lag = ldev;
1359 }
1360 
mlx5_ldev_remove_mdev(struct mlx5_lag * ldev,struct mlx5_core_dev * dev)1361 static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev,
1362 				  struct mlx5_core_dev *dev)
1363 {
1364 	int fn;
1365 
1366 	fn = mlx5_get_dev_index(dev);
1367 	if (ldev->pf[fn].dev != dev)
1368 		return;
1369 
1370 	ldev->pf[fn].dev = NULL;
1371 	dev->priv.lag = NULL;
1372 }
1373 
1374 /* Must be called with HCA devcom component lock held */
__mlx5_lag_dev_add_mdev(struct mlx5_core_dev * dev)1375 static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)
1376 {
1377 	struct mlx5_devcom_comp_dev *pos = NULL;
1378 	struct mlx5_lag *ldev = NULL;
1379 	struct mlx5_core_dev *tmp_dev;
1380 
1381 	tmp_dev = mlx5_devcom_get_next_peer_data(dev->priv.hca_devcom_comp, &pos);
1382 	if (tmp_dev)
1383 		ldev = mlx5_lag_dev(tmp_dev);
1384 
1385 	if (!ldev) {
1386 		ldev = mlx5_lag_dev_alloc(dev);
1387 		if (!ldev) {
1388 			mlx5_core_err(dev, "Failed to alloc lag dev\n");
1389 			return 0;
1390 		}
1391 		mlx5_ldev_add_mdev(ldev, dev);
1392 		return 0;
1393 	}
1394 
1395 	mutex_lock(&ldev->lock);
1396 	if (ldev->mode_changes_in_progress) {
1397 		mutex_unlock(&ldev->lock);
1398 		return -EAGAIN;
1399 	}
1400 	mlx5_ldev_get(ldev);
1401 	mlx5_ldev_add_mdev(ldev, dev);
1402 	mutex_unlock(&ldev->lock);
1403 
1404 	return 0;
1405 }
1406 
mlx5_lag_remove_mdev(struct mlx5_core_dev * dev)1407 void mlx5_lag_remove_mdev(struct mlx5_core_dev *dev)
1408 {
1409 	struct mlx5_lag *ldev;
1410 
1411 	ldev = mlx5_lag_dev(dev);
1412 	if (!ldev)
1413 		return;
1414 
1415 	/* mdev is being removed, might as well remove debugfs
1416 	 * as early as possible.
1417 	 */
1418 	mlx5_ldev_remove_debugfs(dev->priv.dbg.lag_debugfs);
1419 recheck:
1420 	mutex_lock(&ldev->lock);
1421 	if (ldev->mode_changes_in_progress) {
1422 		mutex_unlock(&ldev->lock);
1423 		msleep(100);
1424 		goto recheck;
1425 	}
1426 	mlx5_ldev_remove_mdev(ldev, dev);
1427 	mutex_unlock(&ldev->lock);
1428 	mlx5_ldev_put(ldev);
1429 }
1430 
mlx5_lag_add_mdev(struct mlx5_core_dev * dev)1431 void mlx5_lag_add_mdev(struct mlx5_core_dev *dev)
1432 {
1433 	int err;
1434 
1435 	if (!mlx5_lag_is_supported(dev))
1436 		return;
1437 
1438 	if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp))
1439 		return;
1440 
1441 recheck:
1442 	mlx5_devcom_comp_lock(dev->priv.hca_devcom_comp);
1443 	err = __mlx5_lag_dev_add_mdev(dev);
1444 	mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp);
1445 
1446 	if (err) {
1447 		msleep(100);
1448 		goto recheck;
1449 	}
1450 	mlx5_ldev_add_debugfs(dev);
1451 }
1452 
mlx5_lag_remove_netdev(struct mlx5_core_dev * dev,struct net_device * netdev)1453 void mlx5_lag_remove_netdev(struct mlx5_core_dev *dev,
1454 			    struct net_device *netdev)
1455 {
1456 	struct mlx5_lag *ldev;
1457 	bool lag_is_active;
1458 
1459 	ldev = mlx5_lag_dev(dev);
1460 	if (!ldev)
1461 		return;
1462 
1463 	mutex_lock(&ldev->lock);
1464 	mlx5_ldev_remove_netdev(ldev, netdev);
1465 	clear_bit(MLX5_LAG_FLAG_NDEVS_READY, &ldev->state_flags);
1466 
1467 	lag_is_active = __mlx5_lag_is_active(ldev);
1468 	mutex_unlock(&ldev->lock);
1469 
1470 	if (lag_is_active)
1471 		mlx5_queue_bond_work(ldev, 0);
1472 }
1473 
mlx5_lag_add_netdev(struct mlx5_core_dev * dev,struct net_device * netdev)1474 void mlx5_lag_add_netdev(struct mlx5_core_dev *dev,
1475 			 struct net_device *netdev)
1476 {
1477 	struct mlx5_lag *ldev;
1478 	int num = 0;
1479 
1480 	ldev = mlx5_lag_dev(dev);
1481 	if (!ldev)
1482 		return;
1483 
1484 	mutex_lock(&ldev->lock);
1485 	mlx5_ldev_add_netdev(ldev, dev, netdev);
1486 	num = mlx5_lag_num_netdevs(ldev);
1487 	if (num >= ldev->ports)
1488 		set_bit(MLX5_LAG_FLAG_NDEVS_READY, &ldev->state_flags);
1489 	mutex_unlock(&ldev->lock);
1490 	mlx5_queue_bond_work(ldev, 0);
1491 }
1492 
mlx5_get_pre_ldev_func(struct mlx5_lag * ldev,int start_idx,int end_idx)1493 int mlx5_get_pre_ldev_func(struct mlx5_lag *ldev, int start_idx, int end_idx)
1494 {
1495 	int i;
1496 
1497 	for (i = start_idx; i >= end_idx; i--)
1498 		if (ldev->pf[i].dev)
1499 			return i;
1500 	return -1;
1501 }
1502 
mlx5_get_next_ldev_func(struct mlx5_lag * ldev,int start_idx)1503 int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx)
1504 {
1505 	int i;
1506 
1507 	for (i = start_idx; i < MLX5_MAX_PORTS; i++)
1508 		if (ldev->pf[i].dev)
1509 			return i;
1510 	return MLX5_MAX_PORTS;
1511 }
1512 
mlx5_lag_is_roce(struct mlx5_core_dev * dev)1513 bool mlx5_lag_is_roce(struct mlx5_core_dev *dev)
1514 {
1515 	struct mlx5_lag *ldev;
1516 	unsigned long flags;
1517 	bool res;
1518 
1519 	spin_lock_irqsave(&lag_lock, flags);
1520 	ldev = mlx5_lag_dev(dev);
1521 	res  = ldev && __mlx5_lag_is_roce(ldev);
1522 	spin_unlock_irqrestore(&lag_lock, flags);
1523 
1524 	return res;
1525 }
1526 EXPORT_SYMBOL(mlx5_lag_is_roce);
1527 
mlx5_lag_is_active(struct mlx5_core_dev * dev)1528 bool mlx5_lag_is_active(struct mlx5_core_dev *dev)
1529 {
1530 	struct mlx5_lag *ldev;
1531 	unsigned long flags;
1532 	bool res;
1533 
1534 	spin_lock_irqsave(&lag_lock, flags);
1535 	ldev = mlx5_lag_dev(dev);
1536 	res  = ldev && __mlx5_lag_is_active(ldev);
1537 	spin_unlock_irqrestore(&lag_lock, flags);
1538 
1539 	return res;
1540 }
1541 EXPORT_SYMBOL(mlx5_lag_is_active);
1542 
mlx5_lag_mode_is_hash(struct mlx5_core_dev * dev)1543 bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev)
1544 {
1545 	struct mlx5_lag *ldev;
1546 	unsigned long flags;
1547 	bool res = 0;
1548 
1549 	spin_lock_irqsave(&lag_lock, flags);
1550 	ldev = mlx5_lag_dev(dev);
1551 	if (ldev)
1552 		res = test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags);
1553 	spin_unlock_irqrestore(&lag_lock, flags);
1554 
1555 	return res;
1556 }
1557 EXPORT_SYMBOL(mlx5_lag_mode_is_hash);
1558 
mlx5_lag_is_master(struct mlx5_core_dev * dev)1559 bool mlx5_lag_is_master(struct mlx5_core_dev *dev)
1560 {
1561 	struct mlx5_lag *ldev;
1562 	unsigned long flags;
1563 	bool res = false;
1564 	int idx;
1565 
1566 	spin_lock_irqsave(&lag_lock, flags);
1567 	ldev = mlx5_lag_dev(dev);
1568 	idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
1569 	res = ldev && __mlx5_lag_is_active(ldev) && idx >= 0 && dev == ldev->pf[idx].dev;
1570 	spin_unlock_irqrestore(&lag_lock, flags);
1571 
1572 	return res;
1573 }
1574 EXPORT_SYMBOL(mlx5_lag_is_master);
1575 
mlx5_lag_is_sriov(struct mlx5_core_dev * dev)1576 bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev)
1577 {
1578 	struct mlx5_lag *ldev;
1579 	unsigned long flags;
1580 	bool res;
1581 
1582 	spin_lock_irqsave(&lag_lock, flags);
1583 	ldev = mlx5_lag_dev(dev);
1584 	res  = ldev && __mlx5_lag_is_sriov(ldev);
1585 	spin_unlock_irqrestore(&lag_lock, flags);
1586 
1587 	return res;
1588 }
1589 EXPORT_SYMBOL(mlx5_lag_is_sriov);
1590 
mlx5_lag_is_shared_fdb(struct mlx5_core_dev * dev)1591 bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev)
1592 {
1593 	struct mlx5_lag *ldev;
1594 	unsigned long flags;
1595 	bool res;
1596 
1597 	spin_lock_irqsave(&lag_lock, flags);
1598 	ldev = mlx5_lag_dev(dev);
1599 	res = ldev && test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags);
1600 	spin_unlock_irqrestore(&lag_lock, flags);
1601 
1602 	return res;
1603 }
1604 EXPORT_SYMBOL(mlx5_lag_is_shared_fdb);
1605 
mlx5_lag_disable_change(struct mlx5_core_dev * dev)1606 void mlx5_lag_disable_change(struct mlx5_core_dev *dev)
1607 {
1608 	struct mlx5_lag *ldev;
1609 
1610 	ldev = mlx5_lag_dev(dev);
1611 	if (!ldev)
1612 		return;
1613 
1614 	mlx5_devcom_comp_lock(dev->priv.hca_devcom_comp);
1615 	mutex_lock(&ldev->lock);
1616 
1617 	ldev->mode_changes_in_progress++;
1618 	if (__mlx5_lag_is_active(ldev))
1619 		mlx5_disable_lag(ldev);
1620 
1621 	mutex_unlock(&ldev->lock);
1622 	mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp);
1623 }
1624 
mlx5_lag_enable_change(struct mlx5_core_dev * dev)1625 void mlx5_lag_enable_change(struct mlx5_core_dev *dev)
1626 {
1627 	struct mlx5_lag *ldev;
1628 
1629 	ldev = mlx5_lag_dev(dev);
1630 	if (!ldev)
1631 		return;
1632 
1633 	mutex_lock(&ldev->lock);
1634 	ldev->mode_changes_in_progress--;
1635 	mutex_unlock(&ldev->lock);
1636 	mlx5_queue_bond_work(ldev, 0);
1637 }
1638 
mlx5_lag_get_slave_port(struct mlx5_core_dev * dev,struct net_device * slave)1639 u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev,
1640 			   struct net_device *slave)
1641 {
1642 	struct mlx5_lag *ldev;
1643 	unsigned long flags;
1644 	u8 port = 0;
1645 	int i;
1646 
1647 	spin_lock_irqsave(&lag_lock, flags);
1648 	ldev = mlx5_lag_dev(dev);
1649 	if (!(ldev && __mlx5_lag_is_roce(ldev)))
1650 		goto unlock;
1651 
1652 	mlx5_ldev_for_each(i, 0, ldev) {
1653 		if (ldev->pf[i].netdev == slave) {
1654 			port = i;
1655 			break;
1656 		}
1657 	}
1658 
1659 	port = ldev->v2p_map[port * ldev->buckets];
1660 
1661 unlock:
1662 	spin_unlock_irqrestore(&lag_lock, flags);
1663 	return port;
1664 }
1665 EXPORT_SYMBOL(mlx5_lag_get_slave_port);
1666 
mlx5_lag_get_num_ports(struct mlx5_core_dev * dev)1667 u8 mlx5_lag_get_num_ports(struct mlx5_core_dev *dev)
1668 {
1669 	struct mlx5_lag *ldev;
1670 
1671 	ldev = mlx5_lag_dev(dev);
1672 	if (!ldev)
1673 		return 0;
1674 
1675 	return ldev->ports;
1676 }
1677 EXPORT_SYMBOL(mlx5_lag_get_num_ports);
1678 
mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev * dev,int * i)1679 struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int *i)
1680 {
1681 	struct mlx5_core_dev *peer_dev = NULL;
1682 	struct mlx5_lag *ldev;
1683 	unsigned long flags;
1684 	int idx;
1685 
1686 	spin_lock_irqsave(&lag_lock, flags);
1687 	ldev = mlx5_lag_dev(dev);
1688 	if (!ldev)
1689 		goto unlock;
1690 
1691 	if (*i == MLX5_MAX_PORTS)
1692 		goto unlock;
1693 	mlx5_ldev_for_each(idx, *i, ldev)
1694 		if (ldev->pf[idx].dev != dev)
1695 			break;
1696 
1697 	if (idx == MLX5_MAX_PORTS) {
1698 		*i = idx;
1699 		goto unlock;
1700 	}
1701 	*i = idx + 1;
1702 
1703 	peer_dev = ldev->pf[idx].dev;
1704 
1705 unlock:
1706 	spin_unlock_irqrestore(&lag_lock, flags);
1707 	return peer_dev;
1708 }
1709 EXPORT_SYMBOL(mlx5_lag_get_next_peer_mdev);
1710 
mlx5_lag_query_cong_counters(struct mlx5_core_dev * dev,u64 * values,int num_counters,size_t * offsets)1711 int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
1712 				 u64 *values,
1713 				 int num_counters,
1714 				 size_t *offsets)
1715 {
1716 	int outlen = MLX5_ST_SZ_BYTES(query_cong_statistics_out);
1717 	struct mlx5_core_dev **mdev;
1718 	int ret = 0, i, j, idx = 0;
1719 	struct mlx5_lag *ldev;
1720 	unsigned long flags;
1721 	int num_ports;
1722 	void *out;
1723 
1724 	out = kvzalloc(outlen, GFP_KERNEL);
1725 	if (!out)
1726 		return -ENOMEM;
1727 
1728 	mdev = kvzalloc(sizeof(mdev[0]) * MLX5_MAX_PORTS, GFP_KERNEL);
1729 	if (!mdev) {
1730 		ret = -ENOMEM;
1731 		goto free_out;
1732 	}
1733 
1734 	memset(values, 0, sizeof(*values) * num_counters);
1735 
1736 	spin_lock_irqsave(&lag_lock, flags);
1737 	ldev = mlx5_lag_dev(dev);
1738 	if (ldev && __mlx5_lag_is_active(ldev)) {
1739 		num_ports = ldev->ports;
1740 		mlx5_ldev_for_each(i, 0, ldev)
1741 			mdev[idx++] = ldev->pf[i].dev;
1742 	} else {
1743 		num_ports = 1;
1744 		mdev[MLX5_LAG_P1] = dev;
1745 	}
1746 	spin_unlock_irqrestore(&lag_lock, flags);
1747 
1748 	for (i = 0; i < num_ports; ++i) {
1749 		u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = {};
1750 
1751 		MLX5_SET(query_cong_statistics_in, in, opcode,
1752 			 MLX5_CMD_OP_QUERY_CONG_STATISTICS);
1753 		ret = mlx5_cmd_exec_inout(mdev[i], query_cong_statistics, in,
1754 					  out);
1755 		if (ret)
1756 			goto free_mdev;
1757 
1758 		for (j = 0; j < num_counters; ++j)
1759 			values[j] += be64_to_cpup((__be64 *)(out + offsets[j]));
1760 	}
1761 
1762 free_mdev:
1763 	kvfree(mdev);
1764 free_out:
1765 	kvfree(out);
1766 	return ret;
1767 }
1768 EXPORT_SYMBOL(mlx5_lag_query_cong_counters);
1769