1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3
4 #include <linux/netdevice.h>
5 #include <net/nexthop.h>
6 #include "lag/lag.h"
7 #include "eswitch.h"
8 #include "esw/acl/ofld.h"
9 #include "lib/events.h"
10
mlx5_mpesw_metadata_cleanup(struct mlx5_lag * ldev)11 static void mlx5_mpesw_metadata_cleanup(struct mlx5_lag *ldev)
12 {
13 struct mlx5_core_dev *dev;
14 struct mlx5_eswitch *esw;
15 u32 pf_metadata;
16 int i;
17
18 mlx5_lag_for_each(i, 0, ldev, MLX5_LAG_FILTER_ALL) {
19 dev = mlx5_lag_pf(ldev, i)->dev;
20 esw = dev->priv.eswitch;
21 pf_metadata = ldev->lag_mpesw.pf_metadata[i];
22 if (!pf_metadata)
23 continue;
24 mlx5_esw_acl_ingress_vport_metadata_update(esw, MLX5_VPORT_UPLINK, 0);
25 mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_MULTIPORT_ESW,
26 (void *)0);
27 mlx5_esw_match_metadata_free(esw, pf_metadata);
28 ldev->lag_mpesw.pf_metadata[i] = 0;
29 }
30 }
31
mlx5_mpesw_metadata_set(struct mlx5_lag * ldev)32 static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev)
33 {
34 struct mlx5_core_dev *dev;
35 struct mlx5_eswitch *esw;
36 u32 pf_metadata;
37 int i, err;
38
39 mlx5_lag_for_each(i, 0, ldev, MLX5_LAG_FILTER_ALL) {
40 dev = mlx5_lag_pf(ldev, i)->dev;
41 esw = dev->priv.eswitch;
42 pf_metadata = mlx5_esw_match_metadata_alloc(esw);
43 if (!pf_metadata) {
44 err = -ENOSPC;
45 goto err_metadata;
46 }
47
48 ldev->lag_mpesw.pf_metadata[i] = pf_metadata;
49 err = mlx5_esw_acl_ingress_vport_metadata_update(esw, MLX5_VPORT_UPLINK,
50 pf_metadata);
51 if (err)
52 goto err_metadata;
53 }
54
55 mlx5_lag_for_each(i, 0, ldev, MLX5_LAG_FILTER_ALL) {
56 dev = mlx5_lag_pf(ldev, i)->dev;
57 mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_MULTIPORT_ESW,
58 (void *)0);
59 }
60
61 return 0;
62
63 err_metadata:
64 mlx5_mpesw_metadata_cleanup(ldev);
65 return err;
66 }
67
mlx5_mpesw_restore_sd_fdb(struct mlx5_lag * ldev)68 static void mlx5_mpesw_restore_sd_fdb(struct mlx5_lag *ldev)
69 {
70 struct lag_func *pf;
71 int err, i;
72
73 mlx5_ldev_for_each(i, 0, ldev) {
74 pf = mlx5_lag_pf(ldev, i);
75 err = mlx5_lag_shared_fdb_create(ldev, NULL, 0, pf->group_id);
76 if (err)
77 mlx5_core_warn(pf->dev,
78 "Failed to restore SD shared FDB (%d)\n",
79 err);
80 }
81 }
82
mlx5_mpesw_teardown_sd_fdb(struct mlx5_lag * ldev)83 static int mlx5_mpesw_teardown_sd_fdb(struct mlx5_lag *ldev)
84 {
85 struct lag_func *pf;
86 int i;
87
88 mlx5_ldev_for_each(i, 0, ldev) {
89 pf = mlx5_lag_pf(ldev, i);
90 if (!pf->sd_fdb_active)
91 continue;
92 mlx5_lag_shared_fdb_destroy(ldev, pf->group_id);
93 }
94 return 0;
95 }
96
mlx5_lag_has_sd_group(struct mlx5_lag * ldev)97 static bool mlx5_lag_has_sd_group(struct mlx5_lag *ldev)
98 {
99 struct lag_func *pf;
100 int i;
101
102 mlx5_ldev_for_each(i, 0, ldev) {
103 pf = mlx5_lag_pf(ldev, i);
104 if (pf->group_id)
105 return true;
106 }
107 return false;
108 }
109
mlx5_lag_enable_mpesw(struct mlx5_lag * ldev)110 static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
111 {
112 int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
113 struct mlx5_core_dev *dev0;
114 int err;
115
116 if (ldev->mode == MLX5_LAG_MODE_MPESW)
117 return 0;
118
119 if (ldev->mode != MLX5_LAG_MODE_NONE)
120 return -EINVAL;
121
122 if (idx < 0)
123 return -EINVAL;
124
125 dev0 = mlx5_lag_pf(ldev, idx)->dev;
126 if (mlx5_eswitch_mode(dev0) != MLX5_ESWITCH_OFFLOADS ||
127 !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table) ||
128 !MLX5_CAP_GEN(dev0, create_lag_when_not_master_up) ||
129 !mlx5_lag_check_prereq(ldev) ||
130 !mlx5_lag_shared_fdb_supported_filter(ldev, MLX5_LAG_FILTER_ALL))
131 return -EOPNOTSUPP;
132
133 err = mlx5_mpesw_metadata_set(ldev);
134 if (err)
135 return err;
136
137 if (mlx5_lag_has_sd_group(ldev))
138 mlx5_mpesw_teardown_sd_fdb(ldev);
139
140 err = mlx5_lag_shared_fdb_create(ldev, NULL, MLX5_LAG_MODE_MPESW,
141 MLX5_LAG_FILTER_ALL);
142 if (err) {
143 mlx5_core_warn(dev0,
144 "Failed to create LAG in MPESW mode (%d)\n",
145 err);
146 if (mlx5_lag_has_sd_group(ldev))
147 mlx5_mpesw_restore_sd_fdb(ldev);
148 mlx5_mpesw_metadata_cleanup(ldev);
149 return err;
150 }
151
152 return 0;
153 }
154
mlx5_lag_disable_mpesw(struct mlx5_lag * ldev)155 void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev)
156 {
157 if (ldev->mode != MLX5_LAG_MODE_MPESW)
158 return;
159
160 mlx5_mpesw_metadata_cleanup(ldev);
161 mlx5_lag_shared_fdb_destroy(ldev, MLX5_LAG_FILTER_ALL);
162 if (mlx5_lag_has_sd_group(ldev))
163 mlx5_mpesw_restore_sd_fdb(ldev);
164 }
165
mlx5_mpesw_sd_devcoms_lock(struct mlx5_lag * ldev)166 void mlx5_mpesw_sd_devcoms_lock(struct mlx5_lag *ldev)
167 {
168 struct mlx5_devcom_comp_dev *sd_devcom;
169 int i;
170
171 mlx5_ldev_for_each(i, 0, ldev) {
172 sd_devcom = mlx5_sd_get_devcom(mlx5_lag_pf(ldev, i)->dev);
173 if (sd_devcom)
174 mlx5_devcom_comp_lock(sd_devcom);
175 }
176 }
177
mlx5_mpesw_sd_devcoms_unlock(struct mlx5_lag * ldev)178 void mlx5_mpesw_sd_devcoms_unlock(struct mlx5_lag *ldev)
179 {
180 struct mlx5_devcom_comp_dev *sd_devcom;
181 int i;
182
183 mlx5_ldev_for_each_reverse(i, MLX5_MAX_PORTS, 0, ldev) {
184 sd_devcom = mlx5_sd_get_devcom(mlx5_lag_pf(ldev, i)->dev);
185 if (sd_devcom)
186 mlx5_devcom_comp_unlock(sd_devcom);
187 }
188 }
189
mlx5_mpesw_work(struct work_struct * work)190 static void mlx5_mpesw_work(struct work_struct *work)
191 {
192 struct mlx5_mpesw_work_st *mpesww = container_of(work, struct mlx5_mpesw_work_st, work);
193 struct mlx5_devcom_comp_dev *devcom;
194 struct mlx5_lag *ldev = mpesww->lag;
195
196 devcom = mlx5_lag_get_devcom_comp(ldev);
197 if (!devcom) {
198 mpesww->result = -ENODEV;
199 goto complete;
200 }
201
202 mlx5_devcom_comp_lock(devcom);
203 mlx5_mpesw_sd_devcoms_lock(ldev);
204 mutex_lock(&ldev->lock);
205 if (ldev->mode_changes_in_progress) {
206 mpesww->result = -EAGAIN;
207 goto unlock;
208 }
209
210 if (mpesww->op == MLX5_MPESW_OP_ENABLE)
211 mpesww->result = mlx5_lag_enable_mpesw(ldev);
212 else if (mpesww->op == MLX5_MPESW_OP_DISABLE)
213 mlx5_lag_disable_mpesw(ldev);
214 unlock:
215 mutex_unlock(&ldev->lock);
216 mlx5_mpesw_sd_devcoms_unlock(ldev);
217 mlx5_devcom_comp_unlock(devcom);
218 complete:
219 complete(&mpesww->comp);
220 }
221
mlx5_lag_mpesw_queue_work(struct mlx5_core_dev * dev,enum mpesw_op op)222 static int mlx5_lag_mpesw_queue_work(struct mlx5_core_dev *dev,
223 enum mpesw_op op)
224 {
225 struct mlx5_lag *ldev = mlx5_lag_dev(dev);
226 struct mlx5_mpesw_work_st *work;
227 int err = 0;
228
229 if (!ldev)
230 return 0;
231
232 work = kzalloc_obj(*work);
233 if (!work)
234 return -ENOMEM;
235
236 INIT_WORK(&work->work, mlx5_mpesw_work);
237 init_completion(&work->comp);
238 work->op = op;
239 work->lag = ldev;
240
241 if (!queue_work(ldev->wq, &work->work)) {
242 mlx5_core_warn(dev, "failed to queue mpesw work\n");
243 err = -EINVAL;
244 goto out;
245 }
246 wait_for_completion(&work->comp);
247 err = work->result;
248 out:
249 kfree(work);
250 return err;
251 }
252
mlx5_lag_mpesw_disable(struct mlx5_core_dev * dev)253 void mlx5_lag_mpesw_disable(struct mlx5_core_dev *dev)
254 {
255 mlx5_lag_mpesw_queue_work(dev, MLX5_MPESW_OP_DISABLE);
256 }
257
mlx5_lag_mpesw_enable(struct mlx5_core_dev * dev)258 int mlx5_lag_mpesw_enable(struct mlx5_core_dev *dev)
259 {
260 return mlx5_lag_mpesw_queue_work(dev, MLX5_MPESW_OP_ENABLE);
261 }
262
mlx5_lag_mpesw_do_mirred(struct mlx5_core_dev * mdev,struct net_device * out_dev,struct netlink_ext_ack * extack)263 int mlx5_lag_mpesw_do_mirred(struct mlx5_core_dev *mdev,
264 struct net_device *out_dev,
265 struct netlink_ext_ack *extack)
266 {
267 struct mlx5_lag *ldev = mlx5_lag_dev(mdev);
268
269 if (!netif_is_bond_master(out_dev) || !ldev)
270 return 0;
271
272 if (ldev->mode != MLX5_LAG_MODE_MPESW)
273 return 0;
274
275 NL_SET_ERR_MSG_MOD(extack, "can't forward to bond in mpesw mode");
276 return -EOPNOTSUPP;
277 }
278
mlx5_lag_is_mpesw(struct mlx5_core_dev * dev)279 bool mlx5_lag_is_mpesw(struct mlx5_core_dev *dev)
280 {
281 struct mlx5_lag *ldev = mlx5_lag_dev(dev);
282
283 return ldev && ldev->mode == MLX5_LAG_MODE_MPESW &&
284 __mlx5_lag_dev_is_port(ldev, dev);
285 }
286 EXPORT_SYMBOL(mlx5_lag_is_mpesw);
287
mlx5_mpesw_speed_update_work(struct work_struct * work)288 void mlx5_mpesw_speed_update_work(struct work_struct *work)
289 {
290 struct mlx5_lag *ldev = container_of(work, struct mlx5_lag,
291 speed_update_work);
292
293 mutex_lock(&ldev->lock);
294 if (ldev->mode == MLX5_LAG_MODE_MPESW) {
295 if (ldev->mode_changes_in_progress)
296 queue_work(ldev->wq, &ldev->speed_update_work);
297 else
298 mlx5_lag_set_vports_agg_speed(ldev);
299 }
300
301 mutex_unlock(&ldev->lock);
302 }
303
mlx5_lag_mpesw_port_change_event(struct notifier_block * nb,unsigned long event,void * data)304 int mlx5_lag_mpesw_port_change_event(struct notifier_block *nb,
305 unsigned long event, void *data)
306 {
307 struct mlx5_nb *mlx5_nb = container_of(nb, struct mlx5_nb, nb);
308 struct lag_func *lag_func = container_of(mlx5_nb,
309 struct lag_func,
310 port_change_nb);
311 struct mlx5_core_dev *dev = lag_func->dev;
312 struct mlx5_lag *ldev = dev->priv.lag;
313 struct mlx5_eqe *eqe = data;
314
315 if (!ldev)
316 return NOTIFY_DONE;
317
318 if (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_DOWN ||
319 eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE)
320 queue_work(ldev->wq, &ldev->speed_update_work);
321
322 return NOTIFY_OK;
323 }
324