1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies.
3
4 #include <linux/mlx5/fs.h>
5 #include "eswitch.h"
6 #include "en_tc.h"
7 #include "fs_core.h"
8
9 struct mlx5_termtbl_handle {
10 struct hlist_node termtbl_hlist;
11
12 struct mlx5_flow_table *termtbl;
13 struct mlx5_flow_act flow_act;
14 struct mlx5_flow_destination dest;
15
16 struct mlx5_flow_handle *rule;
17 int ref_count;
18 };
19
20 static u32
mlx5_eswitch_termtbl_hash(struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest)21 mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act,
22 struct mlx5_flow_destination *dest)
23 {
24 u32 hash;
25
26 hash = jhash_1word(flow_act->action, 0);
27 hash = jhash((const void *)&flow_act->vlan,
28 sizeof(flow_act->vlan), hash);
29 hash = jhash((const void *)&dest->vport.num,
30 sizeof(dest->vport.num), hash);
31 hash = jhash((const void *)&dest->vport.vhca_id,
32 sizeof(dest->vport.num), hash);
33 if (flow_act->pkt_reformat)
34 hash = jhash(flow_act->pkt_reformat,
35 sizeof(*flow_act->pkt_reformat),
36 hash);
37 return hash;
38 }
39
40 static int
mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act * flow_act1,struct mlx5_flow_destination * dest1,struct mlx5_flow_act * flow_act2,struct mlx5_flow_destination * dest2)41 mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1,
42 struct mlx5_flow_destination *dest1,
43 struct mlx5_flow_act *flow_act2,
44 struct mlx5_flow_destination *dest2)
45 {
46 int ret;
47
48 ret = flow_act1->action != flow_act2->action ||
49 dest1->vport.num != dest2->vport.num ||
50 dest1->vport.vhca_id != dest2->vport.vhca_id ||
51 memcmp(&flow_act1->vlan, &flow_act2->vlan,
52 sizeof(flow_act1->vlan));
53 if (ret)
54 return ret;
55
56 if (flow_act1->pkt_reformat && flow_act2->pkt_reformat)
57 return memcmp(flow_act1->pkt_reformat, flow_act2->pkt_reformat,
58 sizeof(*flow_act1->pkt_reformat));
59
60 return !(flow_act1->pkt_reformat == flow_act2->pkt_reformat);
61 }
62
63 static int
mlx5_eswitch_termtbl_create(struct mlx5_core_dev * dev,struct mlx5_termtbl_handle * tt,struct mlx5_flow_act * flow_act)64 mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev,
65 struct mlx5_termtbl_handle *tt,
66 struct mlx5_flow_act *flow_act)
67 {
68 struct mlx5_flow_table_attr ft_attr = {};
69 struct mlx5_flow_namespace *root_ns;
70 int err, err2;
71
72 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
73 if (!root_ns) {
74 esw_warn(dev, "Failed to get FDB flow namespace\n");
75 return -EOPNOTSUPP;
76 }
77
78 /* As this is the terminating action then the termination table is the
79 * same prio as the slow path
80 */
81 ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED |
82 MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
83 ft_attr.prio = FDB_TC_OFFLOAD;
84 ft_attr.max_fte = 1;
85 ft_attr.level = 1;
86 ft_attr.autogroup.max_num_groups = 1;
87 tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
88 if (IS_ERR(tt->termtbl)) {
89 err = PTR_ERR(tt->termtbl);
90 esw_warn(dev, "Failed to create termination table, err %pe\n", tt->termtbl);
91 return err;
92 }
93
94 tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act,
95 &tt->dest, 1);
96 if (IS_ERR(tt->rule)) {
97 err = PTR_ERR(tt->rule);
98 esw_warn(dev, "Failed to create termination table rule, err %pe\n", tt->rule);
99 goto add_flow_err;
100 }
101 return 0;
102
103 add_flow_err:
104 err2 = mlx5_destroy_flow_table(tt->termtbl);
105 if (err2)
106 esw_warn(dev, "Failed to destroy termination table, err %d\n", err2);
107
108 return err;
109 }
110
111 static struct mlx5_termtbl_handle *
mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch * esw,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest,struct mlx5_esw_flow_attr * attr)112 mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw,
113 struct mlx5_flow_act *flow_act,
114 struct mlx5_flow_destination *dest,
115 struct mlx5_esw_flow_attr *attr)
116 {
117 struct mlx5_termtbl_handle *tt;
118 bool found = false;
119 u32 hash_key;
120 int err;
121
122 mutex_lock(&esw->offloads.termtbl_mutex);
123 hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest);
124 hash_for_each_possible(esw->offloads.termtbl_tbl, tt,
125 termtbl_hlist, hash_key) {
126 if (!mlx5_eswitch_termtbl_cmp(&tt->flow_act, &tt->dest,
127 flow_act, dest)) {
128 found = true;
129 break;
130 }
131 }
132 if (found)
133 goto tt_add_ref;
134
135 tt = kzalloc_obj(*tt);
136 if (!tt) {
137 err = -ENOMEM;
138 goto tt_create_err;
139 }
140
141 tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
142 tt->dest.vport.num = dest->vport.num;
143 tt->dest.vport.vhca_id = dest->vport.vhca_id;
144 tt->dest.vport.flags = dest->vport.flags;
145 memcpy(&tt->flow_act, flow_act, sizeof(*flow_act));
146
147 err = mlx5_eswitch_termtbl_create(esw->dev, tt, flow_act);
148 if (err)
149 goto tt_create_err;
150
151 hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key);
152 tt_add_ref:
153 tt->ref_count++;
154 mutex_unlock(&esw->offloads.termtbl_mutex);
155 return tt;
156 tt_create_err:
157 kfree(tt);
158 mutex_unlock(&esw->offloads.termtbl_mutex);
159 return ERR_PTR(err);
160 }
161
162 void
mlx5_eswitch_termtbl_put(struct mlx5_eswitch * esw,struct mlx5_termtbl_handle * tt)163 mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw,
164 struct mlx5_termtbl_handle *tt)
165 {
166 bool last;
167
168 mutex_lock(&esw->offloads.termtbl_mutex);
169 last = (--tt->ref_count == 0);
170 if (last)
171 hash_del(&tt->termtbl_hlist);
172 mutex_unlock(&esw->offloads.termtbl_mutex);
173
174 if (last) {
175 mlx5_del_flow_rules(tt->rule);
176 mlx5_destroy_flow_table(tt->termtbl);
177 kfree(tt);
178 }
179 }
180
181 static void
mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act * src,struct mlx5_flow_act * dst)182 mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src,
183 struct mlx5_flow_act *dst)
184 {
185 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
186 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
187 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
188 memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0]));
189 memset(&src->vlan[0], 0, sizeof(src->vlan[0]));
190
191 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
192 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
193 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
194 memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1]));
195 memset(&src->vlan[1], 0, sizeof(src->vlan[1]));
196 }
197 }
198 }
199
mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch * esw,const struct mlx5_flow_spec * spec)200 static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw,
201 const struct mlx5_flow_spec *spec)
202 {
203 u16 port_mask, port_value;
204
205 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
206 return spec->flow_context.flow_source ==
207 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
208
209 port_mask = MLX5_GET(fte_match_param, spec->match_criteria,
210 misc_parameters.source_port);
211 port_value = MLX5_GET(fte_match_param, spec->match_value,
212 misc_parameters.source_port);
213 return (port_mask & port_value) == MLX5_VPORT_UPLINK;
214 }
215
216 bool
mlx5_eswitch_termtbl_required(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act,struct mlx5_flow_spec * spec)217 mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw,
218 struct mlx5_flow_attr *attr,
219 struct mlx5_flow_act *flow_act,
220 struct mlx5_flow_spec *spec)
221 {
222 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
223 int i;
224
225 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) ||
226 !MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level) ||
227 mlx5e_tc_attr_flags_skip(attr->flags) ||
228 (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port))
229 return false;
230
231 /* push vlan on RX */
232 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
233 !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) &
234 MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX))
235 return true;
236
237 /* hairpin */
238 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
239 if (!esw_attr->dest_int_port && esw_attr->dests[i].vport_valid &&
240 esw_attr->dests[i].vport == MLX5_VPORT_UPLINK)
241 return true;
242
243 return false;
244 }
245
246 struct mlx5_flow_handle *
mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest,int num_dest)247 mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw,
248 struct mlx5_flow_table *fdb,
249 struct mlx5_flow_spec *spec,
250 struct mlx5_esw_flow_attr *attr,
251 struct mlx5_flow_act *flow_act,
252 struct mlx5_flow_destination *dest,
253 int num_dest)
254 {
255 struct mlx5_flow_act term_tbl_act = {};
256 struct mlx5_flow_handle *rule = NULL;
257 bool term_table_created = false;
258 int num_vport_dests = 0;
259 int i, curr_dest;
260
261 mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act);
262 term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
263
264 for (i = 0; i < num_dest; i++) {
265 struct mlx5_termtbl_handle *tt;
266
267 /* only vport destinations can be terminated */
268 if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT)
269 continue;
270
271 if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) {
272 term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
273 term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat;
274 } else {
275 term_tbl_act.action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
276 term_tbl_act.pkt_reformat = NULL;
277 }
278
279 /* get the terminating table for the action list */
280 tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act,
281 &dest[i], attr);
282 if (IS_ERR(tt)) {
283 esw_warn(esw->dev, "Failed to get termination table, err %pe\n", tt);
284 goto revert_changes;
285 }
286 attr->dests[num_vport_dests].termtbl = tt;
287 num_vport_dests++;
288
289 /* link the destination with the termination table */
290 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
291 dest[i].ft = tt->termtbl;
292 term_table_created = true;
293 }
294
295 /* at least one destination should reference a termination table */
296 if (!term_table_created)
297 goto revert_changes;
298
299 /* create the FTE */
300 flow_act->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
301 flow_act->pkt_reformat = NULL;
302 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
303 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
304 if (IS_ERR(rule))
305 goto revert_changes;
306
307 goto out;
308
309 revert_changes:
310 /* revert the changes that were made to the original flow_act
311 * and fall-back to the original rule actions
312 */
313 mlx5_eswitch_termtbl_actions_move(&term_tbl_act, flow_act);
314
315 for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) {
316 struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl;
317
318 attr->dests[curr_dest].termtbl = NULL;
319
320 /* search for the destination associated with the
321 * current term table
322 */
323 for (i = 0; i < num_dest; i++) {
324 if (dest[i].ft != tt->termtbl)
325 continue;
326
327 memset(&dest[i], 0, sizeof(dest[i]));
328 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
329 dest[i].vport.num = tt->dest.vport.num;
330 dest[i].vport.vhca_id = tt->dest.vport.vhca_id;
331 mlx5_eswitch_termtbl_put(esw, tt);
332 break;
333 }
334 }
335 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
336 out:
337 return rule;
338 }
339