1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2021 Mellanox Technologies. */
3
4 #include <linux/skbuff.h>
5 #include <net/psample.h>
6 #include "en/mapping.h"
7 #include "en/tc/post_act.h"
8 #include "en/tc/act/sample.h"
9 #include "en/mod_hdr.h"
10 #include "sample.h"
11 #include "eswitch.h"
12 #include "en_tc.h"
13 #include "fs_core.h"
14
15 #define MLX5_ESW_VPORT_TBL_SIZE_SAMPLE (64 * 1024)
16
17 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_sample_ns = {
18 .max_fte = MLX5_ESW_VPORT_TBL_SIZE_SAMPLE,
19 .max_num_groups = 0, /* default num of groups */
20 .flags = 0,
21 };
22
23 struct mlx5e_tc_psample {
24 struct mlx5_eswitch *esw;
25 struct mlx5_flow_table *termtbl;
26 struct mlx5_flow_handle *termtbl_rule;
27 DECLARE_HASHTABLE(hashtbl, 8);
28 struct mutex ht_lock; /* protect hashtbl */
29 DECLARE_HASHTABLE(restore_hashtbl, 8);
30 struct mutex restore_lock; /* protect restore_hashtbl */
31 struct mlx5e_post_act *post_act;
32 };
33
34 struct mlx5e_sampler {
35 struct hlist_node hlist;
36 u32 sampler_id;
37 u32 sample_ratio;
38 u32 sample_table_id;
39 u32 default_table_id;
40 int count;
41 };
42
43 struct mlx5e_sample_flow {
44 struct mlx5e_sampler *sampler;
45 struct mlx5e_sample_restore *restore;
46 struct mlx5_flow_attr *pre_attr;
47 struct mlx5_flow_handle *pre_rule;
48 struct mlx5_flow_attr *post_attr;
49 struct mlx5_flow_handle *post_rule;
50 };
51
52 struct mlx5e_sample_restore {
53 struct hlist_node hlist;
54 struct mlx5_modify_hdr *modify_hdr;
55 struct mlx5_flow_handle *rule;
56 u32 obj_id;
57 int count;
58 };
59
60 static int
sampler_termtbl_create(struct mlx5e_tc_psample * tc_psample)61 sampler_termtbl_create(struct mlx5e_tc_psample *tc_psample)
62 {
63 struct mlx5_eswitch *esw = tc_psample->esw;
64 struct mlx5_flow_table_attr ft_attr = {};
65 struct mlx5_flow_destination dest = {};
66 struct mlx5_core_dev *dev = esw->dev;
67 struct mlx5_flow_namespace *root_ns;
68 struct mlx5_flow_act act = {};
69 int err;
70
71 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, termination_table)) {
72 mlx5_core_warn(dev, "termination table is not supported\n");
73 return -EOPNOTSUPP;
74 }
75
76 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
77 if (!root_ns) {
78 mlx5_core_warn(dev, "failed to get FDB flow namespace\n");
79 return -EOPNOTSUPP;
80 }
81
82 ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED;
83 ft_attr.autogroup.max_num_groups = 1;
84 ft_attr.prio = FDB_SLOW_PATH;
85 ft_attr.max_fte = 1;
86 ft_attr.level = 1;
87 tc_psample->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
88 if (IS_ERR(tc_psample->termtbl)) {
89 err = PTR_ERR(tc_psample->termtbl);
90 mlx5_core_warn(dev, "failed to create termtbl, err: %d\n", err);
91 return err;
92 }
93
94 act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
95 dest.vport.num = esw->manager_vport;
96 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
97 tc_psample->termtbl_rule = mlx5_add_flow_rules(tc_psample->termtbl, NULL, &act, &dest, 1);
98 if (IS_ERR(tc_psample->termtbl_rule)) {
99 err = PTR_ERR(tc_psample->termtbl_rule);
100 mlx5_core_warn(dev, "failed to create termtbl rule, err: %d\n", err);
101 mlx5_destroy_flow_table(tc_psample->termtbl);
102 return err;
103 }
104
105 return 0;
106 }
107
108 static void
sampler_termtbl_destroy(struct mlx5e_tc_psample * tc_psample)109 sampler_termtbl_destroy(struct mlx5e_tc_psample *tc_psample)
110 {
111 mlx5_del_flow_rules(tc_psample->termtbl_rule);
112 mlx5_destroy_flow_table(tc_psample->termtbl);
113 }
114
115 static int
sampler_obj_create(struct mlx5_core_dev * mdev,struct mlx5e_sampler * sampler)116 sampler_obj_create(struct mlx5_core_dev *mdev, struct mlx5e_sampler *sampler)
117 {
118 u32 in[MLX5_ST_SZ_DW(create_sampler_obj_in)] = {};
119 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
120 u64 general_obj_types;
121 void *obj;
122 int err;
123
124 general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
125 if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_SAMPLER))
126 return -EOPNOTSUPP;
127 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, ignore_flow_level))
128 return -EOPNOTSUPP;
129
130 obj = MLX5_ADDR_OF(create_sampler_obj_in, in, sampler_object);
131 MLX5_SET(sampler_obj, obj, table_type, FS_FT_FDB);
132 MLX5_SET(sampler_obj, obj, ignore_flow_level, 1);
133 MLX5_SET(sampler_obj, obj, level, 1);
134 MLX5_SET(sampler_obj, obj, sample_ratio, sampler->sample_ratio);
135 MLX5_SET(sampler_obj, obj, sample_table_id, sampler->sample_table_id);
136 MLX5_SET(sampler_obj, obj, default_table_id, sampler->default_table_id);
137 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
138 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_SAMPLER);
139
140 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
141 if (!err)
142 sampler->sampler_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
143
144 return err;
145 }
146
147 static void
sampler_obj_destroy(struct mlx5_core_dev * mdev,u32 sampler_id)148 sampler_obj_destroy(struct mlx5_core_dev *mdev, u32 sampler_id)
149 {
150 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
151 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
152
153 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
154 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_SAMPLER);
155 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sampler_id);
156
157 mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
158 }
159
160 static u32
sampler_hash(u32 sample_ratio,u32 default_table_id)161 sampler_hash(u32 sample_ratio, u32 default_table_id)
162 {
163 return jhash_2words(sample_ratio, default_table_id, 0);
164 }
165
166 static int
sampler_cmp(u32 sample_ratio1,u32 default_table_id1,u32 sample_ratio2,u32 default_table_id2)167 sampler_cmp(u32 sample_ratio1, u32 default_table_id1, u32 sample_ratio2, u32 default_table_id2)
168 {
169 return sample_ratio1 != sample_ratio2 || default_table_id1 != default_table_id2;
170 }
171
172 static struct mlx5e_sampler *
sampler_get(struct mlx5e_tc_psample * tc_psample,u32 sample_ratio,u32 default_table_id)173 sampler_get(struct mlx5e_tc_psample *tc_psample, u32 sample_ratio, u32 default_table_id)
174 {
175 struct mlx5e_sampler *sampler;
176 u32 hash_key;
177 int err;
178
179 mutex_lock(&tc_psample->ht_lock);
180 hash_key = sampler_hash(sample_ratio, default_table_id);
181 hash_for_each_possible(tc_psample->hashtbl, sampler, hlist, hash_key)
182 if (!sampler_cmp(sampler->sample_ratio, sampler->default_table_id,
183 sample_ratio, default_table_id))
184 goto add_ref;
185
186 sampler = kzalloc_obj(*sampler);
187 if (!sampler) {
188 err = -ENOMEM;
189 goto err_alloc;
190 }
191
192 sampler->sample_table_id = tc_psample->termtbl->id;
193 sampler->default_table_id = default_table_id;
194 sampler->sample_ratio = sample_ratio;
195
196 err = sampler_obj_create(tc_psample->esw->dev, sampler);
197 if (err)
198 goto err_create;
199
200 hash_add(tc_psample->hashtbl, &sampler->hlist, hash_key);
201
202 add_ref:
203 sampler->count++;
204 mutex_unlock(&tc_psample->ht_lock);
205 return sampler;
206
207 err_create:
208 kfree(sampler);
209 err_alloc:
210 mutex_unlock(&tc_psample->ht_lock);
211 return ERR_PTR(err);
212 }
213
214 static void
sampler_put(struct mlx5e_tc_psample * tc_psample,struct mlx5e_sampler * sampler)215 sampler_put(struct mlx5e_tc_psample *tc_psample, struct mlx5e_sampler *sampler)
216 {
217 mutex_lock(&tc_psample->ht_lock);
218 if (--sampler->count == 0) {
219 hash_del(&sampler->hlist);
220 sampler_obj_destroy(tc_psample->esw->dev, sampler->sampler_id);
221 kfree(sampler);
222 }
223 mutex_unlock(&tc_psample->ht_lock);
224 }
225
226 /* obj_id is used to restore the sample parameters.
227 * Set fte_id in original flow table, then match it in the default table.
228 * Only set it for NICs can preserve reg_c or decap action. For other cases,
229 * use the same match in the default table.
230 * Use one header rewrite for both obj_id and fte_id.
231 */
232 static struct mlx5_modify_hdr *
sample_modify_hdr_get(struct mlx5_core_dev * mdev,u32 obj_id,struct mlx5e_tc_mod_hdr_acts * mod_acts)233 sample_modify_hdr_get(struct mlx5_core_dev *mdev, u32 obj_id,
234 struct mlx5e_tc_mod_hdr_acts *mod_acts)
235 {
236 struct mlx5_modify_hdr *modify_hdr;
237 int err;
238
239 err = mlx5e_tc_match_to_reg_set(mdev, mod_acts, MLX5_FLOW_NAMESPACE_FDB,
240 MAPPED_OBJ_TO_REG, obj_id);
241 if (err)
242 goto err_set_regc0;
243
244 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_FDB,
245 mod_acts->num_actions,
246 mod_acts->actions);
247 if (IS_ERR(modify_hdr)) {
248 err = PTR_ERR(modify_hdr);
249 goto err_modify_hdr;
250 }
251
252 mlx5e_mod_hdr_dealloc(mod_acts);
253 return modify_hdr;
254
255 err_modify_hdr:
256 mlx5e_mod_hdr_dealloc(mod_acts);
257 err_set_regc0:
258 return ERR_PTR(err);
259 }
260
261 static struct mlx5e_sample_restore *
sample_restore_get(struct mlx5e_tc_psample * tc_psample,u32 obj_id,struct mlx5e_tc_mod_hdr_acts * mod_acts)262 sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id,
263 struct mlx5e_tc_mod_hdr_acts *mod_acts)
264 {
265 struct mlx5_eswitch *esw = tc_psample->esw;
266 struct mlx5_core_dev *mdev = esw->dev;
267 struct mlx5e_sample_restore *restore;
268 struct mlx5_modify_hdr *modify_hdr;
269 int err;
270
271 mutex_lock(&tc_psample->restore_lock);
272 hash_for_each_possible(tc_psample->restore_hashtbl, restore, hlist, obj_id)
273 if (restore->obj_id == obj_id)
274 goto add_ref;
275
276 restore = kzalloc_obj(*restore);
277 if (!restore) {
278 err = -ENOMEM;
279 goto err_alloc;
280 }
281 restore->obj_id = obj_id;
282
283 modify_hdr = sample_modify_hdr_get(mdev, obj_id, mod_acts);
284 if (IS_ERR(modify_hdr)) {
285 err = PTR_ERR(modify_hdr);
286 goto err_modify_hdr;
287 }
288 restore->modify_hdr = modify_hdr;
289
290 restore->rule = esw_add_restore_rule(esw, obj_id);
291 if (IS_ERR(restore->rule)) {
292 err = PTR_ERR(restore->rule);
293 goto err_restore;
294 }
295
296 hash_add(tc_psample->restore_hashtbl, &restore->hlist, obj_id);
297 add_ref:
298 restore->count++;
299 mutex_unlock(&tc_psample->restore_lock);
300 return restore;
301
302 err_restore:
303 mlx5_modify_header_dealloc(mdev, restore->modify_hdr);
304 err_modify_hdr:
305 kfree(restore);
306 err_alloc:
307 mutex_unlock(&tc_psample->restore_lock);
308 return ERR_PTR(err);
309 }
310
311 static void
sample_restore_put(struct mlx5e_tc_psample * tc_psample,struct mlx5e_sample_restore * restore)312 sample_restore_put(struct mlx5e_tc_psample *tc_psample, struct mlx5e_sample_restore *restore)
313 {
314 bool last;
315
316 mutex_lock(&tc_psample->restore_lock);
317 last = --restore->count == 0;
318 if (last)
319 hash_del(&restore->hlist);
320 mutex_unlock(&tc_psample->restore_lock);
321
322 if (last) {
323 mlx5_del_flow_rules(restore->rule);
324 mlx5_modify_header_dealloc(tc_psample->esw->dev, restore->modify_hdr);
325 kfree(restore);
326 }
327 }
328
mlx5e_tc_sample_skb(struct sk_buff * skb,struct mlx5_mapped_obj * mapped_obj)329 void mlx5e_tc_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj)
330 {
331 u32 trunc_size = mapped_obj->sample.trunc_size;
332 struct psample_group psample_group = {};
333 struct psample_metadata md = {};
334
335 md.trunc_size = trunc_size ? min(trunc_size, skb->len) : skb->len;
336 md.in_ifindex = skb->dev->ifindex;
337 psample_group.group_num = mapped_obj->sample.group_id;
338 psample_group.net = &init_net;
339 skb_push(skb, skb->mac_len);
340
341 psample_sample_packet(&psample_group, skb, mapped_obj->sample.rate, &md);
342 }
343
344 static int
add_post_rule(struct mlx5_eswitch * esw,struct mlx5e_sample_flow * sample_flow,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,u32 * default_tbl_id)345 add_post_rule(struct mlx5_eswitch *esw, struct mlx5e_sample_flow *sample_flow,
346 struct mlx5_flow_spec *spec, struct mlx5_flow_attr *attr,
347 u32 *default_tbl_id)
348 {
349 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
350 u32 attr_sz = ns_to_attr_sz(MLX5_FLOW_NAMESPACE_FDB);
351 struct mlx5_vport_tbl_attr per_vport_tbl_attr;
352 struct mlx5_flow_table *default_tbl;
353 struct mlx5_flow_attr *post_attr;
354 int err;
355
356 /* Allocate default table per vport, chain and prio. Otherwise, there is
357 * only one default table for the same sampler object. Rules with different
358 * prio and chain may overlap. For CT sample action, per vport default
359 * table is needed to resotre the metadata.
360 */
361 per_vport_tbl_attr.chain = attr->chain;
362 per_vport_tbl_attr.prio = attr->prio;
363 per_vport_tbl_attr.vport = esw_attr->in_rep->vport;
364 per_vport_tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
365 default_tbl = mlx5_esw_vporttbl_get(esw, &per_vport_tbl_attr);
366 if (IS_ERR(default_tbl)) {
367 err = PTR_ERR(default_tbl);
368 goto err_default_tbl;
369 }
370 *default_tbl_id = default_tbl->id;
371
372 post_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB);
373 if (!post_attr) {
374 err = -ENOMEM;
375 goto err_attr;
376 }
377 sample_flow->post_attr = post_attr;
378 memcpy(post_attr, attr, attr_sz);
379 /* Perform the original matches on the default table.
380 * Offload all actions except the sample action.
381 */
382 post_attr->chain = 0;
383 post_attr->prio = 0;
384 post_attr->ft = default_tbl;
385 post_attr->flags = MLX5_ATTR_FLAG_NO_IN_PORT;
386
387 /* When offloading sample and encap action, if there is no valid
388 * neigh data struct, a slow path rule is offloaded first. Source
389 * port metadata match is set at that time. A per vport table is
390 * already allocated. No need to match it again. So clear the source
391 * port metadata match.
392 */
393 mlx5_eswitch_clear_rule_source_port(esw, spec);
394 sample_flow->post_rule = mlx5_eswitch_add_offloaded_rule(esw, spec, post_attr);
395 if (IS_ERR(sample_flow->post_rule)) {
396 err = PTR_ERR(sample_flow->post_rule);
397 goto err_rule;
398 }
399 return 0;
400
401 err_rule:
402 kfree(post_attr);
403 err_attr:
404 mlx5_esw_vporttbl_put(esw, &per_vport_tbl_attr);
405 err_default_tbl:
406 return err;
407 }
408
409 static void
del_post_rule(struct mlx5_eswitch * esw,struct mlx5e_sample_flow * sample_flow,struct mlx5_flow_attr * attr)410 del_post_rule(struct mlx5_eswitch *esw, struct mlx5e_sample_flow *sample_flow,
411 struct mlx5_flow_attr *attr)
412 {
413 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
414 struct mlx5_vport_tbl_attr tbl_attr;
415
416 mlx5_eswitch_del_offloaded_rule(esw, sample_flow->post_rule, sample_flow->post_attr);
417 kfree(sample_flow->post_attr);
418 tbl_attr.chain = attr->chain;
419 tbl_attr.prio = attr->prio;
420 tbl_attr.vport = esw_attr->in_rep->vport;
421 tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
422 mlx5_esw_vporttbl_put(esw, &tbl_attr);
423 }
424
425 /* For the following typical flow table:
426 *
427 * +-------------------------------+
428 * + original flow table +
429 * +-------------------------------+
430 * + original match +
431 * +-------------------------------+
432 * + sample action + other actions +
433 * +-------------------------------+
434 *
435 * We translate the tc filter with sample action to the following HW model:
436 *
437 * +---------------------+
438 * + original flow table +
439 * +---------------------+
440 * + original match +
441 * +---------------------+
442 * | set fte_id (if reg_c preserve cap)
443 * | do decap (if required)
444 * v
445 * +------------------------------------------------+
446 * + Flow Sampler Object +
447 * +------------------------------------------------+
448 * + sample ratio +
449 * +------------------------------------------------+
450 * + sample table id | default table id +
451 * +------------------------------------------------+
452 * | |
453 * v v
454 * +-----------------------------+ +-------------------+
455 * + sample table + + default table +
456 * +-----------------------------+ +-------------------+
457 * + forward to management vport + |
458 * +-----------------------------+ |
459 * +-------+------+
460 * | |reg_c preserve cap
461 * | |or decap action
462 * v v
463 * +-----------------+ +-------------+
464 * + per vport table + + post action +
465 * +-----------------+ +-------------+
466 * + original match +
467 * +-----------------+
468 * + other actions +
469 * +-----------------+
470 */
471 struct mlx5_flow_handle *
mlx5e_tc_sample_offload(struct mlx5e_tc_psample * tc_psample,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)472 mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
473 struct mlx5_flow_spec *spec,
474 struct mlx5_flow_attr *attr)
475 {
476 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
477 struct mlx5_esw_flow_attr *pre_esw_attr;
478 struct mlx5_mapped_obj restore_obj = {};
479 struct mlx5e_tc_mod_hdr_acts *mod_acts;
480 struct mlx5e_sample_flow *sample_flow;
481 struct mlx5e_sample_attr *sample_attr;
482 struct mlx5_flow_attr *pre_attr;
483 struct mlx5_eswitch *esw;
484 u32 default_tbl_id;
485 u32 obj_id;
486 int err;
487
488 if (IS_ERR_OR_NULL(tc_psample))
489 return ERR_PTR(-EOPNOTSUPP);
490
491 sample_flow = kzalloc_obj(*sample_flow);
492 if (!sample_flow)
493 return ERR_PTR(-ENOMEM);
494 sample_attr = &attr->sample_attr;
495 sample_attr->sample_flow = sample_flow;
496
497 /* For NICs with reg_c_preserve support or decap action, use
498 * post action instead of the per vport, chain and prio table.
499 * Only match the fte id instead of the same match in the
500 * original flow table.
501 */
502 esw = tc_psample->esw;
503 if (mlx5e_tc_act_sample_is_multi_table(esw->dev, attr)) {
504 struct mlx5_flow_table *ft;
505
506 ft = mlx5e_tc_post_act_get_ft(tc_psample->post_act);
507 default_tbl_id = ft->id;
508 } else {
509 err = add_post_rule(esw, sample_flow, spec, attr, &default_tbl_id);
510 if (err)
511 goto err_post_rule;
512 }
513
514 /* Create sampler object. */
515 sample_flow->sampler = sampler_get(tc_psample, sample_attr->rate, default_tbl_id);
516 if (IS_ERR(sample_flow->sampler)) {
517 err = PTR_ERR(sample_flow->sampler);
518 goto err_sampler;
519 }
520 sample_attr->sampler_id = sample_flow->sampler->sampler_id;
521
522 /* Create an id mapping reg_c0 value to sample object. */
523 restore_obj.type = MLX5_MAPPED_OBJ_SAMPLE;
524 restore_obj.sample.group_id = sample_attr->group_num;
525 restore_obj.sample.rate = sample_attr->rate;
526 restore_obj.sample.trunc_size = sample_attr->trunc_size;
527 restore_obj.sample.tunnel_id = attr->tunnel_id;
528 err = mapping_add(esw->offloads.reg_c0_obj_pool, &restore_obj, &obj_id);
529 if (err)
530 goto err_obj_id;
531 sample_attr->restore_obj_id = obj_id;
532
533 /* Create sample restore context. */
534 mod_acts = &attr->parse_attr->mod_hdr_acts;
535 sample_flow->restore = sample_restore_get(tc_psample, obj_id, mod_acts);
536 if (IS_ERR(sample_flow->restore)) {
537 err = PTR_ERR(sample_flow->restore);
538 goto err_sample_restore;
539 }
540
541 /* Perform the original matches on the original table. Offload the
542 * sample action. The destination is the sampler object.
543 */
544 pre_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB);
545 if (!pre_attr) {
546 err = -ENOMEM;
547 goto err_alloc_pre_flow_attr;
548 }
549 pre_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
550 /* For decap action, do decap in the original flow table instead of the
551 * default flow table.
552 */
553 if (attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
554 pre_attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
555 pre_attr->modify_hdr = sample_flow->restore->modify_hdr;
556 pre_attr->flags = MLX5_ATTR_FLAG_SAMPLE;
557 pre_attr->inner_match_level = attr->inner_match_level;
558 pre_attr->outer_match_level = attr->outer_match_level;
559 pre_attr->chain = attr->chain;
560 pre_attr->prio = attr->prio;
561 pre_attr->ft = attr->ft;
562 pre_attr->sample_attr = *sample_attr;
563 pre_esw_attr = pre_attr->esw_attr;
564 pre_esw_attr->in_mdev = esw_attr->in_mdev;
565 pre_esw_attr->in_rep = esw_attr->in_rep;
566 sample_flow->pre_rule = mlx5_eswitch_add_offloaded_rule(esw, spec, pre_attr);
567 if (IS_ERR(sample_flow->pre_rule)) {
568 err = PTR_ERR(sample_flow->pre_rule);
569 goto err_pre_offload_rule;
570 }
571 sample_flow->pre_attr = pre_attr;
572
573 return sample_flow->pre_rule;
574
575 err_pre_offload_rule:
576 kfree(pre_attr);
577 err_alloc_pre_flow_attr:
578 sample_restore_put(tc_psample, sample_flow->restore);
579 err_sample_restore:
580 mapping_remove(esw->offloads.reg_c0_obj_pool, obj_id);
581 err_obj_id:
582 sampler_put(tc_psample, sample_flow->sampler);
583 err_sampler:
584 if (sample_flow->post_rule)
585 del_post_rule(esw, sample_flow, attr);
586 err_post_rule:
587 kfree(sample_flow);
588 return ERR_PTR(err);
589 }
590
591 void
mlx5e_tc_sample_unoffload(struct mlx5e_tc_psample * tc_psample,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)592 mlx5e_tc_sample_unoffload(struct mlx5e_tc_psample *tc_psample,
593 struct mlx5_flow_handle *rule,
594 struct mlx5_flow_attr *attr)
595 {
596 struct mlx5e_sample_flow *sample_flow;
597 struct mlx5_eswitch *esw;
598
599 if (IS_ERR_OR_NULL(tc_psample))
600 return;
601
602 /* The following delete order can't be changed, otherwise,
603 * will hit fw syndromes.
604 */
605 esw = tc_psample->esw;
606 sample_flow = attr->sample_attr.sample_flow;
607 mlx5_eswitch_del_offloaded_rule(esw, sample_flow->pre_rule, sample_flow->pre_attr);
608
609 sample_restore_put(tc_psample, sample_flow->restore);
610 mapping_remove(esw->offloads.reg_c0_obj_pool, attr->sample_attr.restore_obj_id);
611 sampler_put(tc_psample, sample_flow->sampler);
612 if (sample_flow->post_rule)
613 del_post_rule(esw, sample_flow, attr);
614
615 kfree(sample_flow->pre_attr);
616 kfree(sample_flow);
617 }
618
619 struct mlx5e_tc_psample *
mlx5e_tc_sample_init(struct mlx5_eswitch * esw,struct mlx5e_post_act * post_act)620 mlx5e_tc_sample_init(struct mlx5_eswitch *esw, struct mlx5e_post_act *post_act)
621 {
622 struct mlx5e_tc_psample *tc_psample;
623 int err;
624
625 tc_psample = kzalloc_obj(*tc_psample);
626 if (!tc_psample)
627 return ERR_PTR(-ENOMEM);
628 if (IS_ERR_OR_NULL(post_act)) {
629 err = PTR_ERR(post_act);
630 goto err_post_act;
631 }
632 tc_psample->post_act = post_act;
633 tc_psample->esw = esw;
634 err = sampler_termtbl_create(tc_psample);
635 if (err)
636 goto err_post_act;
637
638 mutex_init(&tc_psample->ht_lock);
639 mutex_init(&tc_psample->restore_lock);
640
641 return tc_psample;
642
643 err_post_act:
644 kfree(tc_psample);
645 return ERR_PTR(err);
646 }
647
648 void
mlx5e_tc_sample_cleanup(struct mlx5e_tc_psample * tc_psample)649 mlx5e_tc_sample_cleanup(struct mlx5e_tc_psample *tc_psample)
650 {
651 if (IS_ERR_OR_NULL(tc_psample))
652 return;
653
654 mutex_destroy(&tc_psample->restore_lock);
655 mutex_destroy(&tc_psample->ht_lock);
656 sampler_termtbl_destroy(tc_psample);
657 kfree(tc_psample);
658 }
659