1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2024 NVIDIA Corporation & Affiliates */
3
4 #include "mlx5hws_internal.h"
5
6 enum mlx5hws_matcher_rtc_type {
7 HWS_MATCHER_RTC_TYPE_MATCH,
8 HWS_MATCHER_RTC_TYPE_STE_ARRAY,
9 HWS_MATCHER_RTC_TYPE_MAX,
10 };
11
12 static const char * const mlx5hws_matcher_rtc_type_str[] = {
13 [HWS_MATCHER_RTC_TYPE_MATCH] = "MATCH",
14 [HWS_MATCHER_RTC_TYPE_STE_ARRAY] = "STE_ARRAY",
15 [HWS_MATCHER_RTC_TYPE_MAX] = "UNKNOWN",
16 };
17
hws_matcher_rtc_type_to_str(enum mlx5hws_matcher_rtc_type rtc_type)18 static const char *hws_matcher_rtc_type_to_str(enum mlx5hws_matcher_rtc_type rtc_type)
19 {
20 if (rtc_type > HWS_MATCHER_RTC_TYPE_MAX)
21 rtc_type = HWS_MATCHER_RTC_TYPE_MAX;
22 return mlx5hws_matcher_rtc_type_str[rtc_type];
23 }
24
hws_matcher_requires_col_tbl(u8 log_num_of_rules)25 static bool hws_matcher_requires_col_tbl(u8 log_num_of_rules)
26 {
27 /* Collision table concatenation is done only for large rule tables */
28 return log_num_of_rules > MLX5HWS_MATCHER_ASSURED_RULES_TH;
29 }
30
hws_matcher_rules_to_tbl_depth(u8 log_num_of_rules)31 static u8 hws_matcher_rules_to_tbl_depth(u8 log_num_of_rules)
32 {
33 if (hws_matcher_requires_col_tbl(log_num_of_rules))
34 return MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH;
35
36 /* For small rule tables we use a single deep table to assure insertion */
37 return min(log_num_of_rules, MLX5HWS_MATCHER_ASSURED_COL_TBL_DEPTH);
38 }
39
hws_matcher_destroy_end_ft(struct mlx5hws_matcher * matcher)40 static void hws_matcher_destroy_end_ft(struct mlx5hws_matcher *matcher)
41 {
42 mlx5hws_table_destroy_default_ft(matcher->tbl, matcher->end_ft_id);
43 }
44
hws_matcher_create_end_ft(struct mlx5hws_matcher * matcher)45 static int hws_matcher_create_end_ft(struct mlx5hws_matcher *matcher)
46 {
47 struct mlx5hws_table *tbl = matcher->tbl;
48 int ret;
49
50 ret = mlx5hws_table_create_default_ft(tbl->ctx->mdev, tbl, &matcher->end_ft_id);
51 if (ret) {
52 mlx5hws_err(tbl->ctx, "Failed to create matcher end flow table\n");
53 return ret;
54 }
55 return 0;
56 }
57
hws_matcher_connect(struct mlx5hws_matcher * matcher)58 static int hws_matcher_connect(struct mlx5hws_matcher *matcher)
59 {
60 struct mlx5hws_table *tbl = matcher->tbl;
61 struct mlx5hws_context *ctx = tbl->ctx;
62 struct mlx5hws_matcher *prev = NULL;
63 struct mlx5hws_matcher *next = NULL;
64 struct mlx5hws_matcher *tmp_matcher;
65 int ret;
66
67 /* Find location in matcher list */
68 if (list_empty(&tbl->matchers_list)) {
69 list_add(&matcher->list_node, &tbl->matchers_list);
70 goto connect;
71 }
72
73 list_for_each_entry(tmp_matcher, &tbl->matchers_list, list_node) {
74 if (tmp_matcher->attr.priority > matcher->attr.priority) {
75 next = tmp_matcher;
76 break;
77 }
78 prev = tmp_matcher;
79 }
80
81 if (next)
82 /* insert before next */
83 list_add_tail(&matcher->list_node, &next->list_node);
84 else
85 /* insert after prev */
86 list_add(&matcher->list_node, &prev->list_node);
87
88 connect:
89 if (next) {
90 /* Connect to next RTC */
91 ret = mlx5hws_table_ft_set_next_rtc(ctx,
92 matcher->end_ft_id,
93 tbl->fw_ft_type,
94 next->match_ste.rtc_0_id,
95 next->match_ste.rtc_1_id);
96 if (ret) {
97 mlx5hws_err(ctx, "Failed to connect new matcher to next RTC\n");
98 goto remove_from_list;
99 }
100 } else {
101 /* Connect last matcher to next miss_tbl if exists */
102 ret = mlx5hws_table_connect_to_miss_table(tbl, tbl->default_miss.miss_tbl);
103 if (ret) {
104 mlx5hws_err(ctx, "Failed connect new matcher to miss_tbl\n");
105 goto remove_from_list;
106 }
107 }
108
109 /* Connect to previous FT */
110 ret = mlx5hws_table_ft_set_next_rtc(ctx,
111 prev ? prev->end_ft_id : tbl->ft_id,
112 tbl->fw_ft_type,
113 matcher->match_ste.rtc_0_id,
114 matcher->match_ste.rtc_1_id);
115 if (ret) {
116 mlx5hws_err(ctx, "Failed to connect new matcher to previous FT\n");
117 goto remove_from_list;
118 }
119
120 /* Reset prev matcher FT default miss (drop refcount) */
121 ret = mlx5hws_table_ft_set_default_next_ft(tbl, prev ? prev->end_ft_id : tbl->ft_id);
122 if (ret) {
123 mlx5hws_err(ctx, "Failed to reset matcher ft default miss\n");
124 goto remove_from_list;
125 }
126
127 if (!prev) {
128 /* Update tables missing to current matcher in the table */
129 ret = mlx5hws_table_update_connected_miss_tables(tbl);
130 if (ret) {
131 mlx5hws_err(ctx, "Fatal error, failed to update connected miss table\n");
132 goto remove_from_list;
133 }
134 }
135
136 return 0;
137
138 remove_from_list:
139 list_del_init(&matcher->list_node);
140 return ret;
141 }
142
hws_matcher_disconnect(struct mlx5hws_matcher * matcher)143 static int hws_matcher_disconnect(struct mlx5hws_matcher *matcher)
144 {
145 struct mlx5hws_matcher *next = NULL, *prev = NULL;
146 struct mlx5hws_table *tbl = matcher->tbl;
147 u32 prev_ft_id = tbl->ft_id;
148 int ret;
149
150 if (!list_is_first(&matcher->list_node, &tbl->matchers_list)) {
151 prev = list_prev_entry(matcher, list_node);
152 prev_ft_id = prev->end_ft_id;
153 }
154
155 if (!list_is_last(&matcher->list_node, &tbl->matchers_list))
156 next = list_next_entry(matcher, list_node);
157
158 list_del_init(&matcher->list_node);
159
160 if (next) {
161 /* Connect previous end FT to next RTC */
162 ret = mlx5hws_table_ft_set_next_rtc(tbl->ctx,
163 prev_ft_id,
164 tbl->fw_ft_type,
165 next->match_ste.rtc_0_id,
166 next->match_ste.rtc_1_id);
167 if (ret) {
168 mlx5hws_err(tbl->ctx, "Failed to disconnect matcher\n");
169 goto matcher_reconnect;
170 }
171 } else {
172 ret = mlx5hws_table_connect_to_miss_table(tbl, tbl->default_miss.miss_tbl);
173 if (ret) {
174 mlx5hws_err(tbl->ctx, "Failed to disconnect last matcher\n");
175 goto matcher_reconnect;
176 }
177 }
178
179 /* Removing first matcher, update connected miss tables if exists */
180 if (prev_ft_id == tbl->ft_id) {
181 ret = mlx5hws_table_update_connected_miss_tables(tbl);
182 if (ret) {
183 mlx5hws_err(tbl->ctx, "Fatal error, failed to update connected miss table\n");
184 goto matcher_reconnect;
185 }
186 }
187
188 ret = mlx5hws_table_ft_set_default_next_ft(tbl, prev_ft_id);
189 if (ret) {
190 mlx5hws_err(tbl->ctx, "Fatal error, failed to restore matcher ft default miss\n");
191 goto matcher_reconnect;
192 }
193
194 return 0;
195
196 matcher_reconnect:
197 if (list_empty(&tbl->matchers_list) || !prev)
198 list_add(&matcher->list_node, &tbl->matchers_list);
199 else
200 /* insert after prev matcher */
201 list_add(&matcher->list_node, &prev->list_node);
202
203 return ret;
204 }
205
hws_matcher_set_rtc_attr_sz(struct mlx5hws_matcher * matcher,struct mlx5hws_cmd_rtc_create_attr * rtc_attr,enum mlx5hws_matcher_rtc_type rtc_type,bool is_mirror)206 static void hws_matcher_set_rtc_attr_sz(struct mlx5hws_matcher *matcher,
207 struct mlx5hws_cmd_rtc_create_attr *rtc_attr,
208 enum mlx5hws_matcher_rtc_type rtc_type,
209 bool is_mirror)
210 {
211 struct mlx5hws_pool_chunk *ste = &matcher->action_ste[MLX5HWS_ACTION_STE_IDX_ANY].ste;
212 enum mlx5hws_matcher_flow_src flow_src = matcher->attr.optimize_flow_src;
213 bool is_match_rtc = rtc_type == HWS_MATCHER_RTC_TYPE_MATCH;
214
215 if ((flow_src == MLX5HWS_MATCHER_FLOW_SRC_VPORT && !is_mirror) ||
216 (flow_src == MLX5HWS_MATCHER_FLOW_SRC_WIRE && is_mirror)) {
217 /* Optimize FDB RTC */
218 rtc_attr->log_size = 0;
219 rtc_attr->log_depth = 0;
220 } else {
221 /* Keep original values */
222 rtc_attr->log_size = is_match_rtc ? matcher->attr.table.sz_row_log : ste->order;
223 rtc_attr->log_depth = is_match_rtc ? matcher->attr.table.sz_col_log : 0;
224 }
225 }
226
hws_matcher_create_rtc(struct mlx5hws_matcher * matcher,enum mlx5hws_matcher_rtc_type rtc_type,u8 action_ste_selector)227 static int hws_matcher_create_rtc(struct mlx5hws_matcher *matcher,
228 enum mlx5hws_matcher_rtc_type rtc_type,
229 u8 action_ste_selector)
230 {
231 struct mlx5hws_matcher_attr *attr = &matcher->attr;
232 struct mlx5hws_cmd_rtc_create_attr rtc_attr = {0};
233 struct mlx5hws_match_template *mt = matcher->mt;
234 struct mlx5hws_context *ctx = matcher->tbl->ctx;
235 struct mlx5hws_action_default_stc *default_stc;
236 struct mlx5hws_matcher_action_ste *action_ste;
237 struct mlx5hws_table *tbl = matcher->tbl;
238 struct mlx5hws_pool *ste_pool, *stc_pool;
239 struct mlx5hws_pool_chunk *ste;
240 u32 *rtc_0_id, *rtc_1_id;
241 u32 obj_id;
242 int ret;
243
244 switch (rtc_type) {
245 case HWS_MATCHER_RTC_TYPE_MATCH:
246 rtc_0_id = &matcher->match_ste.rtc_0_id;
247 rtc_1_id = &matcher->match_ste.rtc_1_id;
248 ste_pool = matcher->match_ste.pool;
249 ste = &matcher->match_ste.ste;
250 ste->order = attr->table.sz_col_log + attr->table.sz_row_log;
251
252 rtc_attr.log_size = attr->table.sz_row_log;
253 rtc_attr.log_depth = attr->table.sz_col_log;
254 rtc_attr.is_frst_jumbo = mlx5hws_matcher_mt_is_jumbo(mt);
255 rtc_attr.is_scnd_range = 0;
256 rtc_attr.miss_ft_id = matcher->end_ft_id;
257
258 if (attr->insert_mode == MLX5HWS_MATCHER_INSERT_BY_HASH) {
259 /* The usual Hash Table */
260 rtc_attr.update_index_mode = MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH;
261
262 /* The first mt is used since all share the same definer */
263 rtc_attr.match_definer_0 = mlx5hws_definer_get_id(mt->definer);
264 } else if (attr->insert_mode == MLX5HWS_MATCHER_INSERT_BY_INDEX) {
265 rtc_attr.update_index_mode = MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET;
266 rtc_attr.num_hash_definer = 1;
267
268 if (attr->distribute_mode == MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH) {
269 /* Hash Split Table */
270 rtc_attr.access_index_mode = MLX5_IFC_RTC_STE_ACCESS_MODE_BY_HASH;
271 rtc_attr.match_definer_0 = mlx5hws_definer_get_id(mt->definer);
272 } else if (attr->distribute_mode == MLX5HWS_MATCHER_DISTRIBUTE_BY_LINEAR) {
273 /* Linear Lookup Table */
274 rtc_attr.access_index_mode = MLX5_IFC_RTC_STE_ACCESS_MODE_LINEAR;
275 rtc_attr.match_definer_0 = ctx->caps->linear_match_definer;
276 }
277 }
278
279 /* Match pool requires implicit allocation */
280 ret = mlx5hws_pool_chunk_alloc(ste_pool, ste);
281 if (ret) {
282 mlx5hws_err(ctx, "Failed to allocate STE for %s RTC",
283 hws_matcher_rtc_type_to_str(rtc_type));
284 return ret;
285 }
286 break;
287
288 case HWS_MATCHER_RTC_TYPE_STE_ARRAY:
289 action_ste = &matcher->action_ste[action_ste_selector];
290
291 rtc_0_id = &action_ste->rtc_0_id;
292 rtc_1_id = &action_ste->rtc_1_id;
293 ste_pool = action_ste->pool;
294 ste = &action_ste->ste;
295 ste->order = ilog2(roundup_pow_of_two(action_ste->max_stes)) +
296 attr->table.sz_row_log;
297 rtc_attr.log_size = ste->order;
298 rtc_attr.log_depth = 0;
299 rtc_attr.update_index_mode = MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET;
300 /* The action STEs use the default always hit definer */
301 rtc_attr.match_definer_0 = ctx->caps->trivial_match_definer;
302 rtc_attr.is_frst_jumbo = false;
303 rtc_attr.miss_ft_id = 0;
304 break;
305
306 default:
307 mlx5hws_err(ctx, "HWS Invalid RTC type\n");
308 return -EINVAL;
309 }
310
311 obj_id = mlx5hws_pool_chunk_get_base_id(ste_pool, ste);
312
313 rtc_attr.pd = ctx->pd_num;
314 rtc_attr.ste_base = obj_id;
315 rtc_attr.ste_offset = ste->offset;
316 rtc_attr.reparse_mode = mlx5hws_context_get_reparse_mode(ctx);
317 rtc_attr.table_type = mlx5hws_table_get_res_fw_ft_type(tbl->type, false);
318 hws_matcher_set_rtc_attr_sz(matcher, &rtc_attr, rtc_type, false);
319
320 /* STC is a single resource (obj_id), use any STC for the ID */
321 stc_pool = ctx->stc_pool[tbl->type];
322 default_stc = ctx->common_res[tbl->type].default_stc;
323 obj_id = mlx5hws_pool_chunk_get_base_id(stc_pool, &default_stc->default_hit);
324 rtc_attr.stc_base = obj_id;
325
326 ret = mlx5hws_cmd_rtc_create(ctx->mdev, &rtc_attr, rtc_0_id);
327 if (ret) {
328 mlx5hws_err(ctx, "Failed to create matcher RTC of type %s",
329 hws_matcher_rtc_type_to_str(rtc_type));
330 goto free_ste;
331 }
332
333 if (tbl->type == MLX5HWS_TABLE_TYPE_FDB) {
334 obj_id = mlx5hws_pool_chunk_get_base_mirror_id(ste_pool, ste);
335 rtc_attr.ste_base = obj_id;
336 rtc_attr.table_type = mlx5hws_table_get_res_fw_ft_type(tbl->type, true);
337
338 obj_id = mlx5hws_pool_chunk_get_base_mirror_id(stc_pool, &default_stc->default_hit);
339 rtc_attr.stc_base = obj_id;
340 hws_matcher_set_rtc_attr_sz(matcher, &rtc_attr, rtc_type, true);
341
342 ret = mlx5hws_cmd_rtc_create(ctx->mdev, &rtc_attr, rtc_1_id);
343 if (ret) {
344 mlx5hws_err(ctx, "Failed to create peer matcher RTC of type %s",
345 hws_matcher_rtc_type_to_str(rtc_type));
346 goto destroy_rtc_0;
347 }
348 }
349
350 return 0;
351
352 destroy_rtc_0:
353 mlx5hws_cmd_rtc_destroy(ctx->mdev, *rtc_0_id);
354 free_ste:
355 if (rtc_type == HWS_MATCHER_RTC_TYPE_MATCH)
356 mlx5hws_pool_chunk_free(ste_pool, ste);
357 return ret;
358 }
359
hws_matcher_destroy_rtc(struct mlx5hws_matcher * matcher,enum mlx5hws_matcher_rtc_type rtc_type,u8 action_ste_selector)360 static void hws_matcher_destroy_rtc(struct mlx5hws_matcher *matcher,
361 enum mlx5hws_matcher_rtc_type rtc_type,
362 u8 action_ste_selector)
363 {
364 struct mlx5hws_matcher_action_ste *action_ste;
365 struct mlx5hws_table *tbl = matcher->tbl;
366 struct mlx5hws_pool_chunk *ste;
367 struct mlx5hws_pool *ste_pool;
368 u32 rtc_0_id, rtc_1_id;
369
370 switch (rtc_type) {
371 case HWS_MATCHER_RTC_TYPE_MATCH:
372 rtc_0_id = matcher->match_ste.rtc_0_id;
373 rtc_1_id = matcher->match_ste.rtc_1_id;
374 ste_pool = matcher->match_ste.pool;
375 ste = &matcher->match_ste.ste;
376 break;
377 case HWS_MATCHER_RTC_TYPE_STE_ARRAY:
378 action_ste = &matcher->action_ste[action_ste_selector];
379 rtc_0_id = action_ste->rtc_0_id;
380 rtc_1_id = action_ste->rtc_1_id;
381 ste_pool = action_ste->pool;
382 ste = &action_ste->ste;
383 break;
384 default:
385 return;
386 }
387
388 if (tbl->type == MLX5HWS_TABLE_TYPE_FDB)
389 mlx5hws_cmd_rtc_destroy(matcher->tbl->ctx->mdev, rtc_1_id);
390
391 mlx5hws_cmd_rtc_destroy(matcher->tbl->ctx->mdev, rtc_0_id);
392 if (rtc_type == HWS_MATCHER_RTC_TYPE_MATCH)
393 mlx5hws_pool_chunk_free(ste_pool, ste);
394 }
395
396 static int
hws_matcher_check_attr_sz(struct mlx5hws_cmd_query_caps * caps,struct mlx5hws_matcher * matcher)397 hws_matcher_check_attr_sz(struct mlx5hws_cmd_query_caps *caps,
398 struct mlx5hws_matcher *matcher)
399 {
400 struct mlx5hws_matcher_attr *attr = &matcher->attr;
401
402 if (attr->table.sz_col_log > caps->rtc_log_depth_max) {
403 mlx5hws_err(matcher->tbl->ctx, "Matcher depth exceeds limit %d\n",
404 caps->rtc_log_depth_max);
405 return -EOPNOTSUPP;
406 }
407
408 if (attr->table.sz_col_log + attr->table.sz_row_log > caps->ste_alloc_log_max) {
409 mlx5hws_err(matcher->tbl->ctx, "Total matcher size exceeds limit %d\n",
410 caps->ste_alloc_log_max);
411 return -EOPNOTSUPP;
412 }
413
414 if (attr->table.sz_col_log + attr->table.sz_row_log < caps->ste_alloc_log_gran) {
415 mlx5hws_err(matcher->tbl->ctx, "Total matcher size below limit %d\n",
416 caps->ste_alloc_log_gran);
417 return -EOPNOTSUPP;
418 }
419
420 return 0;
421 }
422
hws_matcher_set_pool_attr(struct mlx5hws_pool_attr * attr,struct mlx5hws_matcher * matcher)423 static void hws_matcher_set_pool_attr(struct mlx5hws_pool_attr *attr,
424 struct mlx5hws_matcher *matcher)
425 {
426 switch (matcher->attr.optimize_flow_src) {
427 case MLX5HWS_MATCHER_FLOW_SRC_VPORT:
428 attr->opt_type = MLX5HWS_POOL_OPTIMIZE_ORIG;
429 break;
430 case MLX5HWS_MATCHER_FLOW_SRC_WIRE:
431 attr->opt_type = MLX5HWS_POOL_OPTIMIZE_MIRROR;
432 break;
433 default:
434 break;
435 }
436 }
437
hws_matcher_check_and_process_at(struct mlx5hws_matcher * matcher,struct mlx5hws_action_template * at)438 static int hws_matcher_check_and_process_at(struct mlx5hws_matcher *matcher,
439 struct mlx5hws_action_template *at)
440 {
441 struct mlx5hws_context *ctx = matcher->tbl->ctx;
442 bool valid;
443 int ret;
444
445 valid = mlx5hws_action_check_combo(ctx, at->action_type_arr, matcher->tbl->type);
446 if (!valid) {
447 mlx5hws_err(ctx, "Invalid combination in action template\n");
448 return -EINVAL;
449 }
450
451 /* Process action template to setters */
452 ret = mlx5hws_action_template_process(at);
453 if (ret) {
454 mlx5hws_err(ctx, "Failed to process action template\n");
455 return ret;
456 }
457
458 return 0;
459 }
460
hws_matcher_resize_init(struct mlx5hws_matcher * src_matcher)461 static int hws_matcher_resize_init(struct mlx5hws_matcher *src_matcher)
462 {
463 struct mlx5hws_matcher_resize_data *resize_data;
464
465 resize_data = kzalloc(sizeof(*resize_data), GFP_KERNEL);
466 if (!resize_data)
467 return -ENOMEM;
468
469 resize_data->max_stes = src_matcher->action_ste[MLX5HWS_ACTION_STE_IDX_ANY].max_stes;
470
471 resize_data->action_ste[0].stc = src_matcher->action_ste[0].stc;
472 resize_data->action_ste[0].rtc_0_id = src_matcher->action_ste[0].rtc_0_id;
473 resize_data->action_ste[0].rtc_1_id = src_matcher->action_ste[0].rtc_1_id;
474 resize_data->action_ste[0].pool = src_matcher->action_ste[0].max_stes ?
475 src_matcher->action_ste[0].pool :
476 NULL;
477 resize_data->action_ste[1].stc = src_matcher->action_ste[1].stc;
478 resize_data->action_ste[1].rtc_0_id = src_matcher->action_ste[1].rtc_0_id;
479 resize_data->action_ste[1].rtc_1_id = src_matcher->action_ste[1].rtc_1_id;
480 resize_data->action_ste[1].pool = src_matcher->action_ste[1].max_stes ?
481 src_matcher->action_ste[1].pool :
482 NULL;
483
484 /* Place the new resized matcher on the dst matcher's list */
485 list_add(&resize_data->list_node, &src_matcher->resize_dst->resize_data);
486
487 /* Move all the previous resized matchers to the dst matcher's list */
488 while (!list_empty(&src_matcher->resize_data)) {
489 resize_data = list_first_entry(&src_matcher->resize_data,
490 struct mlx5hws_matcher_resize_data,
491 list_node);
492 list_del_init(&resize_data->list_node);
493 list_add(&resize_data->list_node, &src_matcher->resize_dst->resize_data);
494 }
495
496 return 0;
497 }
498
hws_matcher_resize_uninit(struct mlx5hws_matcher * matcher)499 static void hws_matcher_resize_uninit(struct mlx5hws_matcher *matcher)
500 {
501 struct mlx5hws_matcher_resize_data *resize_data;
502
503 if (!mlx5hws_matcher_is_resizable(matcher))
504 return;
505
506 while (!list_empty(&matcher->resize_data)) {
507 resize_data = list_first_entry(&matcher->resize_data,
508 struct mlx5hws_matcher_resize_data,
509 list_node);
510 list_del_init(&resize_data->list_node);
511
512 if (resize_data->max_stes) {
513 mlx5hws_action_free_single_stc(matcher->tbl->ctx,
514 matcher->tbl->type,
515 &resize_data->action_ste[1].stc);
516 mlx5hws_action_free_single_stc(matcher->tbl->ctx,
517 matcher->tbl->type,
518 &resize_data->action_ste[0].stc);
519
520 if (matcher->tbl->type == MLX5HWS_TABLE_TYPE_FDB) {
521 mlx5hws_cmd_rtc_destroy(matcher->tbl->ctx->mdev,
522 resize_data->action_ste[1].rtc_1_id);
523 mlx5hws_cmd_rtc_destroy(matcher->tbl->ctx->mdev,
524 resize_data->action_ste[0].rtc_1_id);
525 }
526 mlx5hws_cmd_rtc_destroy(matcher->tbl->ctx->mdev,
527 resize_data->action_ste[1].rtc_0_id);
528 mlx5hws_cmd_rtc_destroy(matcher->tbl->ctx->mdev,
529 resize_data->action_ste[0].rtc_0_id);
530 if (resize_data->action_ste[MLX5HWS_ACTION_STE_IDX_ANY].pool) {
531 mlx5hws_pool_destroy(resize_data->action_ste[1].pool);
532 mlx5hws_pool_destroy(resize_data->action_ste[0].pool);
533 }
534 }
535
536 kfree(resize_data);
537 }
538 }
539
540 static int
hws_matcher_bind_at_idx(struct mlx5hws_matcher * matcher,u8 action_ste_selector)541 hws_matcher_bind_at_idx(struct mlx5hws_matcher *matcher, u8 action_ste_selector)
542 {
543 struct mlx5hws_cmd_stc_modify_attr stc_attr = {0};
544 struct mlx5hws_matcher_action_ste *action_ste;
545 struct mlx5hws_table *tbl = matcher->tbl;
546 struct mlx5hws_pool_attr pool_attr = {0};
547 struct mlx5hws_context *ctx = tbl->ctx;
548 int ret;
549
550 action_ste = &matcher->action_ste[action_ste_selector];
551
552 /* Allocate action STE mempool */
553 pool_attr.table_type = tbl->type;
554 pool_attr.pool_type = MLX5HWS_POOL_TYPE_STE;
555 pool_attr.flags = MLX5HWS_POOL_FLAGS_FOR_STE_ACTION_POOL;
556 pool_attr.alloc_log_sz = ilog2(roundup_pow_of_two(action_ste->max_stes)) +
557 matcher->attr.table.sz_row_log;
558 hws_matcher_set_pool_attr(&pool_attr, matcher);
559 action_ste->pool = mlx5hws_pool_create(ctx, &pool_attr);
560 if (!action_ste->pool) {
561 mlx5hws_err(ctx, "Failed to create action ste pool\n");
562 return -EINVAL;
563 }
564
565 /* Allocate action RTC */
566 ret = hws_matcher_create_rtc(matcher, HWS_MATCHER_RTC_TYPE_STE_ARRAY, action_ste_selector);
567 if (ret) {
568 mlx5hws_err(ctx, "Failed to create action RTC\n");
569 goto free_ste_pool;
570 }
571
572 /* Allocate STC for jumps to STE */
573 stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_HIT;
574 stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_STE_TABLE;
575 stc_attr.reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
576 stc_attr.ste_table.ste = action_ste->ste;
577 stc_attr.ste_table.ste_pool = action_ste->pool;
578 stc_attr.ste_table.match_definer_id = ctx->caps->trivial_match_definer;
579
580 ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, tbl->type,
581 &action_ste->stc);
582 if (ret) {
583 mlx5hws_err(ctx, "Failed to create action jump to table STC\n");
584 goto free_rtc;
585 }
586
587 return 0;
588
589 free_rtc:
590 hws_matcher_destroy_rtc(matcher, HWS_MATCHER_RTC_TYPE_STE_ARRAY, action_ste_selector);
591 free_ste_pool:
592 mlx5hws_pool_destroy(action_ste->pool);
593 return ret;
594 }
595
hws_matcher_unbind_at_idx(struct mlx5hws_matcher * matcher,u8 action_ste_selector)596 static void hws_matcher_unbind_at_idx(struct mlx5hws_matcher *matcher, u8 action_ste_selector)
597 {
598 struct mlx5hws_matcher_action_ste *action_ste;
599 struct mlx5hws_table *tbl = matcher->tbl;
600
601 action_ste = &matcher->action_ste[action_ste_selector];
602
603 if (!action_ste->max_stes ||
604 matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION ||
605 mlx5hws_matcher_is_in_resize(matcher))
606 return;
607
608 mlx5hws_action_free_single_stc(tbl->ctx, tbl->type, &action_ste->stc);
609 hws_matcher_destroy_rtc(matcher, HWS_MATCHER_RTC_TYPE_STE_ARRAY, action_ste_selector);
610 mlx5hws_pool_destroy(action_ste->pool);
611 }
612
hws_matcher_bind_at(struct mlx5hws_matcher * matcher)613 static int hws_matcher_bind_at(struct mlx5hws_matcher *matcher)
614 {
615 bool is_jumbo = mlx5hws_matcher_mt_is_jumbo(matcher->mt);
616 struct mlx5hws_table *tbl = matcher->tbl;
617 struct mlx5hws_context *ctx = tbl->ctx;
618 u32 required_stes;
619 u8 max_stes = 0;
620 int i, ret;
621
622 if (matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION)
623 return 0;
624
625 for (i = 0; i < matcher->num_of_at; i++) {
626 struct mlx5hws_action_template *at = &matcher->at[i];
627
628 ret = hws_matcher_check_and_process_at(matcher, at);
629 if (ret) {
630 mlx5hws_err(ctx, "Invalid at %d", i);
631 return ret;
632 }
633
634 required_stes = at->num_of_action_stes - (!is_jumbo || at->only_term);
635 max_stes = max(max_stes, required_stes);
636
637 /* Future: Optimize reparse */
638 }
639
640 /* There are no additional STEs required for matcher */
641 if (!max_stes)
642 return 0;
643
644 matcher->action_ste[0].max_stes = max_stes;
645 matcher->action_ste[1].max_stes = max_stes;
646
647 ret = hws_matcher_bind_at_idx(matcher, 0);
648 if (ret)
649 return ret;
650
651 ret = hws_matcher_bind_at_idx(matcher, 1);
652 if (ret)
653 goto free_at_0;
654
655 return 0;
656
657 free_at_0:
658 hws_matcher_unbind_at_idx(matcher, 0);
659 return ret;
660 }
661
hws_matcher_unbind_at(struct mlx5hws_matcher * matcher)662 static void hws_matcher_unbind_at(struct mlx5hws_matcher *matcher)
663 {
664 hws_matcher_unbind_at_idx(matcher, 1);
665 hws_matcher_unbind_at_idx(matcher, 0);
666 }
667
hws_matcher_bind_mt(struct mlx5hws_matcher * matcher)668 static int hws_matcher_bind_mt(struct mlx5hws_matcher *matcher)
669 {
670 struct mlx5hws_context *ctx = matcher->tbl->ctx;
671 struct mlx5hws_pool_attr pool_attr = {0};
672 int ret;
673
674 /* Calculate match, range and hash definers */
675 if (!(matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION)) {
676 ret = mlx5hws_definer_mt_init(ctx, matcher->mt);
677 if (ret) {
678 if (ret == -E2BIG)
679 mlx5hws_err(ctx, "Failed to set matcher templates with match definers\n");
680 return ret;
681 }
682 }
683
684 /* Create an STE pool per matcher*/
685 pool_attr.table_type = matcher->tbl->type;
686 pool_attr.pool_type = MLX5HWS_POOL_TYPE_STE;
687 pool_attr.flags = MLX5HWS_POOL_FLAGS_FOR_MATCHER_STE_POOL;
688 pool_attr.alloc_log_sz = matcher->attr.table.sz_col_log +
689 matcher->attr.table.sz_row_log;
690 hws_matcher_set_pool_attr(&pool_attr, matcher);
691
692 matcher->match_ste.pool = mlx5hws_pool_create(ctx, &pool_attr);
693 if (!matcher->match_ste.pool) {
694 mlx5hws_err(ctx, "Failed to allocate matcher STE pool\n");
695 ret = -EOPNOTSUPP;
696 goto uninit_match_definer;
697 }
698
699 return 0;
700
701 uninit_match_definer:
702 if (!(matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION))
703 mlx5hws_definer_mt_uninit(ctx, matcher->mt);
704 return ret;
705 }
706
hws_matcher_unbind_mt(struct mlx5hws_matcher * matcher)707 static void hws_matcher_unbind_mt(struct mlx5hws_matcher *matcher)
708 {
709 mlx5hws_pool_destroy(matcher->match_ste.pool);
710 if (!(matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION))
711 mlx5hws_definer_mt_uninit(matcher->tbl->ctx, matcher->mt);
712 }
713
714 static int
hws_matcher_validate_insert_mode(struct mlx5hws_cmd_query_caps * caps,struct mlx5hws_matcher * matcher)715 hws_matcher_validate_insert_mode(struct mlx5hws_cmd_query_caps *caps,
716 struct mlx5hws_matcher *matcher)
717 {
718 struct mlx5hws_matcher_attr *attr = &matcher->attr;
719 struct mlx5hws_context *ctx = matcher->tbl->ctx;
720
721 switch (attr->insert_mode) {
722 case MLX5HWS_MATCHER_INSERT_BY_HASH:
723 if (matcher->attr.distribute_mode != MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH) {
724 mlx5hws_err(ctx, "Invalid matcher distribute mode\n");
725 return -EOPNOTSUPP;
726 }
727 break;
728
729 case MLX5HWS_MATCHER_INSERT_BY_INDEX:
730 if (attr->table.sz_col_log) {
731 mlx5hws_err(ctx, "Matcher with INSERT_BY_INDEX supports only Nx1 table size\n");
732 return -EOPNOTSUPP;
733 }
734
735 if (attr->distribute_mode == MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH) {
736 /* Hash Split Table */
737 if (!caps->rtc_hash_split_table) {
738 mlx5hws_err(ctx, "FW doesn't support insert by index and hash distribute\n");
739 return -EOPNOTSUPP;
740 }
741 } else if (attr->distribute_mode == MLX5HWS_MATCHER_DISTRIBUTE_BY_LINEAR) {
742 /* Linear Lookup Table */
743 if (!caps->rtc_linear_lookup_table ||
744 !IS_BIT_SET(caps->access_index_mode,
745 MLX5_IFC_RTC_STE_ACCESS_MODE_LINEAR)) {
746 mlx5hws_err(ctx, "FW doesn't support insert by index and linear distribute\n");
747 return -EOPNOTSUPP;
748 }
749
750 if (attr->table.sz_row_log > MLX5_IFC_RTC_LINEAR_LOOKUP_TBL_LOG_MAX) {
751 mlx5hws_err(ctx, "Matcher with linear distribute: rows exceed limit %d",
752 MLX5_IFC_RTC_LINEAR_LOOKUP_TBL_LOG_MAX);
753 return -EOPNOTSUPP;
754 }
755 } else {
756 mlx5hws_err(ctx, "Matcher has unsupported distribute mode\n");
757 return -EOPNOTSUPP;
758 }
759 break;
760
761 default:
762 mlx5hws_err(ctx, "Matcher has unsupported insert mode\n");
763 return -EOPNOTSUPP;
764 }
765
766 return 0;
767 }
768
769 static int
hws_matcher_process_attr(struct mlx5hws_cmd_query_caps * caps,struct mlx5hws_matcher * matcher)770 hws_matcher_process_attr(struct mlx5hws_cmd_query_caps *caps,
771 struct mlx5hws_matcher *matcher)
772 {
773 struct mlx5hws_matcher_attr *attr = &matcher->attr;
774
775 if (hws_matcher_validate_insert_mode(caps, matcher))
776 return -EOPNOTSUPP;
777
778 if (matcher->tbl->type != MLX5HWS_TABLE_TYPE_FDB && attr->optimize_flow_src) {
779 mlx5hws_err(matcher->tbl->ctx, "NIC domain doesn't support flow_src\n");
780 return -EOPNOTSUPP;
781 }
782
783 /* Convert number of rules to the required depth */
784 if (attr->mode == MLX5HWS_MATCHER_RESOURCE_MODE_RULE &&
785 attr->insert_mode == MLX5HWS_MATCHER_INSERT_BY_HASH)
786 attr->table.sz_col_log = hws_matcher_rules_to_tbl_depth(attr->rule.num_log);
787
788 matcher->flags |= attr->resizable ? MLX5HWS_MATCHER_FLAGS_RESIZABLE : 0;
789
790 return hws_matcher_check_attr_sz(caps, matcher);
791 }
792
hws_matcher_create_and_connect(struct mlx5hws_matcher * matcher)793 static int hws_matcher_create_and_connect(struct mlx5hws_matcher *matcher)
794 {
795 int ret;
796
797 /* Select and create the definers for current matcher */
798 ret = hws_matcher_bind_mt(matcher);
799 if (ret)
800 return ret;
801
802 /* Calculate and verify action combination */
803 ret = hws_matcher_bind_at(matcher);
804 if (ret)
805 goto unbind_mt;
806
807 /* Create matcher end flow table anchor */
808 ret = hws_matcher_create_end_ft(matcher);
809 if (ret)
810 goto unbind_at;
811
812 /* Allocate the RTC for the new matcher */
813 ret = hws_matcher_create_rtc(matcher, HWS_MATCHER_RTC_TYPE_MATCH, 0);
814 if (ret)
815 goto destroy_end_ft;
816
817 /* Connect the matcher to the matcher list */
818 ret = hws_matcher_connect(matcher);
819 if (ret)
820 goto destroy_rtc;
821
822 return 0;
823
824 destroy_rtc:
825 hws_matcher_destroy_rtc(matcher, HWS_MATCHER_RTC_TYPE_MATCH, 0);
826 destroy_end_ft:
827 hws_matcher_destroy_end_ft(matcher);
828 unbind_at:
829 hws_matcher_unbind_at(matcher);
830 unbind_mt:
831 hws_matcher_unbind_mt(matcher);
832 return ret;
833 }
834
hws_matcher_destroy_and_disconnect(struct mlx5hws_matcher * matcher)835 static void hws_matcher_destroy_and_disconnect(struct mlx5hws_matcher *matcher)
836 {
837 hws_matcher_resize_uninit(matcher);
838 hws_matcher_disconnect(matcher);
839 hws_matcher_destroy_rtc(matcher, HWS_MATCHER_RTC_TYPE_MATCH, 0);
840 hws_matcher_destroy_end_ft(matcher);
841 hws_matcher_unbind_at(matcher);
842 hws_matcher_unbind_mt(matcher);
843 }
844
845 static int
hws_matcher_create_col_matcher(struct mlx5hws_matcher * matcher)846 hws_matcher_create_col_matcher(struct mlx5hws_matcher *matcher)
847 {
848 struct mlx5hws_context *ctx = matcher->tbl->ctx;
849 struct mlx5hws_matcher *col_matcher;
850 int ret;
851
852 if (matcher->attr.mode != MLX5HWS_MATCHER_RESOURCE_MODE_RULE ||
853 matcher->attr.insert_mode == MLX5HWS_MATCHER_INSERT_BY_INDEX)
854 return 0;
855
856 if (!hws_matcher_requires_col_tbl(matcher->attr.rule.num_log))
857 return 0;
858
859 col_matcher = kzalloc(sizeof(*matcher), GFP_KERNEL);
860 if (!col_matcher)
861 return -ENOMEM;
862
863 INIT_LIST_HEAD(&col_matcher->resize_data);
864
865 col_matcher->tbl = matcher->tbl;
866 col_matcher->mt = matcher->mt;
867 col_matcher->at = matcher->at;
868 col_matcher->num_of_at = matcher->num_of_at;
869 col_matcher->num_of_mt = matcher->num_of_mt;
870 col_matcher->attr.priority = matcher->attr.priority;
871 col_matcher->flags = matcher->flags;
872 col_matcher->flags |= MLX5HWS_MATCHER_FLAGS_COLLISION;
873 col_matcher->attr.mode = MLX5HWS_MATCHER_RESOURCE_MODE_HTABLE;
874 col_matcher->attr.optimize_flow_src = matcher->attr.optimize_flow_src;
875 col_matcher->attr.table.sz_row_log = matcher->attr.rule.num_log;
876 col_matcher->attr.table.sz_col_log = MLX5HWS_MATCHER_ASSURED_COL_TBL_DEPTH;
877 if (col_matcher->attr.table.sz_row_log > MLX5HWS_MATCHER_ASSURED_ROW_RATIO)
878 col_matcher->attr.table.sz_row_log -= MLX5HWS_MATCHER_ASSURED_ROW_RATIO;
879
880 col_matcher->attr.max_num_of_at_attach = matcher->attr.max_num_of_at_attach;
881
882 ret = hws_matcher_process_attr(ctx->caps, col_matcher);
883 if (ret)
884 goto free_col_matcher;
885
886 ret = hws_matcher_create_and_connect(col_matcher);
887 if (ret)
888 goto free_col_matcher;
889
890 matcher->col_matcher = col_matcher;
891
892 return 0;
893
894 free_col_matcher:
895 kfree(col_matcher);
896 mlx5hws_err(ctx, "Failed to create assured collision matcher\n");
897 return ret;
898 }
899
900 static void
hws_matcher_destroy_col_matcher(struct mlx5hws_matcher * matcher)901 hws_matcher_destroy_col_matcher(struct mlx5hws_matcher *matcher)
902 {
903 if (matcher->attr.mode != MLX5HWS_MATCHER_RESOURCE_MODE_RULE ||
904 matcher->attr.insert_mode == MLX5HWS_MATCHER_INSERT_BY_INDEX)
905 return;
906
907 if (matcher->col_matcher) {
908 hws_matcher_destroy_and_disconnect(matcher->col_matcher);
909 kfree(matcher->col_matcher);
910 }
911 }
912
hws_matcher_init(struct mlx5hws_matcher * matcher)913 static int hws_matcher_init(struct mlx5hws_matcher *matcher)
914 {
915 struct mlx5hws_context *ctx = matcher->tbl->ctx;
916 int ret;
917
918 INIT_LIST_HEAD(&matcher->resize_data);
919
920 mutex_lock(&ctx->ctrl_lock);
921
922 /* Allocate matcher resource and connect to the packet pipe */
923 ret = hws_matcher_create_and_connect(matcher);
924 if (ret)
925 goto unlock_err;
926
927 /* Create additional matcher for collision handling */
928 ret = hws_matcher_create_col_matcher(matcher);
929 if (ret)
930 goto destory_and_disconnect;
931 mutex_unlock(&ctx->ctrl_lock);
932
933 return 0;
934
935 destory_and_disconnect:
936 hws_matcher_destroy_and_disconnect(matcher);
937 unlock_err:
938 mutex_unlock(&ctx->ctrl_lock);
939 return ret;
940 }
941
hws_matcher_uninit(struct mlx5hws_matcher * matcher)942 static int hws_matcher_uninit(struct mlx5hws_matcher *matcher)
943 {
944 struct mlx5hws_context *ctx = matcher->tbl->ctx;
945
946 mutex_lock(&ctx->ctrl_lock);
947 hws_matcher_destroy_col_matcher(matcher);
948 hws_matcher_destroy_and_disconnect(matcher);
949 mutex_unlock(&ctx->ctrl_lock);
950
951 return 0;
952 }
953
mlx5hws_matcher_attach_at(struct mlx5hws_matcher * matcher,struct mlx5hws_action_template * at)954 int mlx5hws_matcher_attach_at(struct mlx5hws_matcher *matcher,
955 struct mlx5hws_action_template *at)
956 {
957 bool is_jumbo = mlx5hws_matcher_mt_is_jumbo(matcher->mt);
958 struct mlx5hws_context *ctx = matcher->tbl->ctx;
959 u32 required_stes;
960 int ret;
961
962 if (!matcher->attr.max_num_of_at_attach) {
963 mlx5hws_dbg(ctx, "Num of current at (%d) exceed allowed value\n",
964 matcher->num_of_at);
965 return -EOPNOTSUPP;
966 }
967
968 ret = hws_matcher_check_and_process_at(matcher, at);
969 if (ret)
970 return ret;
971
972 required_stes = at->num_of_action_stes - (!is_jumbo || at->only_term);
973 if (matcher->action_ste[MLX5HWS_ACTION_STE_IDX_ANY].max_stes < required_stes) {
974 mlx5hws_dbg(ctx, "Required STEs [%d] exceeds initial action template STE [%d]\n",
975 required_stes,
976 matcher->action_ste[MLX5HWS_ACTION_STE_IDX_ANY].max_stes);
977 return -ENOMEM;
978 }
979
980 matcher->at[matcher->num_of_at] = *at;
981 matcher->num_of_at += 1;
982 matcher->attr.max_num_of_at_attach -= 1;
983
984 if (matcher->col_matcher)
985 matcher->col_matcher->num_of_at = matcher->num_of_at;
986
987 return 0;
988 }
989
990 static int
hws_matcher_set_templates(struct mlx5hws_matcher * matcher,struct mlx5hws_match_template * mt[],u8 num_of_mt,struct mlx5hws_action_template * at[],u8 num_of_at)991 hws_matcher_set_templates(struct mlx5hws_matcher *matcher,
992 struct mlx5hws_match_template *mt[],
993 u8 num_of_mt,
994 struct mlx5hws_action_template *at[],
995 u8 num_of_at)
996 {
997 struct mlx5hws_context *ctx = matcher->tbl->ctx;
998 int ret = 0;
999 int i;
1000
1001 if (!num_of_mt || !num_of_at) {
1002 mlx5hws_err(ctx, "Number of action/match template cannot be zero\n");
1003 return -EOPNOTSUPP;
1004 }
1005
1006 matcher->mt = kcalloc(num_of_mt, sizeof(*matcher->mt), GFP_KERNEL);
1007 if (!matcher->mt)
1008 return -ENOMEM;
1009
1010 matcher->at = kcalloc(num_of_at + matcher->attr.max_num_of_at_attach,
1011 sizeof(*matcher->at),
1012 GFP_KERNEL);
1013 if (!matcher->at) {
1014 mlx5hws_err(ctx, "Failed to allocate action template array\n");
1015 ret = -ENOMEM;
1016 goto free_mt;
1017 }
1018
1019 for (i = 0; i < num_of_mt; i++)
1020 matcher->mt[i] = *mt[i];
1021
1022 for (i = 0; i < num_of_at; i++)
1023 matcher->at[i] = *at[i];
1024
1025 matcher->num_of_mt = num_of_mt;
1026 matcher->num_of_at = num_of_at;
1027
1028 return 0;
1029
1030 free_mt:
1031 kfree(matcher->mt);
1032 return ret;
1033 }
1034
1035 static void
hws_matcher_unset_templates(struct mlx5hws_matcher * matcher)1036 hws_matcher_unset_templates(struct mlx5hws_matcher *matcher)
1037 {
1038 kfree(matcher->at);
1039 kfree(matcher->mt);
1040 }
1041
1042 struct mlx5hws_matcher *
mlx5hws_matcher_create(struct mlx5hws_table * tbl,struct mlx5hws_match_template * mt[],u8 num_of_mt,struct mlx5hws_action_template * at[],u8 num_of_at,struct mlx5hws_matcher_attr * attr)1043 mlx5hws_matcher_create(struct mlx5hws_table *tbl,
1044 struct mlx5hws_match_template *mt[],
1045 u8 num_of_mt,
1046 struct mlx5hws_action_template *at[],
1047 u8 num_of_at,
1048 struct mlx5hws_matcher_attr *attr)
1049 {
1050 struct mlx5hws_context *ctx = tbl->ctx;
1051 struct mlx5hws_matcher *matcher;
1052 int ret;
1053
1054 matcher = kzalloc(sizeof(*matcher), GFP_KERNEL);
1055 if (!matcher)
1056 return NULL;
1057
1058 matcher->tbl = tbl;
1059 matcher->attr = *attr;
1060
1061 ret = hws_matcher_process_attr(tbl->ctx->caps, matcher);
1062 if (ret)
1063 goto free_matcher;
1064
1065 ret = hws_matcher_set_templates(matcher, mt, num_of_mt, at, num_of_at);
1066 if (ret)
1067 goto free_matcher;
1068
1069 ret = hws_matcher_init(matcher);
1070 if (ret) {
1071 mlx5hws_err(ctx, "Failed to initialise matcher: %d\n", ret);
1072 goto unset_templates;
1073 }
1074
1075 return matcher;
1076
1077 unset_templates:
1078 hws_matcher_unset_templates(matcher);
1079 free_matcher:
1080 kfree(matcher);
1081 return NULL;
1082 }
1083
mlx5hws_matcher_destroy(struct mlx5hws_matcher * matcher)1084 int mlx5hws_matcher_destroy(struct mlx5hws_matcher *matcher)
1085 {
1086 hws_matcher_uninit(matcher);
1087 hws_matcher_unset_templates(matcher);
1088 kfree(matcher);
1089 return 0;
1090 }
1091
1092 struct mlx5hws_match_template *
mlx5hws_match_template_create(struct mlx5hws_context * ctx,u32 * match_param,u32 match_param_sz,u8 match_criteria_enable)1093 mlx5hws_match_template_create(struct mlx5hws_context *ctx,
1094 u32 *match_param,
1095 u32 match_param_sz,
1096 u8 match_criteria_enable)
1097 {
1098 struct mlx5hws_match_template *mt;
1099
1100 mt = kzalloc(sizeof(*mt), GFP_KERNEL);
1101 if (!mt)
1102 return NULL;
1103
1104 mt->match_param = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
1105 if (!mt->match_param)
1106 goto free_template;
1107
1108 memcpy(mt->match_param, match_param, match_param_sz);
1109 mt->match_criteria_enable = match_criteria_enable;
1110
1111 return mt;
1112
1113 free_template:
1114 kfree(mt);
1115 return NULL;
1116 }
1117
mlx5hws_match_template_destroy(struct mlx5hws_match_template * mt)1118 int mlx5hws_match_template_destroy(struct mlx5hws_match_template *mt)
1119 {
1120 kfree(mt->match_param);
1121 kfree(mt);
1122 return 0;
1123 }
1124
hws_matcher_resize_precheck(struct mlx5hws_matcher * src_matcher,struct mlx5hws_matcher * dst_matcher)1125 static int hws_matcher_resize_precheck(struct mlx5hws_matcher *src_matcher,
1126 struct mlx5hws_matcher *dst_matcher)
1127 {
1128 struct mlx5hws_context *ctx = src_matcher->tbl->ctx;
1129 int i;
1130
1131 if (src_matcher->tbl->type != dst_matcher->tbl->type) {
1132 mlx5hws_err(ctx, "Table type mismatch for src/dst matchers\n");
1133 return -EINVAL;
1134 }
1135
1136 if (!mlx5hws_matcher_is_resizable(src_matcher) ||
1137 !mlx5hws_matcher_is_resizable(dst_matcher)) {
1138 mlx5hws_err(ctx, "Src/dst matcher is not resizable\n");
1139 return -EINVAL;
1140 }
1141
1142 if (mlx5hws_matcher_is_insert_by_idx(src_matcher) !=
1143 mlx5hws_matcher_is_insert_by_idx(dst_matcher)) {
1144 mlx5hws_err(ctx, "Src/dst matchers insert mode mismatch\n");
1145 return -EINVAL;
1146 }
1147
1148 if (mlx5hws_matcher_is_in_resize(src_matcher) ||
1149 mlx5hws_matcher_is_in_resize(dst_matcher)) {
1150 mlx5hws_err(ctx, "Src/dst matcher is already in resize\n");
1151 return -EINVAL;
1152 }
1153
1154 /* Compare match templates - make sure the definers are equivalent */
1155 if (src_matcher->num_of_mt != dst_matcher->num_of_mt) {
1156 mlx5hws_err(ctx, "Src/dst matcher match templates mismatch\n");
1157 return -EINVAL;
1158 }
1159
1160 if (src_matcher->action_ste[MLX5HWS_ACTION_STE_IDX_ANY].max_stes >
1161 dst_matcher->action_ste[0].max_stes) {
1162 mlx5hws_err(ctx, "Src/dst matcher max STEs mismatch\n");
1163 return -EINVAL;
1164 }
1165
1166 for (i = 0; i < src_matcher->num_of_mt; i++) {
1167 if (mlx5hws_definer_compare(src_matcher->mt[i].definer,
1168 dst_matcher->mt[i].definer)) {
1169 mlx5hws_err(ctx, "Src/dst matcher definers mismatch\n");
1170 return -EINVAL;
1171 }
1172 }
1173
1174 return 0;
1175 }
1176
mlx5hws_matcher_resize_set_target(struct mlx5hws_matcher * src_matcher,struct mlx5hws_matcher * dst_matcher)1177 int mlx5hws_matcher_resize_set_target(struct mlx5hws_matcher *src_matcher,
1178 struct mlx5hws_matcher *dst_matcher)
1179 {
1180 int ret = 0;
1181
1182 mutex_lock(&src_matcher->tbl->ctx->ctrl_lock);
1183
1184 ret = hws_matcher_resize_precheck(src_matcher, dst_matcher);
1185 if (ret)
1186 goto out;
1187
1188 src_matcher->resize_dst = dst_matcher;
1189
1190 ret = hws_matcher_resize_init(src_matcher);
1191 if (ret)
1192 src_matcher->resize_dst = NULL;
1193
1194 out:
1195 mutex_unlock(&src_matcher->tbl->ctx->ctrl_lock);
1196 return ret;
1197 }
1198
mlx5hws_matcher_resize_rule_move(struct mlx5hws_matcher * src_matcher,struct mlx5hws_rule * rule,struct mlx5hws_rule_attr * attr)1199 int mlx5hws_matcher_resize_rule_move(struct mlx5hws_matcher *src_matcher,
1200 struct mlx5hws_rule *rule,
1201 struct mlx5hws_rule_attr *attr)
1202 {
1203 struct mlx5hws_context *ctx = src_matcher->tbl->ctx;
1204
1205 if (unlikely(!mlx5hws_matcher_is_in_resize(src_matcher))) {
1206 mlx5hws_err(ctx, "Matcher is not resizable or not in resize\n");
1207 return -EINVAL;
1208 }
1209
1210 if (unlikely(src_matcher != rule->matcher)) {
1211 mlx5hws_err(ctx, "Rule doesn't belong to src matcher\n");
1212 return -EINVAL;
1213 }
1214
1215 return mlx5hws_rule_move_hws_add(rule, attr);
1216 }
1217