1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2024 NVIDIA Corporation & Affiliates */
3
4 #include "internal.h"
5
6 #define HWS_CLEAR_MATCH_PARAM(mask, field) \
7 MLX5_SET(fte_match_param, (mask)->match_buf, field, 0)
8
9 #define HWS_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4)
10
11 static const struct rhashtable_params hws_refcount_hash = {
12 .key_len = sizeof_field(struct mlx5hws_bwc_complex_rule_hash_node,
13 match_buf),
14 .key_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node,
15 match_buf),
16 .head_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node,
17 hash_node),
18 .automatic_shrinking = true,
19 .min_size = 1,
20 };
21
mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context * ctx,u8 match_criteria_enable,struct mlx5hws_match_parameters * mask)22 bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx,
23 u8 match_criteria_enable,
24 struct mlx5hws_match_parameters *mask)
25 {
26 struct mlx5hws_definer match_layout = {0};
27 struct mlx5hws_match_template *mt;
28 bool is_complex = false;
29 int ret;
30
31 if (!match_criteria_enable)
32 return false; /* empty matcher */
33
34 mt = mlx5hws_match_template_create(ctx,
35 mask->match_buf,
36 mask->match_sz,
37 match_criteria_enable);
38 if (!mt) {
39 mlx5hws_err(ctx, "BWC: failed creating match template\n");
40 return false;
41 }
42
43 ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout);
44 if (ret) {
45 /* The only case that we're interested in is E2BIG,
46 * which means that the match parameters need to be
47 * split into complex martcher.
48 * For all other cases (good or bad) - just return true
49 * and let the usual match creation path handle it,
50 * both for good and bad flows.
51 */
52 if (ret == -E2BIG) {
53 is_complex = true;
54 mlx5hws_dbg(ctx, "Matcher definer layout: need complex matcher\n");
55 } else {
56 mlx5hws_err(ctx, "Failed to calculate matcher definer layout\n");
57 }
58 } else {
59 kfree(mt->fc);
60 }
61
62 mlx5hws_match_template_destroy(mt);
63
64 return is_complex;
65 }
66
67 static void
hws_bwc_matcher_complex_params_clear_fld(struct mlx5hws_context * ctx,enum mlx5hws_definer_fname fname,struct mlx5hws_match_parameters * mask)68 hws_bwc_matcher_complex_params_clear_fld(struct mlx5hws_context *ctx,
69 enum mlx5hws_definer_fname fname,
70 struct mlx5hws_match_parameters *mask)
71 {
72 struct mlx5hws_cmd_query_caps *caps = ctx->caps;
73
74 switch (fname) {
75 case MLX5HWS_DEFINER_FNAME_ETH_TYPE_O:
76 case MLX5HWS_DEFINER_FNAME_ETH_TYPE_I:
77 case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_O:
78 case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_I:
79 case MLX5HWS_DEFINER_FNAME_IP_VERSION_O:
80 case MLX5HWS_DEFINER_FNAME_IP_VERSION_I:
81 /* Because of the strict requirements for IP address matching
82 * that require ethtype/ip_version matching as well, don't clear
83 * these fields - have them in both parts of the complex matcher
84 */
85 break;
86 case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O:
87 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_47_16);
88 break;
89 case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I:
90 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_47_16);
91 break;
92 case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O:
93 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_15_0);
94 break;
95 case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I:
96 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_15_0);
97 break;
98 case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O:
99 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_47_16);
100 break;
101 case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I:
102 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_47_16);
103 break;
104 case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O:
105 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_15_0);
106 break;
107 case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I:
108 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_15_0);
109 break;
110 case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_O:
111 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.cvlan_tag);
112 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.svlan_tag);
113 break;
114 case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_I:
115 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.cvlan_tag);
116 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.svlan_tag);
117 break;
118 case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_O:
119 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_prio);
120 break;
121 case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_I:
122 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_prio);
123 break;
124 case MLX5HWS_DEFINER_FNAME_VLAN_CFI_O:
125 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_cfi);
126 break;
127 case MLX5HWS_DEFINER_FNAME_VLAN_CFI_I:
128 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_cfi);
129 break;
130 case MLX5HWS_DEFINER_FNAME_VLAN_ID_O:
131 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_vid);
132 break;
133 case MLX5HWS_DEFINER_FNAME_VLAN_ID_I:
134 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_vid);
135 break;
136 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_O:
137 HWS_CLEAR_MATCH_PARAM(mask,
138 misc_parameters.outer_second_cvlan_tag);
139 HWS_CLEAR_MATCH_PARAM(mask,
140 misc_parameters.outer_second_svlan_tag);
141 break;
142 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_I:
143 HWS_CLEAR_MATCH_PARAM(mask,
144 misc_parameters.inner_second_cvlan_tag);
145 HWS_CLEAR_MATCH_PARAM(mask,
146 misc_parameters.inner_second_svlan_tag);
147 break;
148 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_O:
149 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_prio);
150 break;
151 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_I:
152 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_prio);
153 break;
154 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_O:
155 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_cfi);
156 break;
157 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_I:
158 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_cfi);
159 break;
160 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_O:
161 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_vid);
162 break;
163 case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_I:
164 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_vid);
165 break;
166 case MLX5HWS_DEFINER_FNAME_IPV4_IHL_O:
167 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ipv4_ihl);
168 break;
169 case MLX5HWS_DEFINER_FNAME_IPV4_IHL_I:
170 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ipv4_ihl);
171 break;
172 case MLX5HWS_DEFINER_FNAME_IP_DSCP_O:
173 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_dscp);
174 break;
175 case MLX5HWS_DEFINER_FNAME_IP_DSCP_I:
176 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_dscp);
177 break;
178 case MLX5HWS_DEFINER_FNAME_IP_ECN_O:
179 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_ecn);
180 break;
181 case MLX5HWS_DEFINER_FNAME_IP_ECN_I:
182 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_ecn);
183 break;
184 case MLX5HWS_DEFINER_FNAME_IP_TTL_O:
185 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ttl_hoplimit);
186 break;
187 case MLX5HWS_DEFINER_FNAME_IP_TTL_I:
188 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ttl_hoplimit);
189 break;
190 case MLX5HWS_DEFINER_FNAME_IPV4_DST_O:
191 HWS_CLEAR_MATCH_PARAM(mask,
192 outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
193 break;
194 case MLX5HWS_DEFINER_FNAME_IPV4_SRC_O:
195 HWS_CLEAR_MATCH_PARAM(mask,
196 outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
197 break;
198 case MLX5HWS_DEFINER_FNAME_IPV4_DST_I:
199 HWS_CLEAR_MATCH_PARAM(mask,
200 inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
201 break;
202 case MLX5HWS_DEFINER_FNAME_IPV4_SRC_I:
203 HWS_CLEAR_MATCH_PARAM(mask,
204 inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
205 break;
206 case MLX5HWS_DEFINER_FNAME_IP_FRAG_O:
207 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.frag);
208 break;
209 case MLX5HWS_DEFINER_FNAME_IP_FRAG_I:
210 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.frag);
211 break;
212 case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_O:
213 HWS_CLEAR_MATCH_PARAM(mask,
214 misc_parameters.outer_ipv6_flow_label);
215 break;
216 case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_I:
217 HWS_CLEAR_MATCH_PARAM(mask,
218 misc_parameters.inner_ipv6_flow_label);
219 break;
220 case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O:
221 case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O:
222 case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O:
223 case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O:
224 HWS_CLEAR_MATCH_PARAM(mask,
225 outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96);
226 HWS_CLEAR_MATCH_PARAM(mask,
227 outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64);
228 HWS_CLEAR_MATCH_PARAM(mask,
229 outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32);
230 HWS_CLEAR_MATCH_PARAM(mask,
231 outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
232 break;
233 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O:
234 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O:
235 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O:
236 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O:
237 HWS_CLEAR_MATCH_PARAM(mask,
238 outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96);
239 HWS_CLEAR_MATCH_PARAM(mask,
240 outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64);
241 HWS_CLEAR_MATCH_PARAM(mask,
242 outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32);
243 HWS_CLEAR_MATCH_PARAM(mask,
244 outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
245 break;
246 case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I:
247 case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I:
248 case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I:
249 case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I:
250 HWS_CLEAR_MATCH_PARAM(mask,
251 inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96);
252 HWS_CLEAR_MATCH_PARAM(mask,
253 inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64);
254 HWS_CLEAR_MATCH_PARAM(mask,
255 inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32);
256 HWS_CLEAR_MATCH_PARAM(mask,
257 inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
258 break;
259 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I:
260 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I:
261 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I:
262 case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I:
263 HWS_CLEAR_MATCH_PARAM(mask,
264 inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96);
265 HWS_CLEAR_MATCH_PARAM(mask,
266 inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64);
267 HWS_CLEAR_MATCH_PARAM(mask,
268 inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32);
269 HWS_CLEAR_MATCH_PARAM(mask,
270 inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
271 break;
272 case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_O:
273 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_protocol);
274 break;
275 case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_I:
276 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_protocol);
277 break;
278 case MLX5HWS_DEFINER_FNAME_L4_SPORT_O:
279 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_sport);
280 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_sport);
281 break;
282 case MLX5HWS_DEFINER_FNAME_L4_SPORT_I:
283 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport);
284 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport);
285 break;
286 case MLX5HWS_DEFINER_FNAME_L4_DPORT_O:
287 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_dport);
288 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_dport);
289 break;
290 case MLX5HWS_DEFINER_FNAME_L4_DPORT_I:
291 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport);
292 HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport);
293 break;
294 case MLX5HWS_DEFINER_FNAME_TCP_FLAGS_O:
295 HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_flags);
296 break;
297 case MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM:
298 case MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM:
299 HWS_CLEAR_MATCH_PARAM(mask,
300 misc_parameters_3.outer_tcp_seq_num);
301 HWS_CLEAR_MATCH_PARAM(mask,
302 misc_parameters_3.outer_tcp_ack_num);
303 HWS_CLEAR_MATCH_PARAM(mask,
304 misc_parameters_3.inner_tcp_seq_num);
305 HWS_CLEAR_MATCH_PARAM(mask,
306 misc_parameters_3.inner_tcp_ack_num);
307 break;
308 case MLX5HWS_DEFINER_FNAME_GTP_TEID:
309 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_teid);
310 break;
311 case MLX5HWS_DEFINER_FNAME_GTP_MSG_TYPE:
312 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_type);
313 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_flags);
314 break;
315 case MLX5HWS_DEFINER_FNAME_GTPU_FIRST_EXT_DW0:
316 HWS_CLEAR_MATCH_PARAM(mask,
317 misc_parameters_3.gtpu_first_ext_dw_0);
318 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_0);
319 break;
320 case MLX5HWS_DEFINER_FNAME_GTPU_DW2:
321 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_2);
322 break;
323 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0:
324 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1:
325 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2:
326 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3:
327 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4:
328 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5:
329 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6:
330 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7:
331 HWS_CLEAR_MATCH_PARAM(mask,
332 misc_parameters_2.outer_first_mpls_over_gre);
333 HWS_CLEAR_MATCH_PARAM(mask,
334 misc_parameters_2.outer_first_mpls_over_udp);
335 HWS_CLEAR_MATCH_PARAM(mask,
336 misc_parameters_3.geneve_tlv_option_0_data);
337 HWS_CLEAR_MATCH_PARAM(mask,
338 misc_parameters_4.prog_sample_field_id_0);
339 HWS_CLEAR_MATCH_PARAM(mask,
340 misc_parameters_4.prog_sample_field_value_0);
341 HWS_CLEAR_MATCH_PARAM(mask,
342 misc_parameters_4.prog_sample_field_value_1);
343 HWS_CLEAR_MATCH_PARAM(mask,
344 misc_parameters_4.prog_sample_field_id_2);
345 HWS_CLEAR_MATCH_PARAM(mask,
346 misc_parameters_4.prog_sample_field_value_2);
347 HWS_CLEAR_MATCH_PARAM(mask,
348 misc_parameters_4.prog_sample_field_id_3);
349 HWS_CLEAR_MATCH_PARAM(mask,
350 misc_parameters_4.prog_sample_field_value_3);
351 break;
352 case MLX5HWS_DEFINER_FNAME_VXLAN_VNI:
353 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.vxlan_vni);
354 break;
355 case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_FLAGS:
356 HWS_CLEAR_MATCH_PARAM(mask,
357 misc_parameters_3.outer_vxlan_gpe_flags);
358 break;
359 case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_RSVD0:
360 break;
361 case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_PROTO:
362 HWS_CLEAR_MATCH_PARAM(mask,
363 misc_parameters_3.outer_vxlan_gpe_next_protocol);
364 break;
365 case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_VNI:
366 HWS_CLEAR_MATCH_PARAM(mask,
367 misc_parameters_3.outer_vxlan_gpe_vni);
368 break;
369 case MLX5HWS_DEFINER_FNAME_GENEVE_OPT_LEN:
370 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_opt_len);
371 break;
372 case MLX5HWS_DEFINER_FNAME_GENEVE_OAM:
373 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_oam);
374 break;
375 case MLX5HWS_DEFINER_FNAME_GENEVE_PROTO:
376 HWS_CLEAR_MATCH_PARAM(mask,
377 misc_parameters.geneve_protocol_type);
378 break;
379 case MLX5HWS_DEFINER_FNAME_GENEVE_VNI:
380 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_vni);
381 break;
382 case MLX5HWS_DEFINER_FNAME_SOURCE_QP:
383 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_sqn);
384 break;
385 case MLX5HWS_DEFINER_FNAME_SOURCE_GVMI:
386 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_port);
387 HWS_CLEAR_MATCH_PARAM(mask,
388 misc_parameters.source_eswitch_owner_vhca_id);
389 break;
390 case MLX5HWS_DEFINER_FNAME_REG_0:
391 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_0);
392 break;
393 case MLX5HWS_DEFINER_FNAME_REG_1:
394 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_1);
395 break;
396 case MLX5HWS_DEFINER_FNAME_REG_2:
397 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_2);
398 break;
399 case MLX5HWS_DEFINER_FNAME_REG_3:
400 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_3);
401 break;
402 case MLX5HWS_DEFINER_FNAME_REG_4:
403 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_4);
404 break;
405 case MLX5HWS_DEFINER_FNAME_REG_5:
406 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_5);
407 break;
408 case MLX5HWS_DEFINER_FNAME_REG_7:
409 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_7);
410 break;
411 case MLX5HWS_DEFINER_FNAME_REG_A:
412 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_a);
413 break;
414 case MLX5HWS_DEFINER_FNAME_GRE_C:
415 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_c_present);
416 break;
417 case MLX5HWS_DEFINER_FNAME_GRE_K:
418 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_k_present);
419 break;
420 case MLX5HWS_DEFINER_FNAME_GRE_S:
421 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_s_present);
422 break;
423 case MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL:
424 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_protocol);
425 break;
426 case MLX5HWS_DEFINER_FNAME_GRE_OPT_KEY:
427 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_key.key);
428 break;
429 case MLX5HWS_DEFINER_FNAME_ICMP_DW1:
430 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_header_data);
431 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_type);
432 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_code);
433 HWS_CLEAR_MATCH_PARAM(mask,
434 misc_parameters_3.icmpv6_header_data);
435 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_type);
436 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_code);
437 break;
438 case MLX5HWS_DEFINER_FNAME_MPLS0_O:
439 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.outer_first_mpls);
440 break;
441 case MLX5HWS_DEFINER_FNAME_MPLS0_I:
442 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.inner_first_mpls);
443 break;
444 case MLX5HWS_DEFINER_FNAME_TNL_HDR_0:
445 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_0);
446 break;
447 case MLX5HWS_DEFINER_FNAME_TNL_HDR_1:
448 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_1);
449 break;
450 case MLX5HWS_DEFINER_FNAME_TNL_HDR_2:
451 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_2);
452 break;
453 case MLX5HWS_DEFINER_FNAME_TNL_HDR_3:
454 HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_3);
455 break;
456 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK:
457 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK:
458 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK:
459 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK:
460 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK:
461 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK:
462 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK:
463 case MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK:
464 /* assuming this is flex parser for geneve option */
465 if ((fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK &&
466 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 0) ||
467 (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK &&
468 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 1) ||
469 (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK &&
470 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 2) ||
471 (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK &&
472 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 3) ||
473 (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK &&
474 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 4) ||
475 (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK &&
476 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 5) ||
477 (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK &&
478 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 6) ||
479 (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK &&
480 ctx->caps->flex_parser_id_geneve_tlv_option_0 != 7)) {
481 mlx5hws_err(ctx,
482 "Complex params: unsupported field %s (%d), flex parser ID for geneve is %d\n",
483 mlx5hws_definer_fname_to_str(fname), fname,
484 caps->flex_parser_id_geneve_tlv_option_0);
485 break;
486 }
487 HWS_CLEAR_MATCH_PARAM(mask,
488 misc_parameters.geneve_tlv_option_0_exist);
489 break;
490 case MLX5HWS_DEFINER_FNAME_REG_6:
491 default:
492 mlx5hws_err(ctx, "Complex params: unsupported field %s (%d)\n",
493 mlx5hws_definer_fname_to_str(fname), fname);
494 break;
495 }
496 }
497
498 static bool
hws_bwc_matcher_complex_params_comb_is_valid(struct mlx5hws_definer_fc * fc,int fc_sz,u32 combination_num)499 hws_bwc_matcher_complex_params_comb_is_valid(struct mlx5hws_definer_fc *fc,
500 int fc_sz,
501 u32 combination_num)
502 {
503 bool m1[MLX5HWS_DEFINER_FNAME_MAX] = {0};
504 bool m2[MLX5HWS_DEFINER_FNAME_MAX] = {0};
505 bool is_first_matcher;
506 int i;
507
508 for (i = 0; i < fc_sz; i++) {
509 is_first_matcher = !(combination_num & BIT(i));
510 if (is_first_matcher)
511 m1[fc[i].fname] = true;
512 else
513 m2[fc[i].fname] = true;
514 }
515
516 /* Not all the fields can be split into separate matchers.
517 * Some should be together on the same matcher.
518 * For example, IPv6 parts - the whole IPv6 address should be on the
519 * same matcher in order for us to deduce if it's IPv6 or IPv4 address.
520 */
521 if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] &&
522 (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] ||
523 m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] ||
524 m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] ||
525 m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O]))
526 return false;
527
528 if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] &&
529 (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] ||
530 m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] ||
531 m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] ||
532 m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O]))
533 return false;
534
535 if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] &&
536 (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] ||
537 m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] ||
538 m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] ||
539 m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I]))
540 return false;
541
542 if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] &&
543 (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] ||
544 m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] ||
545 m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] ||
546 m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I]))
547 return false;
548
549 /* Don't split outer IPv6 dest address. */
550 if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] ||
551 m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] ||
552 m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] ||
553 m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O]) &&
554 (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] ||
555 m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] ||
556 m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] ||
557 m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O]))
558 return false;
559
560 /* Don't split outer IPv6 source address. */
561 if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] ||
562 m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] ||
563 m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] ||
564 m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O]) &&
565 (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] ||
566 m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] ||
567 m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] ||
568 m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O]))
569 return false;
570
571 /* Don't split inner IPv6 dest address. */
572 if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] ||
573 m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] ||
574 m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] ||
575 m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I]) &&
576 (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] ||
577 m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] ||
578 m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] ||
579 m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I]))
580 return false;
581
582 /* Don't split inner IPv6 source address. */
583 if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] ||
584 m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] ||
585 m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] ||
586 m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I]) &&
587 (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] ||
588 m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] ||
589 m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] ||
590 m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I]))
591 return false;
592
593 /* Don't split GRE parameters. */
594 if ((m1[MLX5HWS_DEFINER_FNAME_GRE_C] ||
595 m1[MLX5HWS_DEFINER_FNAME_GRE_K] ||
596 m1[MLX5HWS_DEFINER_FNAME_GRE_S] ||
597 m1[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL]) &&
598 (m2[MLX5HWS_DEFINER_FNAME_GRE_C] ||
599 m2[MLX5HWS_DEFINER_FNAME_GRE_K] ||
600 m2[MLX5HWS_DEFINER_FNAME_GRE_S] ||
601 m2[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL]))
602 return false;
603
604 /* Don't split TCP ack/seq numbers. */
605 if ((m1[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] ||
606 m1[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM]) &&
607 (m2[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] ||
608 m2[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM]))
609 return false;
610
611 /* Don't split flex parser. */
612 if ((m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] ||
613 m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] ||
614 m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] ||
615 m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] ||
616 m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] ||
617 m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] ||
618 m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] ||
619 m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7]) &&
620 (m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] ||
621 m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] ||
622 m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] ||
623 m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] ||
624 m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] ||
625 m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] ||
626 m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] ||
627 m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7]))
628 return false;
629
630 return true;
631 }
632
633 static void
hws_bwc_matcher_complex_params_comb_create(struct mlx5hws_context * ctx,struct mlx5hws_match_parameters * m,struct mlx5hws_match_parameters * m1,struct mlx5hws_match_parameters * m2,struct mlx5hws_definer_fc * fc,int fc_sz,u32 combination_num)634 hws_bwc_matcher_complex_params_comb_create(struct mlx5hws_context *ctx,
635 struct mlx5hws_match_parameters *m,
636 struct mlx5hws_match_parameters *m1,
637 struct mlx5hws_match_parameters *m2,
638 struct mlx5hws_definer_fc *fc,
639 int fc_sz,
640 u32 combination_num)
641 {
642 bool is_first_matcher;
643 int i;
644
645 memcpy(m1->match_buf, m->match_buf, m->match_sz);
646 memcpy(m2->match_buf, m->match_buf, m->match_sz);
647
648 for (i = 0; i < fc_sz; i++) {
649 is_first_matcher = !(combination_num & BIT(i));
650 hws_bwc_matcher_complex_params_clear_fld(ctx,
651 fc[i].fname,
652 is_first_matcher ?
653 m2 : m1);
654 }
655
656 MLX5_SET(fte_match_param, m2->match_buf,
657 misc_parameters_2.metadata_reg_c_6, -1);
658 }
659
660 static void
hws_bwc_matcher_complex_params_destroy(struct mlx5hws_match_parameters * mask_1,struct mlx5hws_match_parameters * mask_2)661 hws_bwc_matcher_complex_params_destroy(struct mlx5hws_match_parameters *mask_1,
662 struct mlx5hws_match_parameters *mask_2)
663 {
664 kfree(mask_1->match_buf);
665 kfree(mask_2->match_buf);
666 }
667
668 static int
hws_bwc_matcher_complex_params_create(struct mlx5hws_context * ctx,u8 match_criteria,struct mlx5hws_match_parameters * mask,struct mlx5hws_match_parameters * mask_1,struct mlx5hws_match_parameters * mask_2)669 hws_bwc_matcher_complex_params_create(struct mlx5hws_context *ctx,
670 u8 match_criteria,
671 struct mlx5hws_match_parameters *mask,
672 struct mlx5hws_match_parameters *mask_1,
673 struct mlx5hws_match_parameters *mask_2)
674 {
675 struct mlx5hws_definer_fc *fc;
676 u32 num_of_combinations;
677 int fc_sz = 0;
678 int res = 0;
679 u32 i;
680
681 if (MLX5_GET(fte_match_param, mask->match_buf,
682 misc_parameters_2.metadata_reg_c_6)) {
683 mlx5hws_err(ctx, "Complex matcher: REG_C_6 matching is reserved\n");
684 res = -EINVAL;
685 goto out;
686 }
687
688 mask_1->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param),
689 GFP_KERNEL);
690 mask_2->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param),
691 GFP_KERNEL);
692 if (!mask_1->match_buf || !mask_2->match_buf) {
693 mlx5hws_err(ctx, "Complex matcher: failed to allocate match_param\n");
694 res = -ENOMEM;
695 goto free_params;
696 }
697
698 mask_1->match_sz = mask->match_sz;
699 mask_2->match_sz = mask->match_sz;
700
701 fc = mlx5hws_definer_conv_match_params_to_compressed_fc(ctx,
702 match_criteria,
703 mask->match_buf,
704 &fc_sz);
705 if (!fc) {
706 res = -ENOMEM;
707 goto free_params;
708 }
709
710 if (fc_sz >= sizeof(num_of_combinations) * BITS_PER_BYTE) {
711 mlx5hws_err(ctx,
712 "Complex matcher: too many match parameters (%d)\n",
713 fc_sz);
714 res = -EINVAL;
715 goto free_fc;
716 }
717
718 /* We have list of all the match fields from the match parameter.
719 * Now try all the possibilities of splitting them into two match
720 * buffers and look for the supported combination.
721 */
722 num_of_combinations = 1 << fc_sz;
723
724 /* Start from combination at index 1 - we know that 0 is unsupported */
725 for (i = 1; i < num_of_combinations; i++) {
726 if (!hws_bwc_matcher_complex_params_comb_is_valid(fc, fc_sz, i))
727 continue;
728
729 hws_bwc_matcher_complex_params_comb_create(ctx,
730 mask, mask_1, mask_2,
731 fc, fc_sz, i);
732 /* We now have two separate sets of match params.
733 * Check if each of them can be used in its own matcher.
734 */
735 if (!mlx5hws_bwc_match_params_is_complex(ctx,
736 match_criteria,
737 mask_1) &&
738 !mlx5hws_bwc_match_params_is_complex(ctx,
739 match_criteria,
740 mask_2))
741 break;
742 }
743
744 if (i == num_of_combinations) {
745 /* We've scanned all the combinations, but to no avail */
746 mlx5hws_err(ctx, "Complex matcher: couldn't find match params combination\n");
747 res = -EINVAL;
748 goto free_fc;
749 }
750
751 kfree(fc);
752 return 0;
753
754 free_fc:
755 kfree(fc);
756 free_params:
757 hws_bwc_matcher_complex_params_destroy(mask_1, mask_2);
758 out:
759 return res;
760 }
761
762 static int
hws_bwc_isolated_table_create(struct mlx5hws_bwc_matcher * bwc_matcher,struct mlx5hws_table * table)763 hws_bwc_isolated_table_create(struct mlx5hws_bwc_matcher *bwc_matcher,
764 struct mlx5hws_table *table)
765 {
766 struct mlx5hws_cmd_ft_modify_attr ft_attr = {0};
767 struct mlx5hws_context *ctx = table->ctx;
768 struct mlx5hws_table_attr tbl_attr = {0};
769 struct mlx5hws_table *isolated_tbl;
770 int ret = 0;
771
772 tbl_attr.type = table->type;
773 tbl_attr.level = table->level;
774
775 bwc_matcher->complex->isolated_tbl =
776 mlx5hws_table_create(ctx, &tbl_attr);
777 isolated_tbl = bwc_matcher->complex->isolated_tbl;
778 if (!isolated_tbl)
779 return -EINVAL;
780
781 /* Set the default miss of the isolated table to
782 * point to the end anchor of the original matcher.
783 */
784 mlx5hws_cmd_set_attr_connect_miss_tbl(ctx,
785 isolated_tbl->fw_ft_type,
786 isolated_tbl->type,
787 &ft_attr);
788 ft_attr.table_miss_id = bwc_matcher->matcher->end_ft_id;
789
790 ret = mlx5hws_cmd_flow_table_modify(ctx->mdev,
791 &ft_attr,
792 isolated_tbl->ft_id);
793 if (ret) {
794 mlx5hws_err(ctx, "Failed setting isolated tbl default miss\n");
795 goto destroy_tbl;
796 }
797
798 return 0;
799
800 destroy_tbl:
801 mlx5hws_table_destroy(isolated_tbl);
802 return ret;
803 }
804
hws_bwc_isolated_table_destroy(struct mlx5hws_table * isolated_tbl)805 static void hws_bwc_isolated_table_destroy(struct mlx5hws_table *isolated_tbl)
806 {
807 /* This table is isolated - no table is pointing to it, no need to
808 * disconnect it from anywhere, it won't affect any other table's miss.
809 */
810 mlx5hws_table_destroy(isolated_tbl);
811 }
812
813 static int
hws_bwc_isolated_matcher_create(struct mlx5hws_bwc_matcher * bwc_matcher,struct mlx5hws_table * table,u8 match_criteria_enable,struct mlx5hws_match_parameters * mask)814 hws_bwc_isolated_matcher_create(struct mlx5hws_bwc_matcher *bwc_matcher,
815 struct mlx5hws_table *table,
816 u8 match_criteria_enable,
817 struct mlx5hws_match_parameters *mask)
818 {
819 struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl;
820 struct mlx5hws_bwc_matcher *isolated_bwc_matcher;
821 struct mlx5hws_context *ctx = table->ctx;
822 int ret;
823
824 isolated_bwc_matcher = kzalloc(sizeof(*bwc_matcher), GFP_KERNEL);
825 if (!isolated_bwc_matcher)
826 return -ENOMEM;
827
828 bwc_matcher->complex->isolated_bwc_matcher = isolated_bwc_matcher;
829
830 /* Isolated BWC matcher needs access to the first BWC matcher */
831 isolated_bwc_matcher->complex_first_bwc_matcher = bwc_matcher;
832
833 /* Isolated matcher needs to match on REG_C_6,
834 * so make sure its criteria bit is on.
835 */
836 match_criteria_enable |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;
837
838 ret = mlx5hws_bwc_matcher_create_simple(isolated_bwc_matcher,
839 isolated_tbl,
840 0,
841 match_criteria_enable,
842 mask,
843 NULL);
844 if (ret) {
845 mlx5hws_err(ctx, "Complex matcher: failed creating isolated BWC matcher\n");
846 goto free_matcher;
847 }
848
849 return 0;
850
851 free_matcher:
852 kfree(bwc_matcher->complex->isolated_bwc_matcher);
853 return ret;
854 }
855
856 static void
hws_bwc_isolated_matcher_destroy(struct mlx5hws_bwc_matcher * bwc_matcher)857 hws_bwc_isolated_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher)
858 {
859 mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);
860 kfree(bwc_matcher);
861 }
862
863 static int
hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher * bwc_matcher,struct mlx5hws_table * table)864 hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher,
865 struct mlx5hws_table *table)
866 {
867 struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl;
868 u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
869 struct mlx5hws_context *ctx = table->ctx;
870 struct mlx5hws_action_mh_pattern ptrn;
871 int ret = 0;
872
873 /* Create action to jump to isolated table */
874
875 bwc_matcher->complex->action_go_to_tbl =
876 mlx5hws_action_create_dest_table(ctx,
877 isolated_tbl,
878 MLX5HWS_ACTION_FLAG_HWS_FDB);
879 if (!bwc_matcher->complex->action_go_to_tbl) {
880 mlx5hws_err(ctx, "Complex matcher: failed to create go-to-tbl action\n");
881 return -EINVAL;
882 }
883
884 /* Create modify header action to set REG_C_6 */
885
886 MLX5_SET(set_action_in, modify_hdr_action,
887 action_type, MLX5_MODIFICATION_TYPE_SET);
888 MLX5_SET(set_action_in, modify_hdr_action,
889 field, MLX5_MODI_META_REG_C_6);
890 MLX5_SET(set_action_in, modify_hdr_action,
891 length, 0); /* zero means length of 32 */
892 MLX5_SET(set_action_in, modify_hdr_action, offset, 0);
893 MLX5_SET(set_action_in, modify_hdr_action, data, 0);
894
895 ptrn.data = (void *)modify_hdr_action;
896 ptrn.sz = MLX5HWS_ACTION_DOUBLE_SIZE;
897
898 bwc_matcher->complex->action_metadata =
899 mlx5hws_action_create_modify_header(ctx, 1, &ptrn, 0,
900 MLX5HWS_ACTION_FLAG_HWS_FDB);
901 if (!bwc_matcher->complex->action_metadata) {
902 ret = -EINVAL;
903 goto destroy_action_go_to_tbl;
904 }
905
906 /* Create last action */
907
908 bwc_matcher->complex->action_last =
909 mlx5hws_action_create_last(ctx, MLX5HWS_ACTION_FLAG_HWS_FDB);
910 if (!bwc_matcher->complex->action_last) {
911 mlx5hws_err(ctx, "Complex matcher: failed to create last action\n");
912 ret = -EINVAL;
913 goto destroy_action_metadata;
914 }
915
916 return 0;
917
918 destroy_action_metadata:
919 mlx5hws_action_destroy(bwc_matcher->complex->action_metadata);
920 destroy_action_go_to_tbl:
921 mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl);
922 return ret;
923 }
924
925 static void
hws_bwc_isolated_actions_destroy(struct mlx5hws_bwc_matcher * bwc_matcher)926 hws_bwc_isolated_actions_destroy(struct mlx5hws_bwc_matcher *bwc_matcher)
927 {
928 mlx5hws_action_destroy(bwc_matcher->complex->action_last);
929 mlx5hws_action_destroy(bwc_matcher->complex->action_metadata);
930 mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl);
931 }
932
mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher * bwc_matcher,struct mlx5hws_table * table,u32 priority,u8 match_criteria_enable,struct mlx5hws_match_parameters * mask)933 int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher,
934 struct mlx5hws_table *table,
935 u32 priority,
936 u8 match_criteria_enable,
937 struct mlx5hws_match_parameters *mask)
938 {
939 enum mlx5hws_action_type complex_init_action_types[3];
940 struct mlx5hws_bwc_matcher *isolated_bwc_matcher;
941 struct mlx5hws_match_parameters mask_1 = {0};
942 struct mlx5hws_match_parameters mask_2 = {0};
943 struct mlx5hws_context *ctx = table->ctx;
944 int ret;
945
946 ret = hws_bwc_matcher_complex_params_create(table->ctx,
947 match_criteria_enable,
948 mask, &mask_1, &mask_2);
949 if (ret)
950 goto err;
951
952 bwc_matcher->complex =
953 kzalloc(sizeof(*bwc_matcher->complex), GFP_KERNEL);
954 if (!bwc_matcher->complex) {
955 ret = -ENOMEM;
956 goto free_masks;
957 }
958
959 ret = rhashtable_init(&bwc_matcher->complex->refcount_hash,
960 &hws_refcount_hash);
961 if (ret) {
962 mlx5hws_err(ctx, "Complex matcher: failed to initialize rhashtable\n");
963 goto free_complex;
964 }
965
966 mutex_init(&bwc_matcher->complex->hash_lock);
967 ida_init(&bwc_matcher->complex->metadata_ida);
968
969 /* Create initial action template for the first matcher.
970 * Usually the initial AT is just dummy, but in case of complex
971 * matcher we know exactly which actions should it have.
972 */
973
974 complex_init_action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
975 complex_init_action_types[1] = MLX5HWS_ACTION_TYP_TBL;
976 complex_init_action_types[2] = MLX5HWS_ACTION_TYP_LAST;
977
978 /* Create the first matcher */
979
980 ret = mlx5hws_bwc_matcher_create_simple(bwc_matcher,
981 table,
982 priority,
983 match_criteria_enable,
984 &mask_1,
985 complex_init_action_types);
986 if (ret)
987 goto destroy_ida;
988
989 /* Create isolated table to hold the second isolated matcher */
990
991 ret = hws_bwc_isolated_table_create(bwc_matcher, table);
992 if (ret) {
993 mlx5hws_err(ctx, "Complex matcher: failed creating isolated table\n");
994 goto destroy_first_matcher;
995 }
996
997 /* Now create the second BWC matcher - the isolated one */
998
999 ret = hws_bwc_isolated_matcher_create(bwc_matcher, table,
1000 match_criteria_enable, &mask_2);
1001 if (ret) {
1002 mlx5hws_err(ctx, "Complex matcher: failed creating isolated matcher\n");
1003 goto destroy_isolated_tbl;
1004 }
1005
1006 /* Create action for isolated matcher's rules */
1007
1008 ret = hws_bwc_isolated_actions_create(bwc_matcher, table);
1009 if (ret) {
1010 mlx5hws_err(ctx, "Complex matcher: failed creating isolated actions\n");
1011 goto destroy_isolated_matcher;
1012 }
1013
1014 hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2);
1015 return 0;
1016
1017 destroy_isolated_matcher:
1018 isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher;
1019 hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher);
1020 destroy_isolated_tbl:
1021 hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl);
1022 destroy_first_matcher:
1023 mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);
1024 destroy_ida:
1025 ida_destroy(&bwc_matcher->complex->metadata_ida);
1026 mutex_destroy(&bwc_matcher->complex->hash_lock);
1027 rhashtable_destroy(&bwc_matcher->complex->refcount_hash);
1028 free_complex:
1029 kfree(bwc_matcher->complex);
1030 bwc_matcher->complex = NULL;
1031 free_masks:
1032 hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2);
1033 err:
1034 return ret;
1035 }
1036
1037 void
mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher * bwc_matcher)1038 mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher)
1039 {
1040 struct mlx5hws_bwc_matcher *isolated_bwc_matcher =
1041 bwc_matcher->complex->isolated_bwc_matcher;
1042
1043 hws_bwc_isolated_actions_destroy(bwc_matcher);
1044 hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher);
1045 hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl);
1046 mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);
1047 ida_destroy(&bwc_matcher->complex->metadata_ida);
1048 mutex_destroy(&bwc_matcher->complex->hash_lock);
1049 rhashtable_destroy(&bwc_matcher->complex->refcount_hash);
1050 kfree(bwc_matcher->complex);
1051 bwc_matcher->complex = NULL;
1052 }
1053
1054 static void
hws_bwc_matcher_complex_hash_lock(struct mlx5hws_bwc_matcher * bwc_matcher)1055 hws_bwc_matcher_complex_hash_lock(struct mlx5hws_bwc_matcher *bwc_matcher)
1056 {
1057 mutex_lock(&bwc_matcher->complex->hash_lock);
1058 }
1059
1060 static void
hws_bwc_matcher_complex_hash_unlock(struct mlx5hws_bwc_matcher * bwc_matcher)1061 hws_bwc_matcher_complex_hash_unlock(struct mlx5hws_bwc_matcher *bwc_matcher)
1062 {
1063 mutex_unlock(&bwc_matcher->complex->hash_lock);
1064 }
1065
1066 static int
hws_bwc_rule_complex_hash_node_get(struct mlx5hws_bwc_rule * bwc_rule,struct mlx5hws_match_parameters * params)1067 hws_bwc_rule_complex_hash_node_get(struct mlx5hws_bwc_rule *bwc_rule,
1068 struct mlx5hws_match_parameters *params)
1069 {
1070 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
1071 struct mlx5hws_bwc_complex_rule_hash_node *node, *old_node;
1072 struct rhashtable *refcount_hash;
1073 int ret, i;
1074
1075 bwc_rule->complex_hash_node = NULL;
1076
1077 node = kzalloc(sizeof(*node), GFP_KERNEL);
1078 if (unlikely(!node))
1079 return -ENOMEM;
1080
1081 ret = ida_alloc(&bwc_matcher->complex->metadata_ida, GFP_KERNEL);
1082 if (ret < 0)
1083 goto err_free_node;
1084 node->tag = ret;
1085
1086 refcount_set(&node->refcount, 1);
1087
1088 /* Clear match buffer - turn off all the unrelated fields
1089 * in accordance with the match params mask for the first
1090 * matcher out of the two parts of the complex matcher.
1091 * The resulting mask is the key for the hash.
1092 */
1093 for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
1094 node->match_buf[i] = params->match_buf[i] &
1095 bwc_matcher->mt->match_param[i];
1096
1097 refcount_hash = &bwc_matcher->complex->refcount_hash;
1098 old_node = rhashtable_lookup_get_insert_fast(refcount_hash,
1099 &node->hash_node,
1100 hws_refcount_hash);
1101 if (IS_ERR(old_node)) {
1102 ret = PTR_ERR(old_node);
1103 goto err_free_ida;
1104 }
1105
1106 if (old_node) {
1107 /* Rule with the same tag already exists - update refcount */
1108 refcount_inc(&old_node->refcount);
1109 /* Let the new rule use the same tag as the existing rule.
1110 * Note that we don't have any indication for the rule creation
1111 * process that a rule with similar matching params already
1112 * exists - no harm done when this rule is be overwritten by
1113 * the same STE.
1114 * There's some performance advantage in skipping such cases,
1115 * so this is left for future optimizations.
1116 */
1117 ida_free(&bwc_matcher->complex->metadata_ida, node->tag);
1118 kfree(node);
1119 node = old_node;
1120 }
1121
1122 bwc_rule->complex_hash_node = node;
1123 return 0;
1124
1125 err_free_ida:
1126 ida_free(&bwc_matcher->complex->metadata_ida, node->tag);
1127 err_free_node:
1128 kfree(node);
1129 return ret;
1130 }
1131
1132 static void
hws_bwc_rule_complex_hash_node_put(struct mlx5hws_bwc_rule * bwc_rule,bool * is_last_rule)1133 hws_bwc_rule_complex_hash_node_put(struct mlx5hws_bwc_rule *bwc_rule,
1134 bool *is_last_rule)
1135 {
1136 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
1137 struct mlx5hws_bwc_complex_rule_hash_node *node;
1138
1139 if (is_last_rule)
1140 *is_last_rule = false;
1141
1142 node = bwc_rule->complex_hash_node;
1143 if (refcount_dec_and_test(&node->refcount)) {
1144 rhashtable_remove_fast(&bwc_matcher->complex->refcount_hash,
1145 &node->hash_node,
1146 hws_refcount_hash);
1147 ida_free(&bwc_matcher->complex->metadata_ida, node->tag);
1148 kfree(node);
1149 if (is_last_rule)
1150 *is_last_rule = true;
1151 }
1152
1153 bwc_rule->complex_hash_node = NULL;
1154 }
1155
mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule * bwc_rule,struct mlx5hws_match_parameters * params,u32 flow_source,struct mlx5hws_rule_action rule_actions[],u16 bwc_queue_idx)1156 int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule,
1157 struct mlx5hws_match_parameters *params,
1158 u32 flow_source,
1159 struct mlx5hws_rule_action rule_actions[],
1160 u16 bwc_queue_idx)
1161 {
1162 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
1163 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
1164 u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
1165 struct mlx5hws_rule_action rule_actions_1[3] = {0};
1166 struct mlx5hws_bwc_matcher *isolated_bwc_matcher;
1167 u32 *match_buf_2;
1168 u32 metadata_val;
1169 int ret = 0;
1170
1171 isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher;
1172 bwc_rule->isolated_bwc_rule =
1173 mlx5hws_bwc_rule_alloc(isolated_bwc_matcher);
1174 if (unlikely(!bwc_rule->isolated_bwc_rule))
1175 return -ENOMEM;
1176
1177 hws_bwc_matcher_complex_hash_lock(bwc_matcher);
1178
1179 /* Get a new hash node for this complex rule.
1180 * If this is a unique set of match params for the first matcher,
1181 * we will get a new hash node with newly allocated IDA.
1182 * Otherwise we will get an existing node with IDA and updated refcount.
1183 */
1184 ret = hws_bwc_rule_complex_hash_node_get(bwc_rule, params);
1185 if (unlikely(ret)) {
1186 mlx5hws_err(ctx, "Complex rule: failed getting RHT node for this rule\n");
1187 goto free_isolated_rule;
1188 }
1189
1190 /* No need to clear match buffer's fields in accordance to what
1191 * will actually be matched on first and second matchers.
1192 * Both matchers were created with the appropriate masks
1193 * and each of them holds the appropriate field copy array,
1194 * so rule creation will use only the fields that will be copied
1195 * in accordance with setters in field copy array.
1196 * We do, however, need to temporary allocate match buffer
1197 * for the second (isolated) rule in order to not modify
1198 * user's match params buffer.
1199 */
1200
1201 match_buf_2 = kmemdup(params->match_buf,
1202 MLX5_ST_SZ_BYTES(fte_match_param),
1203 GFP_KERNEL);
1204 if (unlikely(!match_buf_2)) {
1205 mlx5hws_err(ctx, "Complex rule: failed allocating match_buf\n");
1206 ret = -ENOMEM;
1207 goto hash_node_put;
1208 }
1209
1210 /* On 2nd matcher, use unique 32-bit ID as a matching tag */
1211 metadata_val = bwc_rule->complex_hash_node->tag;
1212 MLX5_SET(fte_match_param, match_buf_2,
1213 misc_parameters_2.metadata_reg_c_6, metadata_val);
1214
1215 /* Isolated rule's rule_actions contain all the original actions */
1216 ret = mlx5hws_bwc_rule_create_simple(bwc_rule->isolated_bwc_rule,
1217 match_buf_2,
1218 rule_actions,
1219 flow_source,
1220 bwc_queue_idx);
1221 kfree(match_buf_2);
1222 if (unlikely(ret)) {
1223 mlx5hws_err(ctx,
1224 "Complex rule: failed creating isolated BWC rule (%d)\n",
1225 ret);
1226 goto hash_node_put;
1227 }
1228
1229 /* First rule's rule_actions contain setting metadata and
1230 * jump to isolated table that contains the second matcher.
1231 * Set metadata value to a unique value for this rule.
1232 */
1233
1234 MLX5_SET(set_action_in, modify_hdr_action,
1235 action_type, MLX5_MODIFICATION_TYPE_SET);
1236 MLX5_SET(set_action_in, modify_hdr_action,
1237 field, MLX5_MODI_META_REG_C_6);
1238 MLX5_SET(set_action_in, modify_hdr_action,
1239 length, 0); /* zero means length of 32 */
1240 MLX5_SET(set_action_in, modify_hdr_action,
1241 offset, 0);
1242 MLX5_SET(set_action_in, modify_hdr_action,
1243 data, metadata_val);
1244
1245 rule_actions_1[0].action = bwc_matcher->complex->action_metadata;
1246 rule_actions_1[0].modify_header.offset = 0;
1247 rule_actions_1[0].modify_header.data = modify_hdr_action;
1248
1249 rule_actions_1[1].action = bwc_matcher->complex->action_go_to_tbl;
1250 rule_actions_1[2].action = bwc_matcher->complex->action_last;
1251
1252 ret = mlx5hws_bwc_rule_create_simple(bwc_rule,
1253 params->match_buf,
1254 rule_actions_1,
1255 flow_source,
1256 bwc_queue_idx);
1257
1258 if (unlikely(ret)) {
1259 mlx5hws_err(ctx,
1260 "Complex rule: failed creating first BWC rule (%d)\n",
1261 ret);
1262 goto destroy_isolated_rule;
1263 }
1264
1265 hws_bwc_matcher_complex_hash_unlock(bwc_matcher);
1266
1267 return 0;
1268
1269 destroy_isolated_rule:
1270 mlx5hws_bwc_rule_destroy_simple(bwc_rule->isolated_bwc_rule);
1271 hash_node_put:
1272 hws_bwc_rule_complex_hash_node_put(bwc_rule, NULL);
1273 free_isolated_rule:
1274 hws_bwc_matcher_complex_hash_unlock(bwc_matcher);
1275 mlx5hws_bwc_rule_free(bwc_rule->isolated_bwc_rule);
1276 return ret;
1277 }
1278
mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule * bwc_rule)1279 int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule)
1280 {
1281 struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx;
1282 struct mlx5hws_bwc_rule *isolated_bwc_rule;
1283 int ret_isolated, ret;
1284 bool is_last_rule;
1285
1286 hws_bwc_matcher_complex_hash_lock(bwc_rule->bwc_matcher);
1287
1288 hws_bwc_rule_complex_hash_node_put(bwc_rule, &is_last_rule);
1289 bwc_rule->rule->skip_delete = !is_last_rule;
1290
1291 ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule);
1292 if (unlikely(ret))
1293 mlx5hws_err(ctx, "BWC complex rule: failed destroying first rule\n");
1294
1295 isolated_bwc_rule = bwc_rule->isolated_bwc_rule;
1296 ret_isolated = mlx5hws_bwc_rule_destroy_simple(isolated_bwc_rule);
1297 if (unlikely(ret_isolated))
1298 mlx5hws_err(ctx, "BWC complex rule: failed destroying second (isolated) rule\n");
1299
1300 hws_bwc_matcher_complex_hash_unlock(bwc_rule->bwc_matcher);
1301
1302 mlx5hws_bwc_rule_free(isolated_bwc_rule);
1303
1304 return ret || ret_isolated;
1305 }
1306
1307 static void
hws_bwc_matcher_clear_hash_rtcs(struct mlx5hws_bwc_matcher * bwc_matcher)1308 hws_bwc_matcher_clear_hash_rtcs(struct mlx5hws_bwc_matcher *bwc_matcher)
1309 {
1310 struct mlx5hws_bwc_complex_rule_hash_node *node;
1311 struct rhashtable_iter iter;
1312
1313 rhashtable_walk_enter(&bwc_matcher->complex->refcount_hash, &iter);
1314 rhashtable_walk_start(&iter);
1315
1316 while ((node = rhashtable_walk_next(&iter)) != NULL) {
1317 if (IS_ERR(node))
1318 continue;
1319 node->rtc_valid = false;
1320 }
1321
1322 rhashtable_walk_stop(&iter);
1323 rhashtable_walk_exit(&iter);
1324 }
1325
1326 int
mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher * bwc_matcher)1327 mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher)
1328 {
1329 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
1330 struct mlx5hws_matcher *matcher = bwc_matcher->matcher;
1331 u16 bwc_queues = mlx5hws_bwc_queues(ctx);
1332 struct mlx5hws_bwc_rule *tmp_bwc_rule;
1333 struct mlx5hws_rule_attr rule_attr;
1334 struct mlx5hws_table *isolated_tbl;
1335 int move_error = 0, poll_error = 0;
1336 struct mlx5hws_rule *tmp_rule;
1337 struct list_head *rules_list;
1338 u32 expected_completions = 1;
1339 u32 end_ft_id;
1340 int i, ret;
1341
1342 /* We are rehashing the matcher that is the first part of the complex
1343 * matcher. Need to update the isolated matcher to point to the end_ft
1344 * of this new matcher. This needs to be done before moving any rules
1345 * to prevent possible steering loops.
1346 */
1347 isolated_tbl = bwc_matcher->complex->isolated_tbl;
1348 end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id;
1349 ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl, end_ft_id);
1350 if (ret) {
1351 mlx5hws_err(ctx,
1352 "Failed updating end_ft of isolated matcher (%d)\n",
1353 ret);
1354 return ret;
1355 }
1356
1357 hws_bwc_matcher_clear_hash_rtcs(bwc_matcher);
1358
1359 mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr);
1360
1361 for (i = 0; i < bwc_queues; i++) {
1362 rules_list = &bwc_matcher->rules[i];
1363 if (list_empty(rules_list))
1364 continue;
1365
1366 rule_attr.queue_id = mlx5hws_bwc_get_queue_id(ctx, i);
1367
1368 list_for_each_entry(tmp_bwc_rule, rules_list, list_node) {
1369 /* Check if a rule with similar tag has already
1370 * been moved.
1371 */
1372 if (tmp_bwc_rule->complex_hash_node->rtc_valid) {
1373 /* This rule is a duplicate of rule with similar
1374 * tag that has already been moved earlier.
1375 * Just update this rule's RTCs.
1376 */
1377 tmp_bwc_rule->rule->rtc_0 =
1378 tmp_bwc_rule->complex_hash_node->rtc_0;
1379 tmp_bwc_rule->rule->rtc_1 =
1380 tmp_bwc_rule->complex_hash_node->rtc_1;
1381 tmp_bwc_rule->rule->matcher =
1382 tmp_bwc_rule->rule->matcher->resize_dst;
1383 continue;
1384 }
1385
1386 /* First time we're moving rule with this tag.
1387 * Move it for real.
1388 */
1389 tmp_rule = tmp_bwc_rule->rule;
1390 tmp_rule->skip_delete = false;
1391 ret = mlx5hws_matcher_resize_rule_move(matcher,
1392 tmp_rule,
1393 &rule_attr);
1394 if (unlikely(ret)) {
1395 if (!move_error) {
1396 mlx5hws_err(ctx,
1397 "Moving complex BWC rule: move failed (%d), attempting to move rest of the rules\n",
1398 ret);
1399 move_error = ret;
1400 }
1401 /* Rule wasn't queued, no need to poll */
1402 continue;
1403 }
1404
1405 expected_completions = 1;
1406 ret = mlx5hws_bwc_queue_poll(ctx,
1407 rule_attr.queue_id,
1408 &expected_completions,
1409 true);
1410 if (unlikely(ret)) {
1411 if (ret == -ETIMEDOUT) {
1412 mlx5hws_err(ctx,
1413 "Moving complex BWC rule: timeout polling for completions (%d), aborting rehash\n",
1414 ret);
1415 return ret;
1416 }
1417 if (!poll_error) {
1418 mlx5hws_err(ctx,
1419 "Moving complex BWC rule: polling for completions failed (%d), attempting to move rest of the rules\n",
1420 ret);
1421 poll_error = ret;
1422 }
1423 }
1424
1425 /* Done moving the rule to the new matcher,
1426 * now update RTCs for all the duplicated rules.
1427 */
1428 tmp_bwc_rule->complex_hash_node->rtc_0 =
1429 tmp_bwc_rule->rule->rtc_0;
1430 tmp_bwc_rule->complex_hash_node->rtc_1 =
1431 tmp_bwc_rule->rule->rtc_1;
1432
1433 tmp_bwc_rule->complex_hash_node->rtc_valid = true;
1434 }
1435 }
1436
1437 /* Return the first error that happened */
1438 if (unlikely(move_error))
1439 return move_error;
1440 if (unlikely(poll_error))
1441 return poll_error;
1442
1443 return ret;
1444 }
1445