1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/uverbs_std_types.h>
11 #include <rdma/mlx5_user_ioctl_cmds.h>
12 #include <rdma/mlx5_user_ioctl_verbs.h>
13 #include <rdma/ib_hdrs.h>
14 #include <rdma/ib_umem.h>
15 #include <rdma/ib_ucaps.h>
16 #include <linux/mlx5/driver.h>
17 #include <linux/mlx5/fs.h>
18 #include <linux/mlx5/fs_helpers.h>
19 #include <linux/mlx5/eswitch.h>
20 #include <net/inet_ecn.h>
21 #include "mlx5_ib.h"
22 #include "counters.h"
23 #include "devx.h"
24 #include "fs.h"
25
26 #define UVERBS_MODULE_NAME mlx5_ib
27 #include <rdma/uverbs_named_ioctl.h>
28
29 enum {
30 MATCH_CRITERIA_ENABLE_OUTER_BIT,
31 MATCH_CRITERIA_ENABLE_MISC_BIT,
32 MATCH_CRITERIA_ENABLE_INNER_BIT,
33 MATCH_CRITERIA_ENABLE_MISC2_BIT
34 };
35
36
37 struct mlx5_per_qp_opfc {
38 struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX];
39 };
40
41 #define HEADER_IS_ZERO(match_criteria, headers) \
42 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
43 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
44
get_match_criteria_enable(u32 * match_criteria)45 static u8 get_match_criteria_enable(u32 *match_criteria)
46 {
47 u8 match_criteria_enable;
48
49 match_criteria_enable =
50 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
51 MATCH_CRITERIA_ENABLE_OUTER_BIT;
52 match_criteria_enable |=
53 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
54 MATCH_CRITERIA_ENABLE_MISC_BIT;
55 match_criteria_enable |=
56 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
57 MATCH_CRITERIA_ENABLE_INNER_BIT;
58 match_criteria_enable |=
59 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
60 MATCH_CRITERIA_ENABLE_MISC2_BIT;
61
62 return match_criteria_enable;
63 }
64
set_proto(void * outer_c,void * outer_v,u8 mask,u8 val)65 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
66 {
67 u8 entry_mask;
68 u8 entry_val;
69 int err = 0;
70
71 if (!mask)
72 goto out;
73
74 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
75 ip_protocol);
76 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
77 ip_protocol);
78 if (!entry_mask) {
79 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
80 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
81 goto out;
82 }
83 /* Don't override existing ip protocol */
84 if (mask != entry_mask || val != entry_val)
85 err = -EINVAL;
86 out:
87 return err;
88 }
89
set_flow_label(void * misc_c,void * misc_v,u32 mask,u32 val,bool inner)90 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
91 bool inner)
92 {
93 if (inner) {
94 MLX5_SET(fte_match_set_misc,
95 misc_c, inner_ipv6_flow_label, mask);
96 MLX5_SET(fte_match_set_misc,
97 misc_v, inner_ipv6_flow_label, val);
98 } else {
99 MLX5_SET(fte_match_set_misc,
100 misc_c, outer_ipv6_flow_label, mask);
101 MLX5_SET(fte_match_set_misc,
102 misc_v, outer_ipv6_flow_label, val);
103 }
104 }
105
set_tos(void * outer_c,void * outer_v,u8 mask,u8 val)106 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
107 {
108 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
109 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
110 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
111 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
112 }
113
check_mpls_supp_fields(u32 field_support,const __be32 * set_mask)114 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
115 {
116 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
117 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
118 return -EOPNOTSUPP;
119
120 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
121 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
122 return -EOPNOTSUPP;
123
124 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
125 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
126 return -EOPNOTSUPP;
127
128 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
129 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
130 return -EOPNOTSUPP;
131
132 return 0;
133 }
134
135 #define LAST_ETH_FIELD vlan_tag
136 #define LAST_IPV4_FIELD tos
137 #define LAST_IPV6_FIELD traffic_class
138 #define LAST_TCP_UDP_FIELD src_port
139 #define LAST_TUNNEL_FIELD tunnel_id
140 #define LAST_FLOW_TAG_FIELD tag_id
141 #define LAST_DROP_FIELD size
142 #define LAST_COUNTERS_FIELD counters
143
144 /* Field is the last supported field */
145 #define FIELDS_NOT_SUPPORTED(filter, field) \
146 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \
147 sizeof(filter) - offsetofend(typeof(filter), field))
148
parse_flow_flow_action(struct mlx5_ib_flow_action * maction,bool is_egress,struct mlx5_flow_act * action)149 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
150 bool is_egress,
151 struct mlx5_flow_act *action)
152 {
153
154 switch (maction->ib_action.type) {
155 case IB_FLOW_ACTION_UNSPECIFIED:
156 if (maction->flow_action_raw.sub_type ==
157 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
158 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
159 return -EINVAL;
160 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
161 action->modify_hdr =
162 maction->flow_action_raw.modify_hdr;
163 return 0;
164 }
165 if (maction->flow_action_raw.sub_type ==
166 MLX5_IB_FLOW_ACTION_DECAP) {
167 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
168 return -EINVAL;
169 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
170 return 0;
171 }
172 if (maction->flow_action_raw.sub_type ==
173 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
174 if (action->action &
175 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
176 return -EINVAL;
177 action->action |=
178 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
179 action->pkt_reformat =
180 maction->flow_action_raw.pkt_reformat;
181 return 0;
182 }
183 fallthrough;
184 default:
185 return -EOPNOTSUPP;
186 }
187 }
188
parse_flow_attr(struct mlx5_core_dev * mdev,struct mlx5_flow_spec * spec,const union ib_flow_spec * ib_spec,const struct ib_flow_attr * flow_attr,struct mlx5_flow_act * action,u32 prev_type)189 static int parse_flow_attr(struct mlx5_core_dev *mdev,
190 struct mlx5_flow_spec *spec,
191 const union ib_flow_spec *ib_spec,
192 const struct ib_flow_attr *flow_attr,
193 struct mlx5_flow_act *action, u32 prev_type)
194 {
195 struct mlx5_flow_context *flow_context = &spec->flow_context;
196 u32 *match_c = spec->match_criteria;
197 u32 *match_v = spec->match_value;
198 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
199 misc_parameters);
200 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
201 misc_parameters);
202 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
203 misc_parameters_2);
204 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
205 misc_parameters_2);
206 void *headers_c;
207 void *headers_v;
208 int match_ipv;
209 int ret;
210
211 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
212 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
213 inner_headers);
214 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
215 inner_headers);
216 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
217 ft_field_support.inner_ip_version);
218 } else {
219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
220 outer_headers);
221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
222 outer_headers);
223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
224 ft_field_support.outer_ip_version);
225 }
226
227 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
228 case IB_FLOW_SPEC_ETH:
229 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
230 return -EOPNOTSUPP;
231
232 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
233 dmac_47_16),
234 ib_spec->eth.mask.dst_mac);
235 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
236 dmac_47_16),
237 ib_spec->eth.val.dst_mac);
238
239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
240 smac_47_16),
241 ib_spec->eth.mask.src_mac);
242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
243 smac_47_16),
244 ib_spec->eth.val.src_mac);
245
246 if (ib_spec->eth.mask.vlan_tag) {
247 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
248 cvlan_tag, 1);
249 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
250 cvlan_tag, 1);
251
252 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
253 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
254 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
255 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
256
257 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
258 first_cfi,
259 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
260 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
261 first_cfi,
262 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
263
264 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
265 first_prio,
266 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
267 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
268 first_prio,
269 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
270 }
271 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
272 ethertype, ntohs(ib_spec->eth.mask.ether_type));
273 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
274 ethertype, ntohs(ib_spec->eth.val.ether_type));
275 break;
276 case IB_FLOW_SPEC_IPV4:
277 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
278 return -EOPNOTSUPP;
279
280 if (match_ipv) {
281 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
282 ip_version, 0xf);
283 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
284 ip_version, MLX5_FS_IPV4_VERSION);
285 } else {
286 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
287 ethertype, 0xffff);
288 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
289 ethertype, ETH_P_IP);
290 }
291
292 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
293 src_ipv4_src_ipv6.ipv4_layout.ipv4),
294 &ib_spec->ipv4.mask.src_ip,
295 sizeof(ib_spec->ipv4.mask.src_ip));
296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
297 src_ipv4_src_ipv6.ipv4_layout.ipv4),
298 &ib_spec->ipv4.val.src_ip,
299 sizeof(ib_spec->ipv4.val.src_ip));
300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
301 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
302 &ib_spec->ipv4.mask.dst_ip,
303 sizeof(ib_spec->ipv4.mask.dst_ip));
304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
306 &ib_spec->ipv4.val.dst_ip,
307 sizeof(ib_spec->ipv4.val.dst_ip));
308
309 set_tos(headers_c, headers_v,
310 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
311
312 if (set_proto(headers_c, headers_v,
313 ib_spec->ipv4.mask.proto,
314 ib_spec->ipv4.val.proto))
315 return -EINVAL;
316 break;
317 case IB_FLOW_SPEC_IPV6:
318 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
319 return -EOPNOTSUPP;
320
321 if (match_ipv) {
322 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
323 ip_version, 0xf);
324 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
325 ip_version, MLX5_FS_IPV6_VERSION);
326 } else {
327 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
328 ethertype, 0xffff);
329 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
330 ethertype, ETH_P_IPV6);
331 }
332
333 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
334 src_ipv4_src_ipv6.ipv6_layout.ipv6),
335 &ib_spec->ipv6.mask.src_ip,
336 sizeof(ib_spec->ipv6.mask.src_ip));
337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
338 src_ipv4_src_ipv6.ipv6_layout.ipv6),
339 &ib_spec->ipv6.val.src_ip,
340 sizeof(ib_spec->ipv6.val.src_ip));
341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
342 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
343 &ib_spec->ipv6.mask.dst_ip,
344 sizeof(ib_spec->ipv6.mask.dst_ip));
345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
347 &ib_spec->ipv6.val.dst_ip,
348 sizeof(ib_spec->ipv6.val.dst_ip));
349
350 set_tos(headers_c, headers_v,
351 ib_spec->ipv6.mask.traffic_class,
352 ib_spec->ipv6.val.traffic_class);
353
354 if (set_proto(headers_c, headers_v,
355 ib_spec->ipv6.mask.next_hdr,
356 ib_spec->ipv6.val.next_hdr))
357 return -EINVAL;
358
359 set_flow_label(misc_params_c, misc_params_v,
360 ntohl(ib_spec->ipv6.mask.flow_label),
361 ntohl(ib_spec->ipv6.val.flow_label),
362 ib_spec->type & IB_FLOW_SPEC_INNER);
363 break;
364 case IB_FLOW_SPEC_ESP:
365 return -EOPNOTSUPP;
366 case IB_FLOW_SPEC_TCP:
367 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
368 LAST_TCP_UDP_FIELD))
369 return -EOPNOTSUPP;
370
371 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
372 return -EINVAL;
373
374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
375 ntohs(ib_spec->tcp_udp.mask.src_port));
376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
377 ntohs(ib_spec->tcp_udp.val.src_port));
378
379 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
380 ntohs(ib_spec->tcp_udp.mask.dst_port));
381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
382 ntohs(ib_spec->tcp_udp.val.dst_port));
383 break;
384 case IB_FLOW_SPEC_UDP:
385 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
386 LAST_TCP_UDP_FIELD))
387 return -EOPNOTSUPP;
388
389 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
390 return -EINVAL;
391
392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
393 ntohs(ib_spec->tcp_udp.mask.src_port));
394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
395 ntohs(ib_spec->tcp_udp.val.src_port));
396
397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
398 ntohs(ib_spec->tcp_udp.mask.dst_port));
399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
400 ntohs(ib_spec->tcp_udp.val.dst_port));
401 break;
402 case IB_FLOW_SPEC_GRE:
403 if (ib_spec->gre.mask.c_ks_res0_ver)
404 return -EOPNOTSUPP;
405
406 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
407 return -EINVAL;
408
409 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
410 0xff);
411 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
412 IPPROTO_GRE);
413
414 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
415 ntohs(ib_spec->gre.mask.protocol));
416 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
417 ntohs(ib_spec->gre.val.protocol));
418
419 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
420 gre_key.nvgre.hi),
421 &ib_spec->gre.mask.key,
422 sizeof(ib_spec->gre.mask.key));
423 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
424 gre_key.nvgre.hi),
425 &ib_spec->gre.val.key,
426 sizeof(ib_spec->gre.val.key));
427 break;
428 case IB_FLOW_SPEC_MPLS:
429 switch (prev_type) {
430 case IB_FLOW_SPEC_UDP:
431 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
432 ft_field_support.outer_first_mpls_over_udp),
433 &ib_spec->mpls.mask.tag))
434 return -EOPNOTSUPP;
435
436 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
437 outer_first_mpls_over_udp),
438 &ib_spec->mpls.val.tag,
439 sizeof(ib_spec->mpls.val.tag));
440 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
441 outer_first_mpls_over_udp),
442 &ib_spec->mpls.mask.tag,
443 sizeof(ib_spec->mpls.mask.tag));
444 break;
445 case IB_FLOW_SPEC_GRE:
446 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
447 ft_field_support.outer_first_mpls_over_gre),
448 &ib_spec->mpls.mask.tag))
449 return -EOPNOTSUPP;
450
451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
452 outer_first_mpls_over_gre),
453 &ib_spec->mpls.val.tag,
454 sizeof(ib_spec->mpls.val.tag));
455 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
456 outer_first_mpls_over_gre),
457 &ib_spec->mpls.mask.tag,
458 sizeof(ib_spec->mpls.mask.tag));
459 break;
460 default:
461 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
462 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
463 ft_field_support.inner_first_mpls),
464 &ib_spec->mpls.mask.tag))
465 return -EOPNOTSUPP;
466
467 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
468 inner_first_mpls),
469 &ib_spec->mpls.val.tag,
470 sizeof(ib_spec->mpls.val.tag));
471 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
472 inner_first_mpls),
473 &ib_spec->mpls.mask.tag,
474 sizeof(ib_spec->mpls.mask.tag));
475 } else {
476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
477 ft_field_support.outer_first_mpls),
478 &ib_spec->mpls.mask.tag))
479 return -EOPNOTSUPP;
480
481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
482 outer_first_mpls),
483 &ib_spec->mpls.val.tag,
484 sizeof(ib_spec->mpls.val.tag));
485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
486 outer_first_mpls),
487 &ib_spec->mpls.mask.tag,
488 sizeof(ib_spec->mpls.mask.tag));
489 }
490 }
491 break;
492 case IB_FLOW_SPEC_VXLAN_TUNNEL:
493 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
494 LAST_TUNNEL_FIELD))
495 return -EOPNOTSUPP;
496
497 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
498 ntohl(ib_spec->tunnel.mask.tunnel_id));
499 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
500 ntohl(ib_spec->tunnel.val.tunnel_id));
501 break;
502 case IB_FLOW_SPEC_ACTION_TAG:
503 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
504 LAST_FLOW_TAG_FIELD))
505 return -EOPNOTSUPP;
506 if (ib_spec->flow_tag.tag_id >= BIT(24))
507 return -EINVAL;
508
509 flow_context->flow_tag = ib_spec->flow_tag.tag_id;
510 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
511 break;
512 case IB_FLOW_SPEC_ACTION_DROP:
513 if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
514 LAST_DROP_FIELD))
515 return -EOPNOTSUPP;
516 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
517 break;
518 case IB_FLOW_SPEC_ACTION_HANDLE:
519 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
520 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
521 if (ret)
522 return ret;
523 break;
524 case IB_FLOW_SPEC_ACTION_COUNT:
525 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
526 LAST_COUNTERS_FIELD))
527 return -EOPNOTSUPP;
528
529 /* for now support only one counters spec per flow */
530 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
531 return -EINVAL;
532
533 action->counters = ib_spec->flow_count.counters;
534 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
535 break;
536 default:
537 return -EINVAL;
538 }
539
540 return 0;
541 }
542
543 /* If a flow could catch both multicast and unicast packets,
544 * it won't fall into the multicast flow steering table and this rule
545 * could steal other multicast packets.
546 */
flow_is_multicast_only(const struct ib_flow_attr * ib_attr)547 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
548 {
549 union ib_flow_spec *flow_spec;
550
551 if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
552 ib_attr->num_of_specs < 1)
553 return false;
554
555 flow_spec = (union ib_flow_spec *)(ib_attr + 1);
556 if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
557 struct ib_flow_spec_ipv4 *ipv4_spec;
558
559 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
560 if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
561 return true;
562
563 return false;
564 }
565
566 if (flow_spec->type == IB_FLOW_SPEC_ETH) {
567 struct ib_flow_spec_eth *eth_spec;
568
569 eth_spec = (struct ib_flow_spec_eth *)flow_spec;
570 return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
571 is_multicast_ether_addr(eth_spec->val.dst_mac);
572 }
573
574 return false;
575 }
576
is_valid_ethertype(struct mlx5_core_dev * mdev,const struct ib_flow_attr * flow_attr,bool check_inner)577 static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
578 const struct ib_flow_attr *flow_attr,
579 bool check_inner)
580 {
581 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
582 int match_ipv = check_inner ?
583 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
584 ft_field_support.inner_ip_version) :
585 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
586 ft_field_support.outer_ip_version);
587 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
588 bool ipv4_spec_valid, ipv6_spec_valid;
589 unsigned int ip_spec_type = 0;
590 bool has_ethertype = false;
591 unsigned int spec_index;
592 bool mask_valid = true;
593 u16 eth_type = 0;
594 bool type_valid;
595
596 /* Validate that ethertype is correct */
597 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
598 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
599 ib_spec->eth.mask.ether_type) {
600 mask_valid = (ib_spec->eth.mask.ether_type ==
601 htons(0xffff));
602 has_ethertype = true;
603 eth_type = ntohs(ib_spec->eth.val.ether_type);
604 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
605 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
606 ip_spec_type = ib_spec->type;
607 }
608 ib_spec = (void *)ib_spec + ib_spec->size;
609 }
610
611 type_valid = (!has_ethertype) || (!ip_spec_type);
612 if (!type_valid && mask_valid) {
613 ipv4_spec_valid = (eth_type == ETH_P_IP) &&
614 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
615 ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
616 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
617
618 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
619 (((eth_type == ETH_P_MPLS_UC) ||
620 (eth_type == ETH_P_MPLS_MC)) && match_ipv);
621 }
622
623 return type_valid;
624 }
625
is_valid_attr(struct mlx5_core_dev * mdev,const struct ib_flow_attr * flow_attr)626 static bool is_valid_attr(struct mlx5_core_dev *mdev,
627 const struct ib_flow_attr *flow_attr)
628 {
629 return is_valid_ethertype(mdev, flow_attr, false) &&
630 is_valid_ethertype(mdev, flow_attr, true);
631 }
632
put_flow_table(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * prio,bool ft_added)633 static void put_flow_table(struct mlx5_ib_dev *dev,
634 struct mlx5_ib_flow_prio *prio, bool ft_added)
635 {
636 prio->refcount -= !!ft_added;
637 if (!prio->refcount) {
638 mlx5_destroy_flow_table(prio->flow_table);
639 prio->flow_table = NULL;
640 }
641 }
642
mlx5_ib_destroy_flow(struct ib_flow * flow_id)643 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
644 {
645 struct mlx5_ib_flow_handler *handler = container_of(flow_id,
646 struct mlx5_ib_flow_handler,
647 ibflow);
648 struct mlx5_ib_flow_handler *iter, *tmp;
649 struct mlx5_ib_dev *dev = handler->dev;
650
651 mutex_lock(&dev->flow_db->lock);
652
653 list_for_each_entry_safe(iter, tmp, &handler->list, list) {
654 mlx5_del_flow_rules(iter->rule);
655 put_flow_table(dev, iter->prio, true);
656 list_del(&iter->list);
657 kfree(iter);
658 }
659
660 mlx5_del_flow_rules(handler->rule);
661 put_flow_table(dev, handler->prio, true);
662 mlx5_ib_counters_clear_description(handler->ibcounters);
663 mutex_unlock(&dev->flow_db->lock);
664 if (handler->flow_matcher)
665 atomic_dec(&handler->flow_matcher->usecnt);
666 kfree(handler);
667
668 return 0;
669 }
670
ib_prio_to_core_prio(unsigned int priority,bool dont_trap)671 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
672 {
673 priority *= 2;
674 if (!dont_trap)
675 priority++;
676 return priority;
677 }
678
679 enum flow_table_type {
680 MLX5_IB_FT_RX,
681 MLX5_IB_FT_TX
682 };
683
684 #define MLX5_FS_MAX_TYPES 6
685 #define MLX5_FS_MAX_ENTRIES BIT(16)
686
mlx5_ib_shared_ft_allowed(struct ib_device * device)687 static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device)
688 {
689 struct mlx5_ib_dev *dev = to_mdev(device);
690
691 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed);
692 }
693
_get_prio(struct mlx5_flow_namespace * ns,struct mlx5_ib_flow_prio * prio,struct mlx5_flow_table_attr * ft_attr)694 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,
695 struct mlx5_ib_flow_prio *prio,
696 struct mlx5_flow_table_attr *ft_attr)
697 {
698 struct mlx5_flow_table *ft;
699
700 ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr);
701 if (IS_ERR(ft))
702 return ERR_CAST(ft);
703
704 prio->flow_table = ft;
705 prio->refcount = 0;
706 return prio;
707 }
708
get_flow_table(struct mlx5_ib_dev * dev,struct ib_flow_attr * flow_attr,enum flow_table_type ft_type)709 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
710 struct ib_flow_attr *flow_attr,
711 enum flow_table_type ft_type)
712 {
713 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
714 struct mlx5_flow_table_attr ft_attr = {};
715 struct mlx5_flow_namespace *ns = NULL;
716 enum mlx5_flow_namespace_type fn_type;
717 struct mlx5_ib_flow_prio *prio;
718 struct mlx5_flow_table *ft;
719 int max_table_size;
720 int num_entries;
721 int num_groups;
722 bool esw_encap;
723 u32 flags = 0;
724 int priority;
725
726 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
727 log_max_ft_size));
728 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
729 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
730 switch (flow_attr->type) {
731 case IB_FLOW_ATTR_NORMAL:
732 if (flow_is_multicast_only(flow_attr) && !dont_trap)
733 priority = MLX5_IB_FLOW_MCAST_PRIO;
734 else
735 priority = ib_prio_to_core_prio(flow_attr->priority,
736 dont_trap);
737 if (ft_type == MLX5_IB_FT_RX) {
738 fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
739 prio = &dev->flow_db->prios[priority];
740 if (!dev->is_rep && !esw_encap &&
741 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
742 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
743 if (!dev->is_rep && !esw_encap &&
744 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
745 reformat_l3_tunnel_to_l2))
746 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
747 } else {
748 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(
749 dev->mdev, log_max_ft_size));
750 fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
751 prio = &dev->flow_db->egress_prios[priority];
752 if (!dev->is_rep && !esw_encap &&
753 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
754 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
755 }
756 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
757 num_entries = MLX5_FS_MAX_ENTRIES;
758 num_groups = MLX5_FS_MAX_TYPES;
759 break;
760 case IB_FLOW_ATTR_ALL_DEFAULT:
761 case IB_FLOW_ATTR_MC_DEFAULT:
762 ns = mlx5_get_flow_namespace(dev->mdev,
763 MLX5_FLOW_NAMESPACE_LEFTOVERS);
764 build_leftovers_ft_param(&priority, &num_entries, &num_groups);
765 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
766 break;
767 case IB_FLOW_ATTR_SNIFFER:
768 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
769 allow_sniffer_and_nic_rx_shared_tir))
770 return ERR_PTR(-EOPNOTSUPP);
771
772 ns = mlx5_get_flow_namespace(
773 dev->mdev, ft_type == MLX5_IB_FT_RX ?
774 MLX5_FLOW_NAMESPACE_SNIFFER_RX :
775 MLX5_FLOW_NAMESPACE_SNIFFER_TX);
776
777 prio = &dev->flow_db->sniffer[ft_type];
778 priority = 0;
779 num_entries = 1;
780 num_groups = 1;
781 break;
782 default:
783 break;
784 }
785
786 if (!ns)
787 return ERR_PTR(-EOPNOTSUPP);
788
789 max_table_size = min_t(int, num_entries, max_table_size);
790
791 ft = prio->flow_table;
792 if (ft)
793 return prio;
794
795 ft_attr.prio = priority;
796 ft_attr.max_fte = max_table_size;
797 ft_attr.flags = flags;
798 ft_attr.autogroup.max_num_groups = num_groups;
799 return _get_prio(ns, prio, &ft_attr);
800 }
801
802 enum {
803 RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO,
804 RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO,
805 RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
806 RDMA_RX_ECN_OPCOUNTER_PRIO,
807 RDMA_RX_CNP_OPCOUNTER_PRIO,
808 RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO,
809 };
810
811 enum {
812 RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO,
813 RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
814 RDMA_TX_CNP_OPCOUNTER_PRIO,
815 RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO,
816 };
817
set_vhca_port_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec)818 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num,
819 struct mlx5_flow_spec *spec)
820 {
821 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
822 ft_field_support.source_vhca_port) ||
823 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
824 ft_field_support.source_vhca_port))
825 return -EOPNOTSUPP;
826
827 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria,
828 misc_parameters.source_vhca_port);
829 MLX5_SET(fte_match_param, &spec->match_value,
830 misc_parameters.source_vhca_port, port_num);
831
832 return 0;
833 }
834
set_ecn_ce_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec,int ipv)835 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num,
836 struct mlx5_flow_spec *spec, int ipv)
837 {
838 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
839 ft_field_support.outer_ip_version))
840 return -EOPNOTSUPP;
841
842 if (mlx5_core_mp_enabled(dev->mdev) &&
843 set_vhca_port_spec(dev, port_num, spec))
844 return -EOPNOTSUPP;
845
846 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
847 outer_headers.ip_ecn);
848 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn,
849 INET_ECN_CE);
850 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
851 outer_headers.ip_version);
852 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
853 ipv);
854
855 spec->match_criteria_enable =
856 get_match_criteria_enable(spec->match_criteria);
857
858 return 0;
859 }
860
set_cnp_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec)861 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num,
862 struct mlx5_flow_spec *spec)
863 {
864 if (mlx5_core_mp_enabled(dev->mdev) &&
865 set_vhca_port_spec(dev, port_num, spec))
866 return -EOPNOTSUPP;
867
868 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
869 misc_parameters.bth_opcode);
870 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode,
871 IB_BTH_OPCODE_CNP);
872
873 spec->match_criteria_enable =
874 get_match_criteria_enable(spec->match_criteria);
875
876 return 0;
877 }
878
879 /* Returns the prio we should use for the given optional counter type,
880 * whereas for bytes type we use the packet type, since they share the same
881 * resources.
882 */
get_opfc_prio(struct mlx5_ib_dev * dev,u32 type)883 static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev,
884 u32 type)
885 {
886 u32 prio_type;
887
888 switch (type) {
889 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
890 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;
891 break;
892 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
893 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;
894 break;
895 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
896 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
897 break;
898 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
899 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
900 break;
901 default:
902 prio_type = type;
903 }
904
905 return &dev->flow_db->opfcs[prio_type];
906 }
907
put_per_qp_prio(struct mlx5_ib_dev * dev,enum mlx5_ib_optional_counter_type type)908 static void put_per_qp_prio(struct mlx5_ib_dev *dev,
909 enum mlx5_ib_optional_counter_type type)
910 {
911 enum mlx5_ib_optional_counter_type per_qp_type;
912 struct mlx5_ib_flow_prio *prio;
913
914 switch (type) {
915 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
916 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
917 break;
918 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
919 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
920 break;
921 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
922 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
923 break;
924 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
925 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
926 break;
927 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
928 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
929 break;
930 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
931 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
932 break;
933 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
934 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
935 break;
936 default:
937 return;
938 }
939
940 prio = get_opfc_prio(dev, per_qp_type);
941 put_flow_table(dev, prio, true);
942 }
943
get_per_qp_prio(struct mlx5_ib_dev * dev,enum mlx5_ib_optional_counter_type type)944 static int get_per_qp_prio(struct mlx5_ib_dev *dev,
945 enum mlx5_ib_optional_counter_type type)
946 {
947 enum mlx5_ib_optional_counter_type per_qp_type;
948 struct mlx5_flow_table_attr ft_attr = {};
949 enum mlx5_flow_namespace_type fn_type;
950 struct mlx5_flow_namespace *ns;
951 struct mlx5_ib_flow_prio *prio;
952 int priority;
953
954 switch (type) {
955 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
956 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
957 priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO;
958 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
959 break;
960 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
961 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
962 priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO;
963 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
964 break;
965 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
966 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
967 priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO;
968 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
969 break;
970 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
971 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
972 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
973 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
974 break;
975 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
976 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
977 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
978 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
979 break;
980 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
981 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
982 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
983 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
984 break;
985 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
986 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
987 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
988 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
989 break;
990 default:
991 return -EINVAL;
992 }
993
994 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
995 if (!ns)
996 return -EOPNOTSUPP;
997
998 prio = get_opfc_prio(dev, per_qp_type);
999 if (prio->flow_table)
1000 return 0;
1001
1002 ft_attr.prio = priority;
1003 ft_attr.max_fte = MLX5_FS_MAX_POOL_SIZE;
1004 ft_attr.autogroup.max_num_groups = 1;
1005 prio = _get_prio(ns, prio, &ft_attr);
1006 if (IS_ERR(prio))
1007 return PTR_ERR(prio);
1008
1009 prio->refcount = 1;
1010
1011 return 0;
1012 }
1013
get_per_qp_opfc(struct xarray * qpn_opfc_xa,u32 qp_num,bool * new)1014 static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa,
1015 u32 qp_num, bool *new)
1016 {
1017 struct mlx5_per_qp_opfc *per_qp_opfc;
1018
1019 *new = false;
1020
1021 per_qp_opfc = xa_load(qpn_opfc_xa, qp_num);
1022 if (per_qp_opfc)
1023 return per_qp_opfc;
1024 per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL);
1025
1026 if (!per_qp_opfc)
1027 return NULL;
1028
1029 *new = true;
1030 return per_qp_opfc;
1031 }
1032
add_op_fc_rules(struct mlx5_ib_dev * dev,struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX],struct xarray * qpn_opfc_xa,struct mlx5_per_qp_opfc * per_qp_opfc,struct mlx5_ib_flow_prio * prio,enum mlx5_ib_optional_counter_type type,u32 qp_num,u32 port_num)1033 static int add_op_fc_rules(struct mlx5_ib_dev *dev,
1034 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1035 struct xarray *qpn_opfc_xa,
1036 struct mlx5_per_qp_opfc *per_qp_opfc,
1037 struct mlx5_ib_flow_prio *prio,
1038 enum mlx5_ib_optional_counter_type type,
1039 u32 qp_num, u32 port_num)
1040 {
1041 struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc;
1042 struct mlx5_flow_act flow_act = {};
1043 struct mlx5_flow_destination dst;
1044 struct mlx5_flow_spec *spec;
1045 int i, err, spec_num;
1046 bool is_tx;
1047
1048 if (opfc->fc)
1049 return -EEXIST;
1050
1051 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type,
1052 &in_use_opfc)) {
1053 opfc->fc = in_use_opfc->fc;
1054 opfc->rule[0] = in_use_opfc->rule[0];
1055 return 0;
1056 }
1057
1058 opfc->fc = fc_arr[type];
1059
1060 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1061 if (!spec) {
1062 err = -ENOMEM;
1063 goto null_fc;
1064 }
1065
1066 switch (type) {
1067 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP:
1068 if (set_ecn_ce_spec(dev, port_num, &spec[0],
1069 MLX5_FS_IPV4_VERSION) ||
1070 set_ecn_ce_spec(dev, port_num, &spec[1],
1071 MLX5_FS_IPV6_VERSION)) {
1072 err = -EOPNOTSUPP;
1073 goto free_spec;
1074 }
1075 spec_num = 2;
1076 is_tx = false;
1077
1078 MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria,
1079 misc_parameters.bth_dst_qp);
1080 MLX5_SET(fte_match_param, spec[1].match_value,
1081 misc_parameters.bth_dst_qp, qp_num);
1082 spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1083 break;
1084 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP:
1085 if (!MLX5_CAP_FLOWTABLE(
1086 dev->mdev,
1087 ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1088 set_cnp_spec(dev, port_num, &spec[0])) {
1089 err = -EOPNOTSUPP;
1090 goto free_spec;
1091 }
1092 spec_num = 1;
1093 is_tx = false;
1094 break;
1095 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP:
1096 if (!MLX5_CAP_FLOWTABLE(
1097 dev->mdev,
1098 ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1099 set_cnp_spec(dev, port_num, &spec[0])) {
1100 err = -EOPNOTSUPP;
1101 goto free_spec;
1102 }
1103 spec_num = 1;
1104 is_tx = true;
1105 break;
1106 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1107 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1108 spec_num = 1;
1109 is_tx = true;
1110 break;
1111 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1112 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1113 spec_num = 1;
1114 is_tx = false;
1115 break;
1116 default:
1117 err = -EINVAL;
1118 goto free_spec;
1119 }
1120
1121 if (is_tx) {
1122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1123 misc_parameters.source_sqn);
1124 MLX5_SET(fte_match_param, spec->match_value,
1125 misc_parameters.source_sqn, qp_num);
1126 } else {
1127 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1128 misc_parameters.bth_dst_qp);
1129 MLX5_SET(fte_match_param, spec->match_value,
1130 misc_parameters.bth_dst_qp, qp_num);
1131 }
1132
1133 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1134
1135 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1136 dst.counter = opfc->fc;
1137
1138 flow_act.action =
1139 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1140
1141 for (i = 0; i < spec_num; i++) {
1142 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1143 &flow_act, &dst, 1);
1144 if (IS_ERR(opfc->rule[i])) {
1145 err = PTR_ERR(opfc->rule[i]);
1146 goto del_rules;
1147 }
1148 }
1149 prio->refcount += spec_num;
1150
1151 err = xa_err(xa_store(qpn_opfc_xa, qp_num, per_qp_opfc, GFP_KERNEL));
1152 if (err)
1153 goto del_rules;
1154
1155 kfree(spec);
1156
1157 return 0;
1158
1159 del_rules:
1160 while (i--)
1161 mlx5_del_flow_rules(opfc->rule[i]);
1162 put_flow_table(dev, prio, false);
1163 free_spec:
1164 kfree(spec);
1165 null_fc:
1166 opfc->fc = NULL;
1167 return err;
1168 }
1169
1170 static bool
is_fc_shared_and_in_use(struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX],u32 type,struct mlx5_fc ** fc)1171 is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type,
1172 struct mlx5_fc **fc)
1173 {
1174 u32 shared_fc_type;
1175
1176 switch (type) {
1177 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1178 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
1179 break;
1180 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1181 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
1182 break;
1183 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1184 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
1185 break;
1186 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1187 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
1188 break;
1189 default:
1190 return false;
1191 }
1192
1193 *fc = fc_arr[shared_fc_type];
1194 if (!(*fc))
1195 return false;
1196
1197 return true;
1198 }
1199
mlx5r_fs_destroy_fcs(struct mlx5_ib_dev * dev,struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX])1200 void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,
1201 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX])
1202 {
1203 struct mlx5_fc *in_use_fc;
1204 int i;
1205
1206 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1207 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1208 if (!fc_arr[i])
1209 continue;
1210
1211 if (is_fc_shared_and_in_use(fc_arr, i, &in_use_fc)) {
1212 fc_arr[i] = NULL;
1213 continue;
1214 }
1215
1216 mlx5_fc_destroy(dev->mdev, fc_arr[i]);
1217 fc_arr[i] = NULL;
1218 }
1219 }
1220
mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_ib_op_fc * opfc,enum mlx5_ib_optional_counter_type type)1221 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,
1222 struct mlx5_ib_op_fc *opfc,
1223 enum mlx5_ib_optional_counter_type type)
1224 {
1225 struct mlx5_flow_table_attr ft_attr = {};
1226 enum mlx5_flow_namespace_type fn_type;
1227 int priority, i, err, spec_num;
1228 struct mlx5_flow_act flow_act = {};
1229 struct mlx5_flow_destination dst;
1230 struct mlx5_flow_namespace *ns;
1231 struct mlx5_ib_flow_prio *prio;
1232 struct mlx5_flow_spec *spec;
1233
1234 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1235 if (!spec)
1236 return -ENOMEM;
1237
1238 switch (type) {
1239 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
1240 if (set_ecn_ce_spec(dev, port_num, &spec[0],
1241 MLX5_FS_IPV4_VERSION) ||
1242 set_ecn_ce_spec(dev, port_num, &spec[1],
1243 MLX5_FS_IPV6_VERSION)) {
1244 err = -EOPNOTSUPP;
1245 goto free;
1246 }
1247 spec_num = 2;
1248 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1249 priority = RDMA_RX_ECN_OPCOUNTER_PRIO;
1250 break;
1251
1252 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
1253 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1254 ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1255 set_cnp_spec(dev, port_num, &spec[0])) {
1256 err = -EOPNOTSUPP;
1257 goto free;
1258 }
1259 spec_num = 1;
1260 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1261 priority = RDMA_RX_CNP_OPCOUNTER_PRIO;
1262 break;
1263
1264 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
1265 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1266 ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1267 set_cnp_spec(dev, port_num, &spec[0])) {
1268 err = -EOPNOTSUPP;
1269 goto free;
1270 }
1271 spec_num = 1;
1272 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1273 priority = RDMA_TX_CNP_OPCOUNTER_PRIO;
1274 break;
1275
1276 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
1277 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
1278 spec_num = 1;
1279 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1280 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO;
1281 break;
1282
1283 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
1284 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
1285 spec_num = 1;
1286 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1287 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO;
1288 break;
1289
1290 default:
1291 err = -EOPNOTSUPP;
1292 goto free;
1293 }
1294
1295 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
1296 if (!ns) {
1297 err = -EOPNOTSUPP;
1298 goto free;
1299 }
1300
1301 prio = get_opfc_prio(dev, type);
1302 if (!prio->flow_table) {
1303 err = get_per_qp_prio(dev, type);
1304 if (err)
1305 goto free;
1306
1307 ft_attr.prio = priority;
1308 ft_attr.max_fte = dev->num_ports * MAX_OPFC_RULES;
1309 ft_attr.autogroup.max_num_groups = 1;
1310 prio = _get_prio(ns, prio, &ft_attr);
1311 if (IS_ERR(prio)) {
1312 err = PTR_ERR(prio);
1313 goto put_prio;
1314 }
1315 }
1316
1317 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1318 dst.counter = opfc->fc;
1319
1320 flow_act.action =
1321 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1322
1323 for (i = 0; i < spec_num; i++) {
1324 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1325 &flow_act, &dst, 1);
1326 if (IS_ERR(opfc->rule[i])) {
1327 err = PTR_ERR(opfc->rule[i]);
1328 goto del_rules;
1329 }
1330 }
1331 prio->refcount += spec_num;
1332 kfree(spec);
1333
1334 return 0;
1335
1336 del_rules:
1337 for (i -= 1; i >= 0; i--)
1338 mlx5_del_flow_rules(opfc->rule[i]);
1339 put_flow_table(dev, prio, false);
1340 put_prio:
1341 put_per_qp_prio(dev, type);
1342 free:
1343 kfree(spec);
1344 return err;
1345 }
1346
mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev * dev,struct mlx5_ib_op_fc * opfc,enum mlx5_ib_optional_counter_type type)1347 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
1348 struct mlx5_ib_op_fc *opfc,
1349 enum mlx5_ib_optional_counter_type type)
1350 {
1351 struct mlx5_ib_flow_prio *prio;
1352 int i;
1353
1354 prio = get_opfc_prio(dev, type);
1355
1356 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) {
1357 mlx5_del_flow_rules(opfc->rule[i]);
1358 put_flow_table(dev, prio, true);
1359 }
1360
1361 put_per_qp_prio(dev, type);
1362 }
1363
mlx5r_fs_unbind_op_fc(struct ib_qp * qp,struct xarray * qpn_opfc_xa)1364 void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa)
1365 {
1366 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1367 struct mlx5_per_qp_opfc *per_qp_opfc;
1368 struct mlx5_ib_op_fc *in_use_opfc;
1369 struct mlx5_ib_flow_prio *prio;
1370 int i, j;
1371
1372 per_qp_opfc = xa_load(qpn_opfc_xa, qp->qp_num);
1373 if (!per_qp_opfc)
1374 return;
1375
1376 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1377 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1378 if (!per_qp_opfc->opfcs[i].fc)
1379 continue;
1380
1381 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i,
1382 &in_use_opfc)) {
1383 per_qp_opfc->opfcs[i].fc = NULL;
1384 continue;
1385 }
1386
1387 for (j = 0; j < MAX_OPFC_RULES; j++) {
1388 if (!per_qp_opfc->opfcs[i].rule[j])
1389 continue;
1390 mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]);
1391 prio = get_opfc_prio(dev, i);
1392 put_flow_table(dev, prio, true);
1393 }
1394 per_qp_opfc->opfcs[i].fc = NULL;
1395 }
1396
1397 kfree(per_qp_opfc);
1398 xa_erase(qpn_opfc_xa, qp->qp_num);
1399 }
1400
mlx5r_fs_bind_op_fc(struct ib_qp * qp,struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX],struct xarray * qpn_opfc_xa,u32 port)1401 int mlx5r_fs_bind_op_fc(struct ib_qp *qp,
1402 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1403 struct xarray *qpn_opfc_xa, u32 port)
1404 {
1405 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1406 struct mlx5_per_qp_opfc *per_qp_opfc;
1407 struct mlx5_ib_flow_prio *prio;
1408 struct mlx5_ib_counters *cnts;
1409 struct mlx5_ib_op_fc *opfc;
1410 struct mlx5_fc *in_use_fc;
1411 int i, err, per_qp_type;
1412 bool new;
1413
1414 cnts = &dev->port[port - 1].cnts;
1415
1416 for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) {
1417 opfc = &cnts->opfcs[i];
1418 if (!opfc->fc)
1419 continue;
1420
1421 per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1422 prio = get_opfc_prio(dev, per_qp_type);
1423 WARN_ON(!prio->flow_table);
1424
1425 if (is_fc_shared_and_in_use(fc_arr, per_qp_type, &in_use_fc))
1426 fc_arr[per_qp_type] = in_use_fc;
1427
1428 if (!fc_arr[per_qp_type]) {
1429 fc_arr[per_qp_type] = mlx5_fc_create(dev->mdev, false);
1430 if (IS_ERR(fc_arr[per_qp_type]))
1431 return PTR_ERR(fc_arr[per_qp_type]);
1432 }
1433
1434 per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp->qp_num, &new);
1435 if (!per_qp_opfc) {
1436 err = -ENOMEM;
1437 goto free_fc;
1438 }
1439 err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc,
1440 prio, per_qp_type, qp->qp_num, port);
1441 if (err)
1442 goto del_rules;
1443 }
1444
1445 return 0;
1446
1447 del_rules:
1448 mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa);
1449 if (new)
1450 kfree(per_qp_opfc);
1451 free_fc:
1452 if (xa_empty(qpn_opfc_xa))
1453 mlx5r_fs_destroy_fcs(dev, fc_arr);
1454 return err;
1455 }
1456
set_underlay_qp(struct mlx5_ib_dev * dev,struct mlx5_flow_spec * spec,u32 underlay_qpn)1457 static void set_underlay_qp(struct mlx5_ib_dev *dev,
1458 struct mlx5_flow_spec *spec,
1459 u32 underlay_qpn)
1460 {
1461 void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
1462 spec->match_criteria,
1463 misc_parameters);
1464 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1465 misc_parameters);
1466
1467 if (underlay_qpn &&
1468 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1469 ft_field_support.bth_dst_qp)) {
1470 MLX5_SET(fte_match_set_misc,
1471 misc_params_v, bth_dst_qp, underlay_qpn);
1472 MLX5_SET(fte_match_set_misc,
1473 misc_params_c, bth_dst_qp, 0xffffff);
1474 }
1475 }
1476
mlx5_ib_set_rule_source_port(struct mlx5_ib_dev * dev,struct mlx5_flow_spec * spec,struct mlx5_eswitch_rep * rep)1477 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
1478 struct mlx5_flow_spec *spec,
1479 struct mlx5_eswitch_rep *rep)
1480 {
1481 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
1482 void *misc;
1483
1484 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1485 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1486 misc_parameters_2);
1487
1488 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1489 mlx5_eswitch_get_vport_metadata_for_match(rep->esw,
1490 rep->vport));
1491 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1492 misc_parameters_2);
1493
1494 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1495 mlx5_eswitch_get_vport_metadata_mask());
1496 } else {
1497 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1498 misc_parameters);
1499
1500 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
1501
1502 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1503 misc_parameters);
1504
1505 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1506 }
1507 }
1508
_create_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,const struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst,u32 underlay_qpn,struct mlx5_ib_create_flow * ucmd)1509 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
1510 struct mlx5_ib_flow_prio *ft_prio,
1511 const struct ib_flow_attr *flow_attr,
1512 struct mlx5_flow_destination *dst,
1513 u32 underlay_qpn,
1514 struct mlx5_ib_create_flow *ucmd)
1515 {
1516 struct mlx5_flow_table *ft = ft_prio->flow_table;
1517 struct mlx5_ib_flow_handler *handler;
1518 struct mlx5_flow_act flow_act = {};
1519 struct mlx5_flow_spec *spec;
1520 struct mlx5_flow_destination dest_arr[2] = {};
1521 struct mlx5_flow_destination *rule_dst = dest_arr;
1522 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
1523 unsigned int spec_index;
1524 u32 prev_type = 0;
1525 int err = 0;
1526 int dest_num = 0;
1527 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1528
1529 if (!is_valid_attr(dev->mdev, flow_attr))
1530 return ERR_PTR(-EINVAL);
1531
1532 if (dev->is_rep && is_egress)
1533 return ERR_PTR(-EINVAL);
1534
1535 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1536 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
1537 if (!handler || !spec) {
1538 err = -ENOMEM;
1539 goto free;
1540 }
1541
1542 INIT_LIST_HEAD(&handler->list);
1543
1544 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
1545 err = parse_flow_attr(dev->mdev, spec,
1546 ib_flow, flow_attr, &flow_act,
1547 prev_type);
1548 if (err < 0)
1549 goto free;
1550
1551 prev_type = ((union ib_flow_spec *)ib_flow)->type;
1552 ib_flow += ((union ib_flow_spec *)ib_flow)->size;
1553 }
1554
1555 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
1556 memcpy(&dest_arr[0], dst, sizeof(*dst));
1557 dest_num++;
1558 }
1559
1560 if (!flow_is_multicast_only(flow_attr))
1561 set_underlay_qp(dev, spec, underlay_qpn);
1562
1563 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) {
1564 struct mlx5_eswitch_rep *rep;
1565
1566 rep = dev->port[flow_attr->port - 1].rep;
1567 if (!rep) {
1568 err = -EINVAL;
1569 goto free;
1570 }
1571
1572 mlx5_ib_set_rule_source_port(dev, spec, rep);
1573 }
1574
1575 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
1576
1577 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1578 struct mlx5_ib_mcounters *mcounters;
1579
1580 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);
1581 if (err)
1582 goto free;
1583
1584 mcounters = to_mcounters(flow_act.counters);
1585 handler->ibcounters = flow_act.counters;
1586 dest_arr[dest_num].type =
1587 MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1588 dest_arr[dest_num].counter =
1589 mcounters->hw_cntrs_hndl;
1590 dest_num++;
1591 }
1592
1593 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
1594 if (!dest_num)
1595 rule_dst = NULL;
1596 } else {
1597 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
1598 flow_act.action |=
1599 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1600 if (is_egress)
1601 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1602 else if (dest_num)
1603 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1604 }
1605
1606 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
1607 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1608 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1609 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
1610 spec->flow_context.flow_tag, flow_attr->type);
1611 err = -EINVAL;
1612 goto free;
1613 }
1614 handler->rule = mlx5_add_flow_rules(ft, spec,
1615 &flow_act,
1616 rule_dst, dest_num);
1617
1618 if (IS_ERR(handler->rule)) {
1619 err = PTR_ERR(handler->rule);
1620 goto free;
1621 }
1622
1623 ft_prio->refcount++;
1624 handler->prio = ft_prio;
1625 handler->dev = dev;
1626
1627 ft_prio->flow_table = ft;
1628 free:
1629 if (err && handler) {
1630 mlx5_ib_counters_clear_description(handler->ibcounters);
1631 kfree(handler);
1632 }
1633 kvfree(spec);
1634 return err ? ERR_PTR(err) : handler;
1635 }
1636
create_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,const struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)1637 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
1638 struct mlx5_ib_flow_prio *ft_prio,
1639 const struct ib_flow_attr *flow_attr,
1640 struct mlx5_flow_destination *dst)
1641 {
1642 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
1643 }
1644
create_leftovers_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)1645 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
1646 struct mlx5_ib_flow_prio *ft_prio,
1647 struct ib_flow_attr *flow_attr,
1648 struct mlx5_flow_destination *dst)
1649 {
1650 struct mlx5_ib_flow_handler *handler_ucast = NULL;
1651 struct mlx5_ib_flow_handler *handler = NULL;
1652
1653 static struct {
1654 struct ib_flow_spec_eth eth_flow;
1655 struct ib_flow_attr flow_attr;
1656 } leftovers_wc = { .flow_attr = { .num_of_specs = 1,
1657 .size = sizeof(leftovers_wc) },
1658 .eth_flow = {
1659 .type = IB_FLOW_SPEC_ETH,
1660 .size = sizeof(struct ib_flow_spec_eth),
1661 .mask = { .dst_mac = { 0x1 } },
1662 .val = { .dst_mac = { 0x1 } } } };
1663
1664 static struct {
1665 struct ib_flow_spec_eth eth_flow;
1666 struct ib_flow_attr flow_attr;
1667 } leftovers_uc = { .flow_attr = { .num_of_specs = 1,
1668 .size = sizeof(leftovers_uc) },
1669 .eth_flow = {
1670 .type = IB_FLOW_SPEC_ETH,
1671 .size = sizeof(struct ib_flow_spec_eth),
1672 .mask = { .dst_mac = { 0x1 } },
1673 .val = { .dst_mac = {} } } };
1674
1675 handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst);
1676 if (!IS_ERR(handler) &&
1677 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
1678 handler_ucast = create_flow_rule(dev, ft_prio,
1679 &leftovers_uc.flow_attr, dst);
1680 if (IS_ERR(handler_ucast)) {
1681 mlx5_del_flow_rules(handler->rule);
1682 ft_prio->refcount--;
1683 kfree(handler);
1684 handler = handler_ucast;
1685 } else {
1686 list_add(&handler_ucast->list, &handler->list);
1687 }
1688 }
1689
1690 return handler;
1691 }
1692
create_sniffer_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_rx,struct mlx5_ib_flow_prio * ft_tx,struct mlx5_flow_destination * dst)1693 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
1694 struct mlx5_ib_flow_prio *ft_rx,
1695 struct mlx5_ib_flow_prio *ft_tx,
1696 struct mlx5_flow_destination *dst)
1697 {
1698 struct mlx5_ib_flow_handler *handler_rx;
1699 struct mlx5_ib_flow_handler *handler_tx;
1700 int err;
1701 static const struct ib_flow_attr flow_attr = {
1702 .num_of_specs = 0,
1703 .type = IB_FLOW_ATTR_SNIFFER,
1704 .size = sizeof(flow_attr)
1705 };
1706
1707 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
1708 if (IS_ERR(handler_rx)) {
1709 err = PTR_ERR(handler_rx);
1710 goto err;
1711 }
1712
1713 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
1714 if (IS_ERR(handler_tx)) {
1715 err = PTR_ERR(handler_tx);
1716 goto err_tx;
1717 }
1718
1719 list_add(&handler_tx->list, &handler_rx->list);
1720
1721 return handler_rx;
1722
1723 err_tx:
1724 mlx5_del_flow_rules(handler_rx->rule);
1725 ft_rx->refcount--;
1726 kfree(handler_rx);
1727 err:
1728 return ERR_PTR(err);
1729 }
1730
mlx5_ib_create_flow(struct ib_qp * qp,struct ib_flow_attr * flow_attr,struct ib_udata * udata)1731 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
1732 struct ib_flow_attr *flow_attr,
1733 struct ib_udata *udata)
1734 {
1735 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1736 struct mlx5_ib_qp *mqp = to_mqp(qp);
1737 struct mlx5_ib_flow_handler *handler = NULL;
1738 struct mlx5_flow_destination *dst = NULL;
1739 struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
1740 struct mlx5_ib_flow_prio *ft_prio;
1741 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1742 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
1743 size_t min_ucmd_sz, required_ucmd_sz;
1744 int err;
1745 int underlay_qpn;
1746
1747 if (udata && udata->inlen) {
1748 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);
1749 if (udata->inlen < min_ucmd_sz)
1750 return ERR_PTR(-EOPNOTSUPP);
1751
1752 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
1753 if (err)
1754 return ERR_PTR(err);
1755
1756 /* currently supports only one counters data */
1757 if (ucmd_hdr.ncounters_data > 1)
1758 return ERR_PTR(-EINVAL);
1759
1760 required_ucmd_sz = min_ucmd_sz +
1761 sizeof(struct mlx5_ib_flow_counters_data) *
1762 ucmd_hdr.ncounters_data;
1763 if (udata->inlen > required_ucmd_sz &&
1764 !ib_is_udata_cleared(udata, required_ucmd_sz,
1765 udata->inlen - required_ucmd_sz))
1766 return ERR_PTR(-EOPNOTSUPP);
1767
1768 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
1769 if (!ucmd)
1770 return ERR_PTR(-ENOMEM);
1771
1772 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
1773 if (err)
1774 goto free_ucmd;
1775 }
1776
1777 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
1778 err = -ENOMEM;
1779 goto free_ucmd;
1780 }
1781
1782 if (flow_attr->flags &
1783 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) {
1784 err = -EINVAL;
1785 goto free_ucmd;
1786 }
1787
1788 if (is_egress &&
1789 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1790 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1791 err = -EINVAL;
1792 goto free_ucmd;
1793 }
1794
1795 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1796 if (!dst) {
1797 err = -ENOMEM;
1798 goto free_ucmd;
1799 }
1800
1801 mutex_lock(&dev->flow_db->lock);
1802
1803 ft_prio = get_flow_table(dev, flow_attr,
1804 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
1805 if (IS_ERR(ft_prio)) {
1806 err = PTR_ERR(ft_prio);
1807 goto unlock;
1808 }
1809 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
1810 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
1811 if (IS_ERR(ft_prio_tx)) {
1812 err = PTR_ERR(ft_prio_tx);
1813 ft_prio_tx = NULL;
1814 goto destroy_ft;
1815 }
1816 }
1817
1818 if (is_egress) {
1819 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1820 } else {
1821 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1822 if (mqp->is_rss)
1823 dst->tir_num = mqp->rss_qp.tirn;
1824 else
1825 dst->tir_num = mqp->raw_packet_qp.rq.tirn;
1826 }
1827
1828 switch (flow_attr->type) {
1829 case IB_FLOW_ATTR_NORMAL:
1830 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
1831 mqp->underlay_qpn :
1832 0;
1833 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
1834 underlay_qpn, ucmd);
1835 break;
1836 case IB_FLOW_ATTR_ALL_DEFAULT:
1837 case IB_FLOW_ATTR_MC_DEFAULT:
1838 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);
1839 break;
1840 case IB_FLOW_ATTR_SNIFFER:
1841 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
1842 break;
1843 default:
1844 err = -EINVAL;
1845 goto destroy_ft;
1846 }
1847
1848 if (IS_ERR(handler)) {
1849 err = PTR_ERR(handler);
1850 handler = NULL;
1851 goto destroy_ft;
1852 }
1853
1854 mutex_unlock(&dev->flow_db->lock);
1855 kfree(dst);
1856 kfree(ucmd);
1857
1858 return &handler->ibflow;
1859
1860 destroy_ft:
1861 put_flow_table(dev, ft_prio, false);
1862 if (ft_prio_tx)
1863 put_flow_table(dev, ft_prio_tx, false);
1864 unlock:
1865 mutex_unlock(&dev->flow_db->lock);
1866 kfree(dst);
1867 free_ucmd:
1868 kfree(ucmd);
1869 return ERR_PTR(err);
1870 }
1871
mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev * dev,enum mlx5_flow_namespace_type type,u32 * flags,u16 * vport_idx,u16 * vport,struct mlx5_core_dev ** ft_mdev,u32 ib_port,u16 * esw_owner_vhca_id)1872 static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev,
1873 enum mlx5_flow_namespace_type type,
1874 u32 *flags, u16 *vport_idx,
1875 u16 *vport,
1876 struct mlx5_core_dev **ft_mdev,
1877 u32 ib_port, u16 *esw_owner_vhca_id)
1878 {
1879 struct mlx5_core_dev *esw_mdev;
1880
1881 if (!is_mdev_switchdev_mode(dev->mdev))
1882 return 0;
1883
1884 if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager))
1885 return -EOPNOTSUPP;
1886
1887 if (!dev->port[ib_port - 1].rep)
1888 return -EINVAL;
1889
1890 esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw);
1891 if (esw_mdev != dev->mdev) {
1892 if (!MLX5_CAP_ADV_RDMA(dev->mdev,
1893 rdma_transport_manager_other_eswitch))
1894 return -EOPNOTSUPP;
1895 *flags |= MLX5_FLOW_TABLE_OTHER_ESWITCH;
1896 *esw_owner_vhca_id = MLX5_CAP_GEN(esw_mdev, vhca_id);
1897 }
1898
1899 *flags |= MLX5_FLOW_TABLE_OTHER_VPORT;
1900 *ft_mdev = esw_mdev;
1901 *vport = dev->port[ib_port - 1].rep->vport;
1902 *vport_idx = dev->port[ib_port - 1].rep->vport_index;
1903
1904 return 0;
1905 }
1906
1907 static struct mlx5_ib_flow_prio *
_get_flow_table(struct mlx5_ib_dev * dev,u16 user_priority,enum mlx5_flow_namespace_type ns_type,bool mcast,u32 ib_port)1908 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
1909 enum mlx5_flow_namespace_type ns_type,
1910 bool mcast, u32 ib_port)
1911 {
1912 struct mlx5_core_dev *ft_mdev = dev->mdev;
1913 struct mlx5_flow_table_attr ft_attr = {};
1914 struct mlx5_flow_namespace *ns = NULL;
1915 struct mlx5_ib_flow_prio *prio = NULL;
1916 u16 esw_owner_vhca_id = 0;
1917 int max_table_size = 0;
1918 u16 vport_idx = 0;
1919 bool esw_encap;
1920 u32 flags = 0;
1921 u16 vport = 0;
1922 int priority;
1923 int ret;
1924
1925 if (mcast)
1926 priority = MLX5_IB_FLOW_MCAST_PRIO;
1927 else
1928 priority = ib_prio_to_core_prio(user_priority, false);
1929
1930 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
1931 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
1932 switch (ns_type) {
1933 case MLX5_FLOW_NAMESPACE_BYPASS:
1934 max_table_size = BIT(
1935 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size));
1936 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
1937 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1938 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1939 reformat_l3_tunnel_to_l2) &&
1940 !esw_encap)
1941 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1942 break;
1943 case MLX5_FLOW_NAMESPACE_EGRESS:
1944 max_table_size = BIT(
1945 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
1946 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) &&
1947 !esw_encap)
1948 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1949 break;
1950 case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
1951 max_table_size = BIT(
1952 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
1953 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
1954 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1955 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev,
1956 reformat_l3_tunnel_to_l2) &&
1957 esw_encap)
1958 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1959 priority = user_priority;
1960 break;
1961 case MLX5_FLOW_NAMESPACE_RDMA_RX:
1962 max_table_size = BIT(
1963 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size));
1964 priority = user_priority;
1965 break;
1966 case MLX5_FLOW_NAMESPACE_RDMA_TX:
1967 max_table_size = BIT(
1968 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size));
1969 priority = user_priority;
1970 break;
1971 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
1972 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
1973 if (ib_port == 0 ||
1974 user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO)
1975 return ERR_PTR(-EINVAL);
1976 ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags,
1977 &vport_idx, &vport,
1978 &ft_mdev, ib_port,
1979 &esw_owner_vhca_id);
1980 if (ret)
1981 return ERR_PTR(ret);
1982
1983 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX)
1984 max_table_size =
1985 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(
1986 ft_mdev, log_max_ft_size));
1987 else
1988 max_table_size =
1989 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(
1990 ft_mdev, log_max_ft_size));
1991 priority = user_priority;
1992 break;
1993 default:
1994 break;
1995 }
1996
1997 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
1998
1999 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX ||
2000 ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)
2001 ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx);
2002 else
2003 ns = mlx5_get_flow_namespace(ft_mdev, ns_type);
2004
2005 if (!ns)
2006 return ERR_PTR(-EOPNOTSUPP);
2007
2008 switch (ns_type) {
2009 case MLX5_FLOW_NAMESPACE_BYPASS:
2010 prio = &dev->flow_db->prios[priority];
2011 break;
2012 case MLX5_FLOW_NAMESPACE_EGRESS:
2013 prio = &dev->flow_db->egress_prios[priority];
2014 break;
2015 case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
2016 prio = &dev->flow_db->fdb[priority];
2017 break;
2018 case MLX5_FLOW_NAMESPACE_RDMA_RX:
2019 prio = &dev->flow_db->rdma_rx[priority];
2020 break;
2021 case MLX5_FLOW_NAMESPACE_RDMA_TX:
2022 prio = &dev->flow_db->rdma_tx[priority];
2023 break;
2024 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
2025 prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1];
2026 break;
2027 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
2028 prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1];
2029 break;
2030 default: return ERR_PTR(-EINVAL);
2031 }
2032
2033 if (!prio)
2034 return ERR_PTR(-EINVAL);
2035
2036 if (prio->flow_table)
2037 return prio;
2038
2039 ft_attr.prio = priority;
2040 ft_attr.max_fte = max_table_size;
2041 ft_attr.flags = flags;
2042 ft_attr.vport = vport;
2043 ft_attr.esw_owner_vhca_id = esw_owner_vhca_id;
2044 ft_attr.autogroup.max_num_groups = MLX5_FS_MAX_TYPES;
2045 return _get_prio(ns, prio, &ft_attr);
2046 }
2047
2048 static struct mlx5_ib_flow_handler *
_create_raw_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct mlx5_flow_destination * dst,struct mlx5_ib_flow_matcher * fs_matcher,struct mlx5_flow_context * flow_context,struct mlx5_flow_act * flow_act,void * cmd_in,int inlen,int dst_num)2049 _create_raw_flow_rule(struct mlx5_ib_dev *dev,
2050 struct mlx5_ib_flow_prio *ft_prio,
2051 struct mlx5_flow_destination *dst,
2052 struct mlx5_ib_flow_matcher *fs_matcher,
2053 struct mlx5_flow_context *flow_context,
2054 struct mlx5_flow_act *flow_act,
2055 void *cmd_in, int inlen,
2056 int dst_num)
2057 {
2058 struct mlx5_ib_flow_handler *handler;
2059 struct mlx5_flow_spec *spec;
2060 struct mlx5_flow_table *ft = ft_prio->flow_table;
2061 int err = 0;
2062
2063 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2064 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
2065 if (!handler || !spec) {
2066 err = -ENOMEM;
2067 goto free;
2068 }
2069
2070 INIT_LIST_HEAD(&handler->list);
2071
2072 memcpy(spec->match_value, cmd_in, inlen);
2073 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
2074 fs_matcher->mask_len);
2075 spec->match_criteria_enable = fs_matcher->match_criteria_enable;
2076 spec->flow_context = *flow_context;
2077
2078 handler->rule = mlx5_add_flow_rules(ft, spec,
2079 flow_act, dst, dst_num);
2080
2081 if (IS_ERR(handler->rule)) {
2082 err = PTR_ERR(handler->rule);
2083 goto free;
2084 }
2085
2086 ft_prio->refcount++;
2087 handler->prio = ft_prio;
2088 handler->dev = dev;
2089 ft_prio->flow_table = ft;
2090
2091 free:
2092 if (err)
2093 kfree(handler);
2094 kvfree(spec);
2095 return err ? ERR_PTR(err) : handler;
2096 }
2097
raw_fs_is_multicast(struct mlx5_ib_flow_matcher * fs_matcher,void * match_v)2098 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
2099 void *match_v)
2100 {
2101 void *match_c;
2102 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
2103 void *dmac, *dmac_mask;
2104 void *ipv4, *ipv4_mask;
2105
2106 if (!(fs_matcher->match_criteria_enable &
2107 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
2108 return false;
2109
2110 match_c = fs_matcher->matcher_mask.match_params;
2111 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
2112 outer_headers);
2113 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
2114 outer_headers);
2115
2116 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2117 dmac_47_16);
2118 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2119 dmac_47_16);
2120
2121 if (is_multicast_ether_addr(dmac) &&
2122 is_multicast_ether_addr(dmac_mask))
2123 return true;
2124
2125 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2126 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2127
2128 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2129 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2130
2131 if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
2132 ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
2133 return true;
2134
2135 return false;
2136 }
2137
raw_fs_rule_add(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_matcher * fs_matcher,struct mlx5_flow_context * flow_context,struct mlx5_flow_act * flow_act,struct mlx5_fc * counter,void * cmd_in,int inlen,int dest_id,int dest_type)2138 static struct mlx5_ib_flow_handler *raw_fs_rule_add(
2139 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
2140 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
2141 struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type)
2142 {
2143 struct mlx5_flow_destination *dst;
2144 struct mlx5_ib_flow_prio *ft_prio;
2145 struct mlx5_ib_flow_handler *handler;
2146 int dst_num = 0;
2147 bool mcast;
2148 int err;
2149
2150 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
2151 return ERR_PTR(-EOPNOTSUPP);
2152
2153 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
2154 return ERR_PTR(-ENOMEM);
2155
2156 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
2157 if (!dst)
2158 return ERR_PTR(-ENOMEM);
2159
2160 mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
2161 mutex_lock(&dev->flow_db->lock);
2162
2163 ft_prio = _get_flow_table(dev, fs_matcher->priority,
2164 fs_matcher->ns_type, mcast,
2165 fs_matcher->ib_port);
2166 if (IS_ERR(ft_prio)) {
2167 err = PTR_ERR(ft_prio);
2168 goto unlock;
2169 }
2170
2171 switch (dest_type) {
2172 case MLX5_FLOW_DESTINATION_TYPE_TIR:
2173 dst[dst_num].type = dest_type;
2174 dst[dst_num++].tir_num = dest_id;
2175 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2176 break;
2177 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
2178 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
2179 dst[dst_num++].ft_num = dest_id;
2180 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2181 break;
2182 case MLX5_FLOW_DESTINATION_TYPE_PORT:
2183 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2184 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
2185 break;
2186 default:
2187 break;
2188 }
2189
2190 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
2191 if (WARN_ON(!counter)) {
2192 err = -EINVAL;
2193 goto unlock;
2194 }
2195 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
2196 dst[dst_num].counter = counter;
2197 dst_num++;
2198 }
2199
2200 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL,
2201 fs_matcher, flow_context, flow_act,
2202 cmd_in, inlen, dst_num);
2203
2204 if (IS_ERR(handler)) {
2205 err = PTR_ERR(handler);
2206 goto destroy_ft;
2207 }
2208
2209 mutex_unlock(&dev->flow_db->lock);
2210 atomic_inc(&fs_matcher->usecnt);
2211 handler->flow_matcher = fs_matcher;
2212
2213 kfree(dst);
2214
2215 return handler;
2216
2217 destroy_ft:
2218 put_flow_table(dev, ft_prio, false);
2219 unlock:
2220 mutex_unlock(&dev->flow_db->lock);
2221 kfree(dst);
2222
2223 return ERR_PTR(err);
2224 }
2225
destroy_flow_action_raw(struct mlx5_ib_flow_action * maction)2226 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
2227 {
2228 switch (maction->flow_action_raw.sub_type) {
2229 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
2230 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
2231 maction->flow_action_raw.modify_hdr);
2232 break;
2233 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
2234 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
2235 maction->flow_action_raw.pkt_reformat);
2236 break;
2237 case MLX5_IB_FLOW_ACTION_DECAP:
2238 break;
2239 default:
2240 break;
2241 }
2242 }
2243
mlx5_ib_destroy_flow_action(struct ib_flow_action * action)2244 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
2245 {
2246 struct mlx5_ib_flow_action *maction = to_mflow_act(action);
2247
2248 switch (action->type) {
2249 case IB_FLOW_ACTION_UNSPECIFIED:
2250 destroy_flow_action_raw(maction);
2251 break;
2252 default:
2253 WARN_ON(true);
2254 break;
2255 }
2256
2257 kfree(maction);
2258 return 0;
2259 }
2260
2261 static int
mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,enum mlx5_flow_namespace_type * namespace)2262 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
2263 enum mlx5_flow_namespace_type *namespace)
2264 {
2265 switch (table_type) {
2266 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
2267 *namespace = MLX5_FLOW_NAMESPACE_BYPASS;
2268 break;
2269 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
2270 *namespace = MLX5_FLOW_NAMESPACE_EGRESS;
2271 break;
2272 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
2273 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS;
2274 break;
2275 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
2276 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
2277 break;
2278 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
2279 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
2280 break;
2281 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX:
2282 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX;
2283 break;
2284 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX:
2285 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX;
2286 break;
2287 default:
2288 return -EINVAL;
2289 }
2290
2291 return 0;
2292 }
2293
2294 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
2295 [MLX5_IB_FLOW_TYPE_NORMAL] = {
2296 .type = UVERBS_ATTR_TYPE_PTR_IN,
2297 .u.ptr = {
2298 .len = sizeof(u16), /* data is priority */
2299 .min_len = sizeof(u16),
2300 }
2301 },
2302 [MLX5_IB_FLOW_TYPE_SNIFFER] = {
2303 .type = UVERBS_ATTR_TYPE_PTR_IN,
2304 UVERBS_ATTR_NO_DATA(),
2305 },
2306 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
2307 .type = UVERBS_ATTR_TYPE_PTR_IN,
2308 UVERBS_ATTR_NO_DATA(),
2309 },
2310 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
2311 .type = UVERBS_ATTR_TYPE_PTR_IN,
2312 UVERBS_ATTR_NO_DATA(),
2313 },
2314 };
2315
is_flow_dest(void * obj,int * dest_id,int * dest_type)2316 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
2317 {
2318 struct devx_obj *devx_obj = obj;
2319 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2320
2321 switch (opcode) {
2322 case MLX5_CMD_OP_DESTROY_TIR:
2323 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2324 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
2325 obj_id);
2326 return true;
2327
2328 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
2329 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2330 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
2331 table_id);
2332 return true;
2333 default:
2334 return false;
2335 }
2336 }
2337
get_dests(struct uverbs_attr_bundle * attrs,struct mlx5_ib_flow_matcher * fs_matcher,int * dest_id,int * dest_type,struct ib_qp ** qp,u32 * flags)2338 static int get_dests(struct uverbs_attr_bundle *attrs,
2339 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
2340 int *dest_type, struct ib_qp **qp, u32 *flags)
2341 {
2342 bool dest_devx, dest_qp;
2343 void *devx_obj;
2344 int err;
2345
2346 dest_devx = uverbs_attr_is_valid(attrs,
2347 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2348 dest_qp = uverbs_attr_is_valid(attrs,
2349 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2350
2351 *flags = 0;
2352 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
2353 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
2354 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
2355 if (err)
2356 return err;
2357
2358 /* Both flags are not allowed */
2359 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
2360 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2361 return -EINVAL;
2362
2363 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
2364 if (dest_devx && (dest_qp || *flags))
2365 return -EINVAL;
2366 else if (dest_qp && *flags)
2367 return -EINVAL;
2368 }
2369
2370 /* Allow only DEVX object, drop as dest for FDB */
2371 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2372 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
2373 return -EINVAL;
2374
2375 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
2376 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2377 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2378 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
2379 return -EINVAL;
2380
2381 *qp = NULL;
2382 if (dest_devx) {
2383 devx_obj =
2384 uverbs_attr_get_obj(attrs,
2385 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2386
2387 /* Verify that the given DEVX object is a flow
2388 * steering destination.
2389 */
2390 if (!is_flow_dest(devx_obj, dest_id, dest_type))
2391 return -EINVAL;
2392 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */
2393 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS ||
2394 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2395 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2396 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
2397 return -EINVAL;
2398 } else if (dest_qp) {
2399 struct mlx5_ib_qp *mqp;
2400
2401 *qp = uverbs_attr_get_obj(attrs,
2402 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2403 if (IS_ERR(*qp))
2404 return PTR_ERR(*qp);
2405
2406 if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
2407 return -EINVAL;
2408
2409 mqp = to_mqp(*qp);
2410 if (mqp->is_rss)
2411 *dest_id = mqp->rss_qp.tirn;
2412 else
2413 *dest_id = mqp->raw_packet_qp.rq.tirn;
2414 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2415 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2416 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2417 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) &&
2418 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) {
2419 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2420 }
2421
2422 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
2423 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2424 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2425 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX))
2426 return -EINVAL;
2427
2428 return 0;
2429 }
2430
2431 static bool
is_flow_counter(void * obj,u32 offset,u32 * counter_id,u32 * fc_bulk_size)2432 is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size)
2433 {
2434 struct devx_obj *devx_obj = obj;
2435 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2436
2437 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
2438
2439 if (offset && offset >= devx_obj->flow_counter_bulk_size)
2440 return false;
2441
2442 *fc_bulk_size = devx_obj->flow_counter_bulk_size;
2443 *counter_id = MLX5_GET(dealloc_flow_counter_in,
2444 devx_obj->dinbox,
2445 flow_counter_id);
2446 *counter_id += offset;
2447 return true;
2448 }
2449
2450 return false;
2451 }
2452
2453 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)2454 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
2455 struct uverbs_attr_bundle *attrs)
2456 {
2457 struct mlx5_flow_context flow_context = {.flow_tag =
2458 MLX5_FS_DEFAULT_FLOW_TAG};
2459 int dest_id, dest_type = -1, inlen, len, ret, i;
2460 struct mlx5_ib_flow_handler *flow_handler;
2461 struct mlx5_ib_flow_matcher *fs_matcher;
2462 struct ib_uobject **arr_flow_actions;
2463 struct ib_uflow_resources *uflow_res;
2464 struct mlx5_flow_act flow_act = {};
2465 struct mlx5_fc *counter = NULL;
2466 struct ib_qp *qp = NULL;
2467 void *devx_obj, *cmd_in;
2468 struct ib_uobject *uobj;
2469 struct mlx5_ib_dev *dev;
2470 u32 flags;
2471
2472 if (!rdma_uattrs_has_raw_cap(attrs))
2473 return -EPERM;
2474
2475 fs_matcher = uverbs_attr_get_obj(attrs,
2476 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
2477 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
2478 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2479
2480 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
2481 return -EINVAL;
2482
2483 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
2484 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
2485
2486 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2487 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
2488
2489 len = uverbs_attr_get_uobjs_arr(attrs,
2490 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
2491 if (len) {
2492 u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0;
2493 devx_obj = arr_flow_actions[0]->object;
2494
2495 if (uverbs_attr_is_valid(attrs,
2496 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
2497
2498 int num_offsets = uverbs_attr_ptr_get_array_size(
2499 attrs,
2500 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
2501 sizeof(u32));
2502
2503 if (num_offsets != 1)
2504 return -EINVAL;
2505
2506 offset_attr = uverbs_attr_get_alloced_ptr(
2507 attrs,
2508 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
2509 offset = *offset_attr;
2510 }
2511
2512 if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size))
2513 return -EINVAL;
2514 counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size);
2515 if (IS_ERR(counter))
2516 return PTR_ERR(counter);
2517
2518 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2519 }
2520
2521 cmd_in = uverbs_attr_get_alloced_ptr(
2522 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2523 inlen = uverbs_attr_get_len(attrs,
2524 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2525
2526 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
2527 if (!uflow_res) {
2528 ret = -ENOMEM;
2529 goto destroy_counter;
2530 }
2531
2532 len = uverbs_attr_get_uobjs_arr(attrs,
2533 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
2534 for (i = 0; i < len; i++) {
2535 struct mlx5_ib_flow_action *maction =
2536 to_mflow_act(arr_flow_actions[i]->object);
2537
2538 ret = parse_flow_flow_action(maction, false, &flow_act);
2539 if (ret)
2540 goto err_out;
2541 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
2542 arr_flow_actions[i]->object);
2543 }
2544
2545 ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
2546 MLX5_IB_ATTR_CREATE_FLOW_TAG);
2547 if (!ret) {
2548 if (flow_context.flow_tag >= BIT(24)) {
2549 ret = -EINVAL;
2550 goto err_out;
2551 }
2552 flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
2553 }
2554
2555 flow_handler =
2556 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act,
2557 counter, cmd_in, inlen, dest_id, dest_type);
2558 if (IS_ERR(flow_handler)) {
2559 ret = PTR_ERR(flow_handler);
2560 goto err_out;
2561 }
2562
2563 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
2564
2565 return 0;
2566 err_out:
2567 ib_uverbs_flow_resources_free(uflow_res);
2568 destroy_counter:
2569 if (counter)
2570 mlx5_fc_local_destroy(counter);
2571 return ret;
2572 }
2573
flow_matcher_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2574 static int flow_matcher_cleanup(struct ib_uobject *uobject,
2575 enum rdma_remove_reason why,
2576 struct uverbs_attr_bundle *attrs)
2577 {
2578 struct mlx5_ib_flow_matcher *obj = uobject->object;
2579
2580 if (atomic_read(&obj->usecnt))
2581 return -EBUSY;
2582
2583 kfree(obj);
2584 return 0;
2585 }
2586
steering_anchor_create_ft(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,enum mlx5_flow_namespace_type ns_type)2587 static int steering_anchor_create_ft(struct mlx5_ib_dev *dev,
2588 struct mlx5_ib_flow_prio *ft_prio,
2589 enum mlx5_flow_namespace_type ns_type)
2590 {
2591 struct mlx5_flow_table_attr ft_attr = {};
2592 struct mlx5_flow_namespace *ns;
2593 struct mlx5_flow_table *ft;
2594
2595 if (ft_prio->anchor.ft)
2596 return 0;
2597
2598 ns = mlx5_get_flow_namespace(dev->mdev, ns_type);
2599 if (!ns)
2600 return -EOPNOTSUPP;
2601
2602 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
2603 ft_attr.uid = MLX5_SHARED_RESOURCE_UID;
2604 ft_attr.prio = 0;
2605 ft_attr.max_fte = 2;
2606 ft_attr.level = 1;
2607
2608 ft = mlx5_create_flow_table(ns, &ft_attr);
2609 if (IS_ERR(ft))
2610 return PTR_ERR(ft);
2611
2612 ft_prio->anchor.ft = ft;
2613
2614 return 0;
2615 }
2616
steering_anchor_destroy_ft(struct mlx5_ib_flow_prio * ft_prio)2617 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)
2618 {
2619 if (ft_prio->anchor.ft) {
2620 mlx5_destroy_flow_table(ft_prio->anchor.ft);
2621 ft_prio->anchor.ft = NULL;
2622 }
2623 }
2624
2625 static int
steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio * ft_prio)2626 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2627 {
2628 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2629 struct mlx5_flow_group *fg;
2630 void *flow_group_in;
2631 int err = 0;
2632
2633 if (ft_prio->anchor.fg_drop)
2634 return 0;
2635
2636 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2637 if (!flow_group_in)
2638 return -ENOMEM;
2639
2640 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
2641 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
2642
2643 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2644 if (IS_ERR(fg)) {
2645 err = PTR_ERR(fg);
2646 goto out;
2647 }
2648
2649 ft_prio->anchor.fg_drop = fg;
2650
2651 out:
2652 kvfree(flow_group_in);
2653
2654 return err;
2655 }
2656
2657 static void
steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio * ft_prio)2658 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2659 {
2660 if (ft_prio->anchor.fg_drop) {
2661 mlx5_destroy_flow_group(ft_prio->anchor.fg_drop);
2662 ft_prio->anchor.fg_drop = NULL;
2663 }
2664 }
2665
2666 static int
steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio * ft_prio)2667 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2668 {
2669 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2670 struct mlx5_flow_group *fg;
2671 void *flow_group_in;
2672 int err = 0;
2673
2674 if (ft_prio->anchor.fg_goto_table)
2675 return 0;
2676
2677 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2678 if (!flow_group_in)
2679 return -ENOMEM;
2680
2681 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2682 if (IS_ERR(fg)) {
2683 err = PTR_ERR(fg);
2684 goto out;
2685 }
2686 ft_prio->anchor.fg_goto_table = fg;
2687
2688 out:
2689 kvfree(flow_group_in);
2690
2691 return err;
2692 }
2693
2694 static void
steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio * ft_prio)2695 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2696 {
2697 if (ft_prio->anchor.fg_goto_table) {
2698 mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table);
2699 ft_prio->anchor.fg_goto_table = NULL;
2700 }
2701 }
2702
2703 static int
steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio * ft_prio)2704 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2705 {
2706 struct mlx5_flow_act flow_act = {};
2707 struct mlx5_flow_handle *handle;
2708
2709 if (ft_prio->anchor.rule_drop)
2710 return 0;
2711
2712 flow_act.fg = ft_prio->anchor.fg_drop;
2713 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2714
2715 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2716 NULL, 0);
2717 if (IS_ERR(handle))
2718 return PTR_ERR(handle);
2719
2720 ft_prio->anchor.rule_drop = handle;
2721
2722 return 0;
2723 }
2724
steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio * ft_prio)2725 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2726 {
2727 if (ft_prio->anchor.rule_drop) {
2728 mlx5_del_flow_rules(ft_prio->anchor.rule_drop);
2729 ft_prio->anchor.rule_drop = NULL;
2730 }
2731 }
2732
2733 static int
steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio * ft_prio)2734 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2735 {
2736 struct mlx5_flow_destination dest = {};
2737 struct mlx5_flow_act flow_act = {};
2738 struct mlx5_flow_handle *handle;
2739
2740 if (ft_prio->anchor.rule_goto_table)
2741 return 0;
2742
2743 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2744 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
2745 flow_act.fg = ft_prio->anchor.fg_goto_table;
2746
2747 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2748 dest.ft = ft_prio->flow_table;
2749
2750 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2751 &dest, 1);
2752 if (IS_ERR(handle))
2753 return PTR_ERR(handle);
2754
2755 ft_prio->anchor.rule_goto_table = handle;
2756
2757 return 0;
2758 }
2759
2760 static void
steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio * ft_prio)2761 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2762 {
2763 if (ft_prio->anchor.rule_goto_table) {
2764 mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table);
2765 ft_prio->anchor.rule_goto_table = NULL;
2766 }
2767 }
2768
steering_anchor_create_res(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,enum mlx5_flow_namespace_type ns_type)2769 static int steering_anchor_create_res(struct mlx5_ib_dev *dev,
2770 struct mlx5_ib_flow_prio *ft_prio,
2771 enum mlx5_flow_namespace_type ns_type)
2772 {
2773 int err;
2774
2775 err = steering_anchor_create_ft(dev, ft_prio, ns_type);
2776 if (err)
2777 return err;
2778
2779 err = steering_anchor_create_fg_drop(ft_prio);
2780 if (err)
2781 goto destroy_ft;
2782
2783 err = steering_anchor_create_fg_goto_table(ft_prio);
2784 if (err)
2785 goto destroy_fg_drop;
2786
2787 err = steering_anchor_create_rule_drop(ft_prio);
2788 if (err)
2789 goto destroy_fg_goto_table;
2790
2791 err = steering_anchor_create_rule_goto_table(ft_prio);
2792 if (err)
2793 goto destroy_rule_drop;
2794
2795 return 0;
2796
2797 destroy_rule_drop:
2798 steering_anchor_destroy_rule_drop(ft_prio);
2799 destroy_fg_goto_table:
2800 steering_anchor_destroy_fg_goto_table(ft_prio);
2801 destroy_fg_drop:
2802 steering_anchor_destroy_fg_drop(ft_prio);
2803 destroy_ft:
2804 steering_anchor_destroy_ft(ft_prio);
2805
2806 return err;
2807 }
2808
mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio * ft_prio)2809 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)
2810 {
2811 steering_anchor_destroy_rule_goto_table(ft_prio);
2812 steering_anchor_destroy_rule_drop(ft_prio);
2813 steering_anchor_destroy_fg_goto_table(ft_prio);
2814 steering_anchor_destroy_fg_drop(ft_prio);
2815 steering_anchor_destroy_ft(ft_prio);
2816 }
2817
steering_anchor_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2818 static int steering_anchor_cleanup(struct ib_uobject *uobject,
2819 enum rdma_remove_reason why,
2820 struct uverbs_attr_bundle *attrs)
2821 {
2822 struct mlx5_ib_steering_anchor *obj = uobject->object;
2823
2824 if (atomic_read(&obj->usecnt))
2825 return -EBUSY;
2826
2827 mutex_lock(&obj->dev->flow_db->lock);
2828 if (!--obj->ft_prio->anchor.rule_goto_table_ref)
2829 steering_anchor_destroy_rule_goto_table(obj->ft_prio);
2830
2831 put_flow_table(obj->dev, obj->ft_prio, true);
2832 mutex_unlock(&obj->dev->flow_db->lock);
2833
2834 kfree(obj);
2835 return 0;
2836 }
2837
fs_cleanup_anchor(struct mlx5_ib_flow_prio * prio,int count)2838 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio,
2839 int count)
2840 {
2841 while (count--)
2842 mlx5_steering_anchor_destroy_res(&prio[count]);
2843 }
2844
mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev * dev)2845 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev)
2846 {
2847 fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT);
2848 fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT);
2849 fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS);
2850 fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS);
2851 fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS);
2852 fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT);
2853 fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT);
2854 }
2855
mlx5_ib_matcher_ns(struct uverbs_attr_bundle * attrs,struct mlx5_ib_flow_matcher * obj)2856 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
2857 struct mlx5_ib_flow_matcher *obj)
2858 {
2859 enum mlx5_ib_uapi_flow_table_type ft_type =
2860 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
2861 u32 flags;
2862 int err;
2863
2864 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
2865 * users should switch to it. We leave this to not break userspace
2866 */
2867 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
2868 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
2869 return -EINVAL;
2870
2871 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
2872 err = uverbs_get_const(&ft_type, attrs,
2873 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
2874 if (err)
2875 return err;
2876
2877 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
2878 if (err)
2879 return err;
2880
2881 return 0;
2882 }
2883
2884 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
2885 err = uverbs_get_flags32(&flags, attrs,
2886 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
2887 IB_FLOW_ATTR_FLAGS_EGRESS);
2888 if (err)
2889 return err;
2890
2891 if (flags)
2892 return mlx5_ib_ft_type_to_namespace(
2893 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
2894 &obj->ns_type);
2895 }
2896
2897 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
2898
2899 return 0;
2900 }
2901
verify_context_caps(struct mlx5_ib_dev * dev,u64 enabled_caps)2902 static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps)
2903 {
2904 if (is_mdev_switchdev_mode(dev->mdev))
2905 return UCAP_ENABLED(enabled_caps,
2906 RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
2907
2908 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL);
2909 }
2910
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)2911 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
2912 struct uverbs_attr_bundle *attrs)
2913 {
2914 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2915 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
2916 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2917 struct mlx5_ib_flow_matcher *obj;
2918 int err;
2919
2920 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
2921 if (!obj)
2922 return -ENOMEM;
2923
2924 obj->mask_len = uverbs_attr_get_len(
2925 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2926 err = uverbs_copy_from(&obj->matcher_mask,
2927 attrs,
2928 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2929 if (err)
2930 goto end;
2931
2932 obj->flow_type = uverbs_attr_get_enum_id(
2933 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2934
2935 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
2936 err = uverbs_copy_from(&obj->priority,
2937 attrs,
2938 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2939 if (err)
2940 goto end;
2941 }
2942
2943 err = uverbs_copy_from(&obj->match_criteria_enable,
2944 attrs,
2945 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
2946 if (err)
2947 goto end;
2948
2949 err = mlx5_ib_matcher_ns(attrs, obj);
2950 if (err)
2951 goto end;
2952
2953 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2954 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) {
2955 err = -EINVAL;
2956 goto end;
2957 }
2958
2959 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) {
2960 err = uverbs_copy_from(&obj->ib_port, attrs,
2961 MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT);
2962 if (err)
2963 goto end;
2964 if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) {
2965 err = -EINVAL;
2966 goto end;
2967 }
2968 if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX &&
2969 obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) {
2970 err = -EINVAL;
2971 goto end;
2972 }
2973 if (!verify_context_caps(dev, uobj->context->enabled_caps)) {
2974 err = -EOPNOTSUPP;
2975 goto end;
2976 }
2977 }
2978
2979 uobj->object = obj;
2980 obj->mdev = dev->mdev;
2981 atomic_set(&obj->usecnt, 0);
2982 return 0;
2983
2984 end:
2985 kfree(obj);
2986 return err;
2987 }
2988
UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)2989 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
2990 struct uverbs_attr_bundle *attrs)
2991 {
2992 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2993 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE);
2994 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2995 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type;
2996 enum mlx5_flow_namespace_type ns_type;
2997 struct mlx5_ib_steering_anchor *obj;
2998 struct mlx5_ib_flow_prio *ft_prio;
2999 u16 priority;
3000 u32 ft_id;
3001 int err;
3002
3003 if (!rdma_dev_has_raw_cap(&dev->ib_dev))
3004 return -EPERM;
3005
3006 err = uverbs_get_const(&ib_uapi_ft_type, attrs,
3007 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE);
3008 if (err)
3009 return err;
3010
3011 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type);
3012 if (err)
3013 return err;
3014
3015 err = uverbs_copy_from(&priority, attrs,
3016 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY);
3017 if (err)
3018 return err;
3019
3020 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3021 if (!obj)
3022 return -ENOMEM;
3023
3024 mutex_lock(&dev->flow_db->lock);
3025
3026 ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0);
3027 if (IS_ERR(ft_prio)) {
3028 err = PTR_ERR(ft_prio);
3029 goto free_obj;
3030 }
3031
3032 ft_prio->refcount++;
3033
3034 if (!ft_prio->anchor.rule_goto_table_ref) {
3035 err = steering_anchor_create_res(dev, ft_prio, ns_type);
3036 if (err)
3037 goto put_flow_table;
3038 }
3039
3040 ft_prio->anchor.rule_goto_table_ref++;
3041
3042 ft_id = mlx5_flow_table_id(ft_prio->anchor.ft);
3043
3044 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3045 &ft_id, sizeof(ft_id));
3046 if (err)
3047 goto destroy_res;
3048
3049 mutex_unlock(&dev->flow_db->lock);
3050
3051 uobj->object = obj;
3052 obj->dev = dev;
3053 obj->ft_prio = ft_prio;
3054 atomic_set(&obj->usecnt, 0);
3055
3056 return 0;
3057
3058 destroy_res:
3059 --ft_prio->anchor.rule_goto_table_ref;
3060 mlx5_steering_anchor_destroy_res(ft_prio);
3061 put_flow_table:
3062 put_flow_table(dev, ft_prio, true);
3063 free_obj:
3064 mutex_unlock(&dev->flow_db->lock);
3065 kfree(obj);
3066
3067 return err;
3068 }
3069
3070 static struct ib_flow_action *
mlx5_ib_create_modify_header(struct mlx5_ib_dev * dev,enum mlx5_ib_uapi_flow_table_type ft_type,u8 num_actions,void * in)3071 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
3072 enum mlx5_ib_uapi_flow_table_type ft_type,
3073 u8 num_actions, void *in)
3074 {
3075 enum mlx5_flow_namespace_type namespace;
3076 struct mlx5_ib_flow_action *maction;
3077 int ret;
3078
3079 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3080 if (ret)
3081 return ERR_PTR(-EINVAL);
3082
3083 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3084 if (!maction)
3085 return ERR_PTR(-ENOMEM);
3086
3087 maction->flow_action_raw.modify_hdr =
3088 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
3089
3090 if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
3091 ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
3092 kfree(maction);
3093 return ERR_PTR(ret);
3094 }
3095 maction->flow_action_raw.sub_type =
3096 MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
3097 maction->flow_action_raw.dev = dev;
3098
3099 return &maction->ib_action;
3100 }
3101
mlx5_ib_modify_header_supported(struct mlx5_ib_dev * dev)3102 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
3103 {
3104 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3105 max_modify_header_actions) ||
3106 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
3107 max_modify_header_actions) ||
3108 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
3109 max_modify_header_actions);
3110 }
3111
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)3112 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
3113 struct uverbs_attr_bundle *attrs)
3114 {
3115 struct ib_uobject *uobj = uverbs_attr_get_uobject(
3116 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
3117 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3118 enum mlx5_ib_uapi_flow_table_type ft_type;
3119 struct ib_flow_action *action;
3120 int num_actions;
3121 void *in;
3122 int ret;
3123
3124 if (!mlx5_ib_modify_header_supported(mdev))
3125 return -EOPNOTSUPP;
3126
3127 in = uverbs_attr_get_alloced_ptr(attrs,
3128 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
3129
3130 num_actions = uverbs_attr_ptr_get_array_size(
3131 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3132 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
3133 if (num_actions < 0)
3134 return num_actions;
3135
3136 ret = uverbs_get_const(&ft_type, attrs,
3137 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
3138 if (ret)
3139 return ret;
3140 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
3141 if (IS_ERR(action))
3142 return PTR_ERR(action);
3143
3144 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
3145 IB_FLOW_ACTION_UNSPECIFIED);
3146
3147 return 0;
3148 }
3149
mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev * ibdev,u8 packet_reformat_type,u8 ft_type)3150 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
3151 u8 packet_reformat_type,
3152 u8 ft_type)
3153 {
3154 switch (packet_reformat_type) {
3155 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3156 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3157 return MLX5_CAP_FLOWTABLE(ibdev->mdev,
3158 encap_general_header);
3159 break;
3160 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3161 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3162 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
3163 reformat_l2_to_l3_tunnel);
3164 break;
3165 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3166 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3167 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
3168 reformat_l3_tunnel_to_l2);
3169 break;
3170 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
3171 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3172 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
3173 break;
3174 default:
3175 break;
3176 }
3177
3178 return false;
3179 }
3180
mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt,u8 * prm_prt)3181 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
3182 {
3183 switch (dv_prt) {
3184 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3185 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
3186 break;
3187 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3188 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
3189 break;
3190 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3191 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3192 break;
3193 default:
3194 return -EINVAL;
3195 }
3196
3197 return 0;
3198 }
3199
mlx5_ib_flow_action_create_packet_reformat_ctx(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_action * maction,u8 ft_type,u8 dv_prt,void * in,size_t len)3200 static int mlx5_ib_flow_action_create_packet_reformat_ctx(
3201 struct mlx5_ib_dev *dev,
3202 struct mlx5_ib_flow_action *maction,
3203 u8 ft_type, u8 dv_prt,
3204 void *in, size_t len)
3205 {
3206 struct mlx5_pkt_reformat_params reformat_params;
3207 enum mlx5_flow_namespace_type namespace;
3208 u8 prm_prt;
3209 int ret;
3210
3211 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3212 if (ret)
3213 return ret;
3214
3215 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
3216 if (ret)
3217 return ret;
3218
3219 memset(&reformat_params, 0, sizeof(reformat_params));
3220 reformat_params.type = prm_prt;
3221 reformat_params.size = len;
3222 reformat_params.data = in;
3223 maction->flow_action_raw.pkt_reformat =
3224 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params,
3225 namespace);
3226 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
3227 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
3228 return ret;
3229 }
3230
3231 maction->flow_action_raw.sub_type =
3232 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
3233 maction->flow_action_raw.dev = dev;
3234
3235 return 0;
3236 }
3237
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)3238 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
3239 struct uverbs_attr_bundle *attrs)
3240 {
3241 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
3242 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
3243 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3244 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
3245 enum mlx5_ib_uapi_flow_table_type ft_type;
3246 struct mlx5_ib_flow_action *maction;
3247 int ret;
3248
3249 ret = uverbs_get_const(&ft_type, attrs,
3250 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
3251 if (ret)
3252 return ret;
3253
3254 ret = uverbs_get_const(&dv_prt, attrs,
3255 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
3256 if (ret)
3257 return ret;
3258
3259 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
3260 return -EOPNOTSUPP;
3261
3262 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3263 if (!maction)
3264 return -ENOMEM;
3265
3266 if (dv_prt ==
3267 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
3268 maction->flow_action_raw.sub_type =
3269 MLX5_IB_FLOW_ACTION_DECAP;
3270 maction->flow_action_raw.dev = mdev;
3271 } else {
3272 void *in;
3273 int len;
3274
3275 in = uverbs_attr_get_alloced_ptr(attrs,
3276 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3277 if (IS_ERR(in)) {
3278 ret = PTR_ERR(in);
3279 goto free_maction;
3280 }
3281
3282 len = uverbs_attr_get_len(attrs,
3283 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3284
3285 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
3286 maction, ft_type, dv_prt, in, len);
3287 if (ret)
3288 goto free_maction;
3289 }
3290
3291 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
3292 IB_FLOW_ACTION_UNSPECIFIED);
3293 return 0;
3294
3295 free_maction:
3296 kfree(maction);
3297 return ret;
3298 }
3299
3300 DECLARE_UVERBS_NAMED_METHOD(
3301 MLX5_IB_METHOD_CREATE_FLOW,
3302 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3303 UVERBS_OBJECT_FLOW,
3304 UVERBS_ACCESS_NEW,
3305 UA_MANDATORY),
3306 UVERBS_ATTR_PTR_IN(
3307 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
3308 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3309 UA_MANDATORY,
3310 UA_ALLOC_AND_COPY),
3311 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
3312 MLX5_IB_OBJECT_FLOW_MATCHER,
3313 UVERBS_ACCESS_READ,
3314 UA_MANDATORY),
3315 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
3316 UVERBS_OBJECT_QP,
3317 UVERBS_ACCESS_READ),
3318 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
3319 MLX5_IB_OBJECT_DEVX_OBJ,
3320 UVERBS_ACCESS_READ),
3321 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
3322 UVERBS_OBJECT_FLOW_ACTION,
3323 UVERBS_ACCESS_READ, 1,
3324 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
3325 UA_OPTIONAL),
3326 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
3327 UVERBS_ATTR_TYPE(u32),
3328 UA_OPTIONAL),
3329 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
3330 MLX5_IB_OBJECT_DEVX_OBJ,
3331 UVERBS_ACCESS_READ, 1, 1,
3332 UA_OPTIONAL),
3333 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
3334 UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
3335 UA_OPTIONAL,
3336 UA_ALLOC_AND_COPY),
3337 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
3338 enum mlx5_ib_create_flow_flags,
3339 UA_OPTIONAL));
3340
3341 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3342 MLX5_IB_METHOD_DESTROY_FLOW,
3343 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3344 UVERBS_OBJECT_FLOW,
3345 UVERBS_ACCESS_DESTROY,
3346 UA_MANDATORY));
3347
3348 ADD_UVERBS_METHODS(mlx5_ib_fs,
3349 UVERBS_OBJECT_FLOW,
3350 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
3351 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
3352
3353 DECLARE_UVERBS_NAMED_METHOD(
3354 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
3355 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
3356 UVERBS_OBJECT_FLOW_ACTION,
3357 UVERBS_ACCESS_NEW,
3358 UA_MANDATORY),
3359 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3360 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
3361 set_add_copy_action_in_auto)),
3362 UA_MANDATORY,
3363 UA_ALLOC_AND_COPY),
3364 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
3365 enum mlx5_ib_uapi_flow_table_type,
3366 UA_MANDATORY));
3367
3368 DECLARE_UVERBS_NAMED_METHOD(
3369 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
3370 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
3371 UVERBS_OBJECT_FLOW_ACTION,
3372 UVERBS_ACCESS_NEW,
3373 UA_MANDATORY),
3374 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
3375 UVERBS_ATTR_MIN_SIZE(1),
3376 UA_ALLOC_AND_COPY,
3377 UA_OPTIONAL),
3378 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
3379 enum mlx5_ib_uapi_flow_action_packet_reformat_type,
3380 UA_MANDATORY),
3381 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
3382 enum mlx5_ib_uapi_flow_table_type,
3383 UA_MANDATORY));
3384
3385 ADD_UVERBS_METHODS(
3386 mlx5_ib_flow_actions,
3387 UVERBS_OBJECT_FLOW_ACTION,
3388 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
3389 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
3390
3391 DECLARE_UVERBS_NAMED_METHOD(
3392 MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
3393 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
3394 MLX5_IB_OBJECT_FLOW_MATCHER,
3395 UVERBS_ACCESS_NEW,
3396 UA_MANDATORY),
3397 UVERBS_ATTR_PTR_IN(
3398 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
3399 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3400 UA_MANDATORY),
3401 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
3402 mlx5_ib_flow_type,
3403 UA_MANDATORY),
3404 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
3405 UVERBS_ATTR_TYPE(u8),
3406 UA_MANDATORY),
3407 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
3408 enum ib_flow_flags,
3409 UA_OPTIONAL),
3410 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
3411 enum mlx5_ib_uapi_flow_table_type,
3412 UA_OPTIONAL),
3413 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT,
3414 UVERBS_ATTR_TYPE(u32),
3415 UA_OPTIONAL));
3416
3417 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3418 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
3419 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
3420 MLX5_IB_OBJECT_FLOW_MATCHER,
3421 UVERBS_ACCESS_DESTROY,
3422 UA_MANDATORY));
3423
3424 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
3425 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
3426 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
3427 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
3428
3429 DECLARE_UVERBS_NAMED_METHOD(
3430 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE,
3431 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE,
3432 MLX5_IB_OBJECT_STEERING_ANCHOR,
3433 UVERBS_ACCESS_NEW,
3434 UA_MANDATORY),
3435 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE,
3436 enum mlx5_ib_uapi_flow_table_type,
3437 UA_MANDATORY),
3438 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY,
3439 UVERBS_ATTR_TYPE(u16),
3440 UA_MANDATORY),
3441 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3442 UVERBS_ATTR_TYPE(u32),
3443 UA_MANDATORY));
3444
3445 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3446 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY,
3447 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE,
3448 MLX5_IB_OBJECT_STEERING_ANCHOR,
3449 UVERBS_ACCESS_DESTROY,
3450 UA_MANDATORY));
3451
3452 DECLARE_UVERBS_NAMED_OBJECT(
3453 MLX5_IB_OBJECT_STEERING_ANCHOR,
3454 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup),
3455 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE),
3456 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY));
3457
3458 const struct uapi_definition mlx5_ib_flow_defs[] = {
3459 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3460 MLX5_IB_OBJECT_FLOW_MATCHER),
3461 UAPI_DEF_CHAIN_OBJ_TREE(
3462 UVERBS_OBJECT_FLOW,
3463 &mlx5_ib_fs),
3464 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
3465 &mlx5_ib_flow_actions),
3466 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3467 MLX5_IB_OBJECT_STEERING_ANCHOR,
3468 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)),
3469 {},
3470 };
3471
3472 static const struct ib_device_ops flow_ops = {
3473 .create_flow = mlx5_ib_create_flow,
3474 .destroy_flow = mlx5_ib_destroy_flow,
3475 .destroy_flow_action = mlx5_ib_destroy_flow_action,
3476 };
3477
mlx5_ib_fs_init(struct mlx5_ib_dev * dev)3478 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
3479 {
3480 int i, j;
3481
3482 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
3483
3484 if (!dev->flow_db)
3485 return -ENOMEM;
3486
3487 for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) {
3488 dev->flow_db->rdma_transport_rx[i] =
3489 kcalloc(dev->num_ports,
3490 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3491 if (!dev->flow_db->rdma_transport_rx[i])
3492 goto free_rdma_transport_rx;
3493 }
3494
3495 for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) {
3496 dev->flow_db->rdma_transport_tx[j] =
3497 kcalloc(dev->num_ports,
3498 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3499 if (!dev->flow_db->rdma_transport_tx[j])
3500 goto free_rdma_transport_tx;
3501 }
3502
3503 mutex_init(&dev->flow_db->lock);
3504
3505 ib_set_device_ops(&dev->ib_dev, &flow_ops);
3506 return 0;
3507
3508 free_rdma_transport_tx:
3509 while (j--)
3510 kfree(dev->flow_db->rdma_transport_tx[j]);
3511 free_rdma_transport_rx:
3512 while (i--)
3513 kfree(dev->flow_db->rdma_transport_rx[i]);
3514 kfree(dev->flow_db);
3515 return -ENOMEM;
3516 }
3517