1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/uverbs_std_types.h>
11 #include <rdma/mlx5_user_ioctl_cmds.h>
12 #include <rdma/mlx5_user_ioctl_verbs.h>
13 #include <rdma/ib_hdrs.h>
14 #include <rdma/ib_umem.h>
15 #include <rdma/ib_ucaps.h>
16 #include <linux/mlx5/driver.h>
17 #include <linux/mlx5/fs.h>
18 #include <linux/mlx5/fs_helpers.h>
19 #include <linux/mlx5/eswitch.h>
20 #include <net/inet_ecn.h>
21 #include "mlx5_ib.h"
22 #include "counters.h"
23 #include "devx.h"
24 #include "fs.h"
25
26 #define UVERBS_MODULE_NAME mlx5_ib
27 #include <rdma/uverbs_named_ioctl.h>
28
29 enum {
30 MATCH_CRITERIA_ENABLE_OUTER_BIT,
31 MATCH_CRITERIA_ENABLE_MISC_BIT,
32 MATCH_CRITERIA_ENABLE_INNER_BIT,
33 MATCH_CRITERIA_ENABLE_MISC2_BIT
34 };
35
36
37 struct mlx5_per_qp_opfc {
38 struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX];
39 };
40
41 #define HEADER_IS_ZERO(match_criteria, headers) \
42 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
43 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
44
get_match_criteria_enable(u32 * match_criteria)45 static u8 get_match_criteria_enable(u32 *match_criteria)
46 {
47 u8 match_criteria_enable;
48
49 match_criteria_enable =
50 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
51 MATCH_CRITERIA_ENABLE_OUTER_BIT;
52 match_criteria_enable |=
53 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
54 MATCH_CRITERIA_ENABLE_MISC_BIT;
55 match_criteria_enable |=
56 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
57 MATCH_CRITERIA_ENABLE_INNER_BIT;
58 match_criteria_enable |=
59 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
60 MATCH_CRITERIA_ENABLE_MISC2_BIT;
61
62 return match_criteria_enable;
63 }
64
set_proto(void * outer_c,void * outer_v,u8 mask,u8 val)65 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
66 {
67 u8 entry_mask;
68 u8 entry_val;
69 int err = 0;
70
71 if (!mask)
72 goto out;
73
74 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
75 ip_protocol);
76 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
77 ip_protocol);
78 if (!entry_mask) {
79 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
80 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
81 goto out;
82 }
83 /* Don't override existing ip protocol */
84 if (mask != entry_mask || val != entry_val)
85 err = -EINVAL;
86 out:
87 return err;
88 }
89
set_flow_label(void * misc_c,void * misc_v,u32 mask,u32 val,bool inner)90 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
91 bool inner)
92 {
93 if (inner) {
94 MLX5_SET(fte_match_set_misc,
95 misc_c, inner_ipv6_flow_label, mask);
96 MLX5_SET(fte_match_set_misc,
97 misc_v, inner_ipv6_flow_label, val);
98 } else {
99 MLX5_SET(fte_match_set_misc,
100 misc_c, outer_ipv6_flow_label, mask);
101 MLX5_SET(fte_match_set_misc,
102 misc_v, outer_ipv6_flow_label, val);
103 }
104 }
105
set_tos(void * outer_c,void * outer_v,u8 mask,u8 val)106 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
107 {
108 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
109 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
110 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
111 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
112 }
113
check_mpls_supp_fields(u32 field_support,const __be32 * set_mask)114 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
115 {
116 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
117 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
118 return -EOPNOTSUPP;
119
120 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
121 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
122 return -EOPNOTSUPP;
123
124 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
125 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
126 return -EOPNOTSUPP;
127
128 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
129 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
130 return -EOPNOTSUPP;
131
132 return 0;
133 }
134
135 #define LAST_ETH_FIELD vlan_tag
136 #define LAST_IPV4_FIELD tos
137 #define LAST_IPV6_FIELD traffic_class
138 #define LAST_TCP_UDP_FIELD src_port
139 #define LAST_TUNNEL_FIELD tunnel_id
140 #define LAST_FLOW_TAG_FIELD tag_id
141 #define LAST_DROP_FIELD size
142 #define LAST_COUNTERS_FIELD counters
143
144 /* Field is the last supported field */
145 #define FIELDS_NOT_SUPPORTED(filter, field) \
146 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \
147 sizeof(filter) - offsetofend(typeof(filter), field))
148
parse_flow_flow_action(struct mlx5_ib_flow_action * maction,bool is_egress,struct mlx5_flow_act * action)149 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
150 bool is_egress,
151 struct mlx5_flow_act *action)
152 {
153
154 switch (maction->ib_action.type) {
155 case IB_FLOW_ACTION_UNSPECIFIED:
156 if (maction->flow_action_raw.sub_type ==
157 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
158 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
159 return -EINVAL;
160 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
161 action->modify_hdr =
162 maction->flow_action_raw.modify_hdr;
163 return 0;
164 }
165 if (maction->flow_action_raw.sub_type ==
166 MLX5_IB_FLOW_ACTION_DECAP) {
167 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
168 return -EINVAL;
169 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
170 return 0;
171 }
172 if (maction->flow_action_raw.sub_type ==
173 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
174 if (action->action &
175 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
176 return -EINVAL;
177 action->action |=
178 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
179 action->pkt_reformat =
180 maction->flow_action_raw.pkt_reformat;
181 return 0;
182 }
183 fallthrough;
184 default:
185 return -EOPNOTSUPP;
186 }
187 }
188
parse_flow_attr(struct mlx5_core_dev * mdev,struct mlx5_flow_spec * spec,const union ib_flow_spec * ib_spec,const struct ib_flow_attr * flow_attr,struct mlx5_flow_act * action,u32 prev_type)189 static int parse_flow_attr(struct mlx5_core_dev *mdev,
190 struct mlx5_flow_spec *spec,
191 const union ib_flow_spec *ib_spec,
192 const struct ib_flow_attr *flow_attr,
193 struct mlx5_flow_act *action, u32 prev_type)
194 {
195 struct mlx5_flow_context *flow_context = &spec->flow_context;
196 u32 *match_c = spec->match_criteria;
197 u32 *match_v = spec->match_value;
198 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
199 misc_parameters);
200 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
201 misc_parameters);
202 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
203 misc_parameters_2);
204 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
205 misc_parameters_2);
206 void *headers_c;
207 void *headers_v;
208 int match_ipv;
209 int ret;
210
211 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
212 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
213 inner_headers);
214 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
215 inner_headers);
216 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
217 ft_field_support.inner_ip_version);
218 } else {
219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
220 outer_headers);
221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
222 outer_headers);
223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
224 ft_field_support.outer_ip_version);
225 }
226
227 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
228 case IB_FLOW_SPEC_ETH:
229 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
230 return -EOPNOTSUPP;
231
232 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
233 dmac_47_16),
234 ib_spec->eth.mask.dst_mac);
235 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
236 dmac_47_16),
237 ib_spec->eth.val.dst_mac);
238
239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
240 smac_47_16),
241 ib_spec->eth.mask.src_mac);
242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
243 smac_47_16),
244 ib_spec->eth.val.src_mac);
245
246 if (ib_spec->eth.mask.vlan_tag) {
247 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
248 cvlan_tag, 1);
249 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
250 cvlan_tag, 1);
251
252 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
253 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
254 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
255 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
256
257 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
258 first_cfi,
259 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
260 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
261 first_cfi,
262 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
263
264 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
265 first_prio,
266 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
267 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
268 first_prio,
269 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
270 }
271 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
272 ethertype, ntohs(ib_spec->eth.mask.ether_type));
273 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
274 ethertype, ntohs(ib_spec->eth.val.ether_type));
275 break;
276 case IB_FLOW_SPEC_IPV4:
277 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
278 return -EOPNOTSUPP;
279
280 if (match_ipv) {
281 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
282 ip_version, 0xf);
283 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
284 ip_version, MLX5_FS_IPV4_VERSION);
285 } else {
286 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
287 ethertype, 0xffff);
288 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
289 ethertype, ETH_P_IP);
290 }
291
292 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
293 src_ipv4_src_ipv6.ipv4_layout.ipv4),
294 &ib_spec->ipv4.mask.src_ip,
295 sizeof(ib_spec->ipv4.mask.src_ip));
296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
297 src_ipv4_src_ipv6.ipv4_layout.ipv4),
298 &ib_spec->ipv4.val.src_ip,
299 sizeof(ib_spec->ipv4.val.src_ip));
300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
301 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
302 &ib_spec->ipv4.mask.dst_ip,
303 sizeof(ib_spec->ipv4.mask.dst_ip));
304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
306 &ib_spec->ipv4.val.dst_ip,
307 sizeof(ib_spec->ipv4.val.dst_ip));
308
309 set_tos(headers_c, headers_v,
310 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
311
312 if (set_proto(headers_c, headers_v,
313 ib_spec->ipv4.mask.proto,
314 ib_spec->ipv4.val.proto))
315 return -EINVAL;
316 break;
317 case IB_FLOW_SPEC_IPV6:
318 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
319 return -EOPNOTSUPP;
320
321 if (match_ipv) {
322 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
323 ip_version, 0xf);
324 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
325 ip_version, MLX5_FS_IPV6_VERSION);
326 } else {
327 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
328 ethertype, 0xffff);
329 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
330 ethertype, ETH_P_IPV6);
331 }
332
333 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
334 src_ipv4_src_ipv6.ipv6_layout.ipv6),
335 &ib_spec->ipv6.mask.src_ip,
336 sizeof(ib_spec->ipv6.mask.src_ip));
337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
338 src_ipv4_src_ipv6.ipv6_layout.ipv6),
339 &ib_spec->ipv6.val.src_ip,
340 sizeof(ib_spec->ipv6.val.src_ip));
341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
342 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
343 &ib_spec->ipv6.mask.dst_ip,
344 sizeof(ib_spec->ipv6.mask.dst_ip));
345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
347 &ib_spec->ipv6.val.dst_ip,
348 sizeof(ib_spec->ipv6.val.dst_ip));
349
350 set_tos(headers_c, headers_v,
351 ib_spec->ipv6.mask.traffic_class,
352 ib_spec->ipv6.val.traffic_class);
353
354 if (set_proto(headers_c, headers_v,
355 ib_spec->ipv6.mask.next_hdr,
356 ib_spec->ipv6.val.next_hdr))
357 return -EINVAL;
358
359 set_flow_label(misc_params_c, misc_params_v,
360 ntohl(ib_spec->ipv6.mask.flow_label),
361 ntohl(ib_spec->ipv6.val.flow_label),
362 ib_spec->type & IB_FLOW_SPEC_INNER);
363 break;
364 case IB_FLOW_SPEC_ESP:
365 return -EOPNOTSUPP;
366 case IB_FLOW_SPEC_TCP:
367 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
368 LAST_TCP_UDP_FIELD))
369 return -EOPNOTSUPP;
370
371 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
372 return -EINVAL;
373
374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
375 ntohs(ib_spec->tcp_udp.mask.src_port));
376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
377 ntohs(ib_spec->tcp_udp.val.src_port));
378
379 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
380 ntohs(ib_spec->tcp_udp.mask.dst_port));
381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
382 ntohs(ib_spec->tcp_udp.val.dst_port));
383 break;
384 case IB_FLOW_SPEC_UDP:
385 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
386 LAST_TCP_UDP_FIELD))
387 return -EOPNOTSUPP;
388
389 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
390 return -EINVAL;
391
392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
393 ntohs(ib_spec->tcp_udp.mask.src_port));
394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
395 ntohs(ib_spec->tcp_udp.val.src_port));
396
397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
398 ntohs(ib_spec->tcp_udp.mask.dst_port));
399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
400 ntohs(ib_spec->tcp_udp.val.dst_port));
401 break;
402 case IB_FLOW_SPEC_GRE:
403 if (ib_spec->gre.mask.c_ks_res0_ver)
404 return -EOPNOTSUPP;
405
406 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
407 return -EINVAL;
408
409 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
410 0xff);
411 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
412 IPPROTO_GRE);
413
414 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
415 ntohs(ib_spec->gre.mask.protocol));
416 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
417 ntohs(ib_spec->gre.val.protocol));
418
419 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
420 gre_key.nvgre.hi),
421 &ib_spec->gre.mask.key,
422 sizeof(ib_spec->gre.mask.key));
423 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
424 gre_key.nvgre.hi),
425 &ib_spec->gre.val.key,
426 sizeof(ib_spec->gre.val.key));
427 break;
428 case IB_FLOW_SPEC_MPLS:
429 switch (prev_type) {
430 case IB_FLOW_SPEC_UDP:
431 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
432 ft_field_support.outer_first_mpls_over_udp),
433 &ib_spec->mpls.mask.tag))
434 return -EOPNOTSUPP;
435
436 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
437 outer_first_mpls_over_udp),
438 &ib_spec->mpls.val.tag,
439 sizeof(ib_spec->mpls.val.tag));
440 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
441 outer_first_mpls_over_udp),
442 &ib_spec->mpls.mask.tag,
443 sizeof(ib_spec->mpls.mask.tag));
444 break;
445 case IB_FLOW_SPEC_GRE:
446 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
447 ft_field_support.outer_first_mpls_over_gre),
448 &ib_spec->mpls.mask.tag))
449 return -EOPNOTSUPP;
450
451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
452 outer_first_mpls_over_gre),
453 &ib_spec->mpls.val.tag,
454 sizeof(ib_spec->mpls.val.tag));
455 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
456 outer_first_mpls_over_gre),
457 &ib_spec->mpls.mask.tag,
458 sizeof(ib_spec->mpls.mask.tag));
459 break;
460 default:
461 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
462 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
463 ft_field_support.inner_first_mpls),
464 &ib_spec->mpls.mask.tag))
465 return -EOPNOTSUPP;
466
467 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
468 inner_first_mpls),
469 &ib_spec->mpls.val.tag,
470 sizeof(ib_spec->mpls.val.tag));
471 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
472 inner_first_mpls),
473 &ib_spec->mpls.mask.tag,
474 sizeof(ib_spec->mpls.mask.tag));
475 } else {
476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
477 ft_field_support.outer_first_mpls),
478 &ib_spec->mpls.mask.tag))
479 return -EOPNOTSUPP;
480
481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
482 outer_first_mpls),
483 &ib_spec->mpls.val.tag,
484 sizeof(ib_spec->mpls.val.tag));
485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
486 outer_first_mpls),
487 &ib_spec->mpls.mask.tag,
488 sizeof(ib_spec->mpls.mask.tag));
489 }
490 }
491 break;
492 case IB_FLOW_SPEC_VXLAN_TUNNEL:
493 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
494 LAST_TUNNEL_FIELD))
495 return -EOPNOTSUPP;
496
497 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
498 ntohl(ib_spec->tunnel.mask.tunnel_id));
499 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
500 ntohl(ib_spec->tunnel.val.tunnel_id));
501 break;
502 case IB_FLOW_SPEC_ACTION_TAG:
503 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
504 LAST_FLOW_TAG_FIELD))
505 return -EOPNOTSUPP;
506 if (ib_spec->flow_tag.tag_id >= BIT(24))
507 return -EINVAL;
508
509 flow_context->flow_tag = ib_spec->flow_tag.tag_id;
510 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
511 break;
512 case IB_FLOW_SPEC_ACTION_DROP:
513 if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
514 LAST_DROP_FIELD))
515 return -EOPNOTSUPP;
516 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
517 break;
518 case IB_FLOW_SPEC_ACTION_HANDLE:
519 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
520 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
521 if (ret)
522 return ret;
523 break;
524 case IB_FLOW_SPEC_ACTION_COUNT:
525 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
526 LAST_COUNTERS_FIELD))
527 return -EOPNOTSUPP;
528
529 /* for now support only one counters spec per flow */
530 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
531 return -EINVAL;
532
533 action->counters = ib_spec->flow_count.counters;
534 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
535 break;
536 default:
537 return -EINVAL;
538 }
539
540 return 0;
541 }
542
543 /* If a flow could catch both multicast and unicast packets,
544 * it won't fall into the multicast flow steering table and this rule
545 * could steal other multicast packets.
546 */
flow_is_multicast_only(const struct ib_flow_attr * ib_attr)547 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
548 {
549 union ib_flow_spec *flow_spec;
550
551 if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
552 ib_attr->num_of_specs < 1)
553 return false;
554
555 flow_spec = (union ib_flow_spec *)(ib_attr + 1);
556 if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
557 struct ib_flow_spec_ipv4 *ipv4_spec;
558
559 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
560 if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
561 return true;
562
563 return false;
564 }
565
566 if (flow_spec->type == IB_FLOW_SPEC_ETH) {
567 struct ib_flow_spec_eth *eth_spec;
568
569 eth_spec = (struct ib_flow_spec_eth *)flow_spec;
570 return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
571 is_multicast_ether_addr(eth_spec->val.dst_mac);
572 }
573
574 return false;
575 }
576
is_valid_ethertype(struct mlx5_core_dev * mdev,const struct ib_flow_attr * flow_attr,bool check_inner)577 static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
578 const struct ib_flow_attr *flow_attr,
579 bool check_inner)
580 {
581 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
582 int match_ipv = check_inner ?
583 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
584 ft_field_support.inner_ip_version) :
585 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
586 ft_field_support.outer_ip_version);
587 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
588 bool ipv4_spec_valid, ipv6_spec_valid;
589 unsigned int ip_spec_type = 0;
590 bool has_ethertype = false;
591 unsigned int spec_index;
592 bool mask_valid = true;
593 u16 eth_type = 0;
594 bool type_valid;
595
596 /* Validate that ethertype is correct */
597 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
598 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
599 ib_spec->eth.mask.ether_type) {
600 mask_valid = (ib_spec->eth.mask.ether_type ==
601 htons(0xffff));
602 has_ethertype = true;
603 eth_type = ntohs(ib_spec->eth.val.ether_type);
604 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
605 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
606 ip_spec_type = ib_spec->type;
607 }
608 ib_spec = (void *)ib_spec + ib_spec->size;
609 }
610
611 type_valid = (!has_ethertype) || (!ip_spec_type);
612 if (!type_valid && mask_valid) {
613 ipv4_spec_valid = (eth_type == ETH_P_IP) &&
614 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
615 ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
616 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
617
618 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
619 (((eth_type == ETH_P_MPLS_UC) ||
620 (eth_type == ETH_P_MPLS_MC)) && match_ipv);
621 }
622
623 return type_valid;
624 }
625
is_valid_attr(struct mlx5_core_dev * mdev,const struct ib_flow_attr * flow_attr)626 static bool is_valid_attr(struct mlx5_core_dev *mdev,
627 const struct ib_flow_attr *flow_attr)
628 {
629 return is_valid_ethertype(mdev, flow_attr, false) &&
630 is_valid_ethertype(mdev, flow_attr, true);
631 }
632
put_flow_table(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * prio,bool ft_added)633 static void put_flow_table(struct mlx5_ib_dev *dev,
634 struct mlx5_ib_flow_prio *prio, bool ft_added)
635 {
636 prio->refcount -= !!ft_added;
637 if (!prio->refcount) {
638 mlx5_destroy_flow_table(prio->flow_table);
639 prio->flow_table = NULL;
640 }
641 }
642
mlx5_ib_destroy_flow(struct ib_flow * flow_id)643 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
644 {
645 struct mlx5_ib_flow_handler *handler = container_of(flow_id,
646 struct mlx5_ib_flow_handler,
647 ibflow);
648 struct mlx5_ib_flow_handler *iter, *tmp;
649 struct mlx5_ib_dev *dev = handler->dev;
650
651 mutex_lock(&dev->flow_db->lock);
652
653 list_for_each_entry_safe(iter, tmp, &handler->list, list) {
654 mlx5_del_flow_rules(iter->rule);
655 put_flow_table(dev, iter->prio, true);
656 list_del(&iter->list);
657 kfree(iter);
658 }
659
660 mlx5_del_flow_rules(handler->rule);
661 put_flow_table(dev, handler->prio, true);
662 mlx5_ib_counters_clear_description(handler->ibcounters);
663 mutex_unlock(&dev->flow_db->lock);
664 if (handler->flow_matcher)
665 atomic_dec(&handler->flow_matcher->usecnt);
666 kfree(handler);
667
668 return 0;
669 }
670
ib_prio_to_core_prio(unsigned int priority,bool dont_trap)671 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
672 {
673 priority *= 2;
674 if (!dont_trap)
675 priority++;
676 return priority;
677 }
678
679 enum flow_table_type {
680 MLX5_IB_FT_RX,
681 MLX5_IB_FT_TX
682 };
683
684 #define MLX5_FS_MAX_TYPES 6
685 #define MLX5_FS_MAX_ENTRIES BIT(16)
686
mlx5_ib_shared_ft_allowed(struct ib_device * device)687 static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device)
688 {
689 struct mlx5_ib_dev *dev = to_mdev(device);
690
691 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed);
692 }
693
_get_prio(struct mlx5_ib_dev * dev,struct mlx5_flow_namespace * ns,struct mlx5_ib_flow_prio * prio,int priority,int num_entries,int num_groups,u32 flags,u16 vport)694 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_ib_dev *dev,
695 struct mlx5_flow_namespace *ns,
696 struct mlx5_ib_flow_prio *prio,
697 int priority,
698 int num_entries, int num_groups,
699 u32 flags, u16 vport)
700 {
701 struct mlx5_flow_table_attr ft_attr = {};
702 struct mlx5_flow_table *ft;
703
704 ft_attr.prio = priority;
705 ft_attr.max_fte = num_entries;
706 ft_attr.flags = flags;
707 ft_attr.vport = vport;
708 ft_attr.autogroup.max_num_groups = num_groups;
709 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
710 if (IS_ERR(ft))
711 return ERR_CAST(ft);
712
713 prio->flow_table = ft;
714 prio->refcount = 0;
715 return prio;
716 }
717
get_flow_table(struct mlx5_ib_dev * dev,struct ib_flow_attr * flow_attr,enum flow_table_type ft_type)718 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
719 struct ib_flow_attr *flow_attr,
720 enum flow_table_type ft_type)
721 {
722 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
723 struct mlx5_flow_namespace *ns = NULL;
724 enum mlx5_flow_namespace_type fn_type;
725 struct mlx5_ib_flow_prio *prio;
726 struct mlx5_flow_table *ft;
727 int max_table_size;
728 int num_entries;
729 int num_groups;
730 bool esw_encap;
731 u32 flags = 0;
732 int priority;
733
734 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
735 log_max_ft_size));
736 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
737 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
738 switch (flow_attr->type) {
739 case IB_FLOW_ATTR_NORMAL:
740 if (flow_is_multicast_only(flow_attr) && !dont_trap)
741 priority = MLX5_IB_FLOW_MCAST_PRIO;
742 else
743 priority = ib_prio_to_core_prio(flow_attr->priority,
744 dont_trap);
745 if (ft_type == MLX5_IB_FT_RX) {
746 fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
747 prio = &dev->flow_db->prios[priority];
748 if (!dev->is_rep && !esw_encap &&
749 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
750 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
751 if (!dev->is_rep && !esw_encap &&
752 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
753 reformat_l3_tunnel_to_l2))
754 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
755 } else {
756 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(
757 dev->mdev, log_max_ft_size));
758 fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
759 prio = &dev->flow_db->egress_prios[priority];
760 if (!dev->is_rep && !esw_encap &&
761 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
762 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
763 }
764 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
765 num_entries = MLX5_FS_MAX_ENTRIES;
766 num_groups = MLX5_FS_MAX_TYPES;
767 break;
768 case IB_FLOW_ATTR_ALL_DEFAULT:
769 case IB_FLOW_ATTR_MC_DEFAULT:
770 ns = mlx5_get_flow_namespace(dev->mdev,
771 MLX5_FLOW_NAMESPACE_LEFTOVERS);
772 build_leftovers_ft_param(&priority, &num_entries, &num_groups);
773 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
774 break;
775 case IB_FLOW_ATTR_SNIFFER:
776 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
777 allow_sniffer_and_nic_rx_shared_tir))
778 return ERR_PTR(-EOPNOTSUPP);
779
780 ns = mlx5_get_flow_namespace(
781 dev->mdev, ft_type == MLX5_IB_FT_RX ?
782 MLX5_FLOW_NAMESPACE_SNIFFER_RX :
783 MLX5_FLOW_NAMESPACE_SNIFFER_TX);
784
785 prio = &dev->flow_db->sniffer[ft_type];
786 priority = 0;
787 num_entries = 1;
788 num_groups = 1;
789 break;
790 default:
791 break;
792 }
793
794 if (!ns)
795 return ERR_PTR(-EOPNOTSUPP);
796
797 max_table_size = min_t(int, num_entries, max_table_size);
798
799 ft = prio->flow_table;
800 if (!ft)
801 return _get_prio(dev, ns, prio, priority, max_table_size,
802 num_groups, flags, 0);
803
804 return prio;
805 }
806
807 enum {
808 RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO,
809 RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO,
810 RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
811 RDMA_RX_ECN_OPCOUNTER_PRIO,
812 RDMA_RX_CNP_OPCOUNTER_PRIO,
813 RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO,
814 };
815
816 enum {
817 RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO,
818 RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
819 RDMA_TX_CNP_OPCOUNTER_PRIO,
820 RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO,
821 };
822
set_vhca_port_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec)823 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num,
824 struct mlx5_flow_spec *spec)
825 {
826 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
827 ft_field_support.source_vhca_port) ||
828 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
829 ft_field_support.source_vhca_port))
830 return -EOPNOTSUPP;
831
832 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria,
833 misc_parameters.source_vhca_port);
834 MLX5_SET(fte_match_param, &spec->match_value,
835 misc_parameters.source_vhca_port, port_num);
836
837 return 0;
838 }
839
set_ecn_ce_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec,int ipv)840 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num,
841 struct mlx5_flow_spec *spec, int ipv)
842 {
843 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
844 ft_field_support.outer_ip_version))
845 return -EOPNOTSUPP;
846
847 if (mlx5_core_mp_enabled(dev->mdev) &&
848 set_vhca_port_spec(dev, port_num, spec))
849 return -EOPNOTSUPP;
850
851 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
852 outer_headers.ip_ecn);
853 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn,
854 INET_ECN_CE);
855 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
856 outer_headers.ip_version);
857 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
858 ipv);
859
860 spec->match_criteria_enable =
861 get_match_criteria_enable(spec->match_criteria);
862
863 return 0;
864 }
865
set_cnp_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec)866 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num,
867 struct mlx5_flow_spec *spec)
868 {
869 if (mlx5_core_mp_enabled(dev->mdev) &&
870 set_vhca_port_spec(dev, port_num, spec))
871 return -EOPNOTSUPP;
872
873 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
874 misc_parameters.bth_opcode);
875 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode,
876 IB_BTH_OPCODE_CNP);
877
878 spec->match_criteria_enable =
879 get_match_criteria_enable(spec->match_criteria);
880
881 return 0;
882 }
883
884 /* Returns the prio we should use for the given optional counter type,
885 * whereas for bytes type we use the packet type, since they share the same
886 * resources.
887 */
get_opfc_prio(struct mlx5_ib_dev * dev,u32 type)888 static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev,
889 u32 type)
890 {
891 u32 prio_type;
892
893 switch (type) {
894 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
895 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;
896 break;
897 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
898 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;
899 break;
900 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
901 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
902 break;
903 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
904 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
905 break;
906 default:
907 prio_type = type;
908 }
909
910 return &dev->flow_db->opfcs[prio_type];
911 }
912
put_per_qp_prio(struct mlx5_ib_dev * dev,enum mlx5_ib_optional_counter_type type)913 static void put_per_qp_prio(struct mlx5_ib_dev *dev,
914 enum mlx5_ib_optional_counter_type type)
915 {
916 enum mlx5_ib_optional_counter_type per_qp_type;
917 struct mlx5_ib_flow_prio *prio;
918
919 switch (type) {
920 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
921 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
922 break;
923 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
924 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
925 break;
926 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
927 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
928 break;
929 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
930 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
931 break;
932 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
933 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
934 break;
935 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
936 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
937 break;
938 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
939 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
940 break;
941 default:
942 return;
943 }
944
945 prio = get_opfc_prio(dev, per_qp_type);
946 put_flow_table(dev, prio, true);
947 }
948
get_per_qp_prio(struct mlx5_ib_dev * dev,enum mlx5_ib_optional_counter_type type)949 static int get_per_qp_prio(struct mlx5_ib_dev *dev,
950 enum mlx5_ib_optional_counter_type type)
951 {
952 enum mlx5_ib_optional_counter_type per_qp_type;
953 enum mlx5_flow_namespace_type fn_type;
954 struct mlx5_flow_namespace *ns;
955 struct mlx5_ib_flow_prio *prio;
956 int priority;
957
958 switch (type) {
959 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
960 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
961 priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO;
962 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
963 break;
964 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
965 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
966 priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO;
967 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
968 break;
969 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
970 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
971 priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO;
972 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
973 break;
974 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
975 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
976 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
977 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
978 break;
979 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
980 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
981 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
982 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
983 break;
984 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
985 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
986 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
987 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
988 break;
989 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
990 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
991 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
992 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
993 break;
994 default:
995 return -EINVAL;
996 }
997
998 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
999 if (!ns)
1000 return -EOPNOTSUPP;
1001
1002 prio = get_opfc_prio(dev, per_qp_type);
1003 if (prio->flow_table)
1004 return 0;
1005
1006 prio = _get_prio(dev, ns, prio, priority, MLX5_FS_MAX_POOL_SIZE, 1, 0, 0);
1007 if (IS_ERR(prio))
1008 return PTR_ERR(prio);
1009
1010 prio->refcount = 1;
1011
1012 return 0;
1013 }
1014
1015 static struct mlx5_per_qp_opfc *
get_per_qp_opfc(struct mlx5_rdma_counter * mcounter,u32 qp_num,bool * new)1016 get_per_qp_opfc(struct mlx5_rdma_counter *mcounter, u32 qp_num, bool *new)
1017 {
1018 struct mlx5_per_qp_opfc *per_qp_opfc;
1019
1020 *new = false;
1021
1022 per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp_num);
1023 if (per_qp_opfc)
1024 return per_qp_opfc;
1025 per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL);
1026
1027 if (!per_qp_opfc)
1028 return NULL;
1029
1030 *new = true;
1031 return per_qp_opfc;
1032 }
1033
add_op_fc_rules(struct mlx5_ib_dev * dev,struct mlx5_rdma_counter * mcounter,struct mlx5_per_qp_opfc * per_qp_opfc,struct mlx5_ib_flow_prio * prio,enum mlx5_ib_optional_counter_type type,u32 qp_num,u32 port_num)1034 static int add_op_fc_rules(struct mlx5_ib_dev *dev,
1035 struct mlx5_rdma_counter *mcounter,
1036 struct mlx5_per_qp_opfc *per_qp_opfc,
1037 struct mlx5_ib_flow_prio *prio,
1038 enum mlx5_ib_optional_counter_type type,
1039 u32 qp_num, u32 port_num)
1040 {
1041 struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc;
1042 struct mlx5_flow_act flow_act = {};
1043 struct mlx5_flow_destination dst;
1044 struct mlx5_flow_spec *spec;
1045 int i, err, spec_num;
1046 bool is_tx;
1047
1048 if (opfc->fc)
1049 return -EEXIST;
1050
1051 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type,
1052 &in_use_opfc)) {
1053 opfc->fc = in_use_opfc->fc;
1054 opfc->rule[0] = in_use_opfc->rule[0];
1055 return 0;
1056 }
1057
1058 opfc->fc = mcounter->fc[type];
1059
1060 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1061 if (!spec) {
1062 err = -ENOMEM;
1063 goto null_fc;
1064 }
1065
1066 switch (type) {
1067 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP:
1068 if (set_ecn_ce_spec(dev, port_num, &spec[0],
1069 MLX5_FS_IPV4_VERSION) ||
1070 set_ecn_ce_spec(dev, port_num, &spec[1],
1071 MLX5_FS_IPV6_VERSION)) {
1072 err = -EOPNOTSUPP;
1073 goto free_spec;
1074 }
1075 spec_num = 2;
1076 is_tx = false;
1077
1078 MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria,
1079 misc_parameters.bth_dst_qp);
1080 MLX5_SET(fte_match_param, spec[1].match_value,
1081 misc_parameters.bth_dst_qp, qp_num);
1082 spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1083 break;
1084 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP:
1085 if (!MLX5_CAP_FLOWTABLE(
1086 dev->mdev,
1087 ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1088 set_cnp_spec(dev, port_num, &spec[0])) {
1089 err = -EOPNOTSUPP;
1090 goto free_spec;
1091 }
1092 spec_num = 1;
1093 is_tx = false;
1094 break;
1095 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP:
1096 if (!MLX5_CAP_FLOWTABLE(
1097 dev->mdev,
1098 ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1099 set_cnp_spec(dev, port_num, &spec[0])) {
1100 err = -EOPNOTSUPP;
1101 goto free_spec;
1102 }
1103 spec_num = 1;
1104 is_tx = true;
1105 break;
1106 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1107 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1108 spec_num = 1;
1109 is_tx = true;
1110 break;
1111 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1112 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1113 spec_num = 1;
1114 is_tx = false;
1115 break;
1116 default:
1117 err = -EINVAL;
1118 goto free_spec;
1119 }
1120
1121 if (is_tx) {
1122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1123 misc_parameters.source_sqn);
1124 MLX5_SET(fte_match_param, spec->match_value,
1125 misc_parameters.source_sqn, qp_num);
1126 } else {
1127 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1128 misc_parameters.bth_dst_qp);
1129 MLX5_SET(fte_match_param, spec->match_value,
1130 misc_parameters.bth_dst_qp, qp_num);
1131 }
1132
1133 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1134
1135 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1136 dst.counter = opfc->fc;
1137
1138 flow_act.action =
1139 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1140
1141 for (i = 0; i < spec_num; i++) {
1142 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1143 &flow_act, &dst, 1);
1144 if (IS_ERR(opfc->rule[i])) {
1145 err = PTR_ERR(opfc->rule[i]);
1146 goto del_rules;
1147 }
1148 }
1149 prio->refcount += spec_num;
1150
1151 err = xa_err(xa_store(&mcounter->qpn_opfc_xa, qp_num, per_qp_opfc,
1152 GFP_KERNEL));
1153 if (err)
1154 goto del_rules;
1155
1156 kfree(spec);
1157
1158 return 0;
1159
1160 del_rules:
1161 while (i--)
1162 mlx5_del_flow_rules(opfc->rule[i]);
1163 put_flow_table(dev, prio, false);
1164 free_spec:
1165 kfree(spec);
1166 null_fc:
1167 opfc->fc = NULL;
1168 return err;
1169 }
1170
is_fc_shared_and_in_use(struct mlx5_rdma_counter * mcounter,u32 type,struct mlx5_fc ** fc)1171 static bool is_fc_shared_and_in_use(struct mlx5_rdma_counter *mcounter,
1172 u32 type, struct mlx5_fc **fc)
1173 {
1174 u32 shared_fc_type;
1175
1176 switch (type) {
1177 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1178 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
1179 break;
1180 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1181 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
1182 break;
1183 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1184 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
1185 break;
1186 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1187 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
1188 break;
1189 default:
1190 return false;
1191 }
1192
1193 *fc = mcounter->fc[shared_fc_type];
1194 if (!(*fc))
1195 return false;
1196
1197 return true;
1198 }
1199
mlx5r_fs_destroy_fcs(struct mlx5_ib_dev * dev,struct rdma_counter * counter)1200 void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,
1201 struct rdma_counter *counter)
1202 {
1203 struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
1204 struct mlx5_fc *in_use_fc;
1205 int i;
1206
1207 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1208 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1209 if (!mcounter->fc[i])
1210 continue;
1211
1212 if (is_fc_shared_and_in_use(mcounter, i, &in_use_fc)) {
1213 mcounter->fc[i] = NULL;
1214 continue;
1215 }
1216
1217 mlx5_fc_destroy(dev->mdev, mcounter->fc[i]);
1218 mcounter->fc[i] = NULL;
1219 }
1220 }
1221
mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_ib_op_fc * opfc,enum mlx5_ib_optional_counter_type type)1222 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,
1223 struct mlx5_ib_op_fc *opfc,
1224 enum mlx5_ib_optional_counter_type type)
1225 {
1226 enum mlx5_flow_namespace_type fn_type;
1227 int priority, i, err, spec_num;
1228 struct mlx5_flow_act flow_act = {};
1229 struct mlx5_flow_destination dst;
1230 struct mlx5_flow_namespace *ns;
1231 struct mlx5_ib_flow_prio *prio;
1232 struct mlx5_flow_spec *spec;
1233
1234 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1235 if (!spec)
1236 return -ENOMEM;
1237
1238 switch (type) {
1239 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
1240 if (set_ecn_ce_spec(dev, port_num, &spec[0],
1241 MLX5_FS_IPV4_VERSION) ||
1242 set_ecn_ce_spec(dev, port_num, &spec[1],
1243 MLX5_FS_IPV6_VERSION)) {
1244 err = -EOPNOTSUPP;
1245 goto free;
1246 }
1247 spec_num = 2;
1248 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1249 priority = RDMA_RX_ECN_OPCOUNTER_PRIO;
1250 break;
1251
1252 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
1253 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1254 ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1255 set_cnp_spec(dev, port_num, &spec[0])) {
1256 err = -EOPNOTSUPP;
1257 goto free;
1258 }
1259 spec_num = 1;
1260 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1261 priority = RDMA_RX_CNP_OPCOUNTER_PRIO;
1262 break;
1263
1264 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
1265 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1266 ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1267 set_cnp_spec(dev, port_num, &spec[0])) {
1268 err = -EOPNOTSUPP;
1269 goto free;
1270 }
1271 spec_num = 1;
1272 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1273 priority = RDMA_TX_CNP_OPCOUNTER_PRIO;
1274 break;
1275
1276 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
1277 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
1278 spec_num = 1;
1279 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1280 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO;
1281 break;
1282
1283 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
1284 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
1285 spec_num = 1;
1286 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1287 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO;
1288 break;
1289
1290 default:
1291 err = -EOPNOTSUPP;
1292 goto free;
1293 }
1294
1295 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
1296 if (!ns) {
1297 err = -EOPNOTSUPP;
1298 goto free;
1299 }
1300
1301 prio = get_opfc_prio(dev, type);
1302 if (!prio->flow_table) {
1303 err = get_per_qp_prio(dev, type);
1304 if (err)
1305 goto free;
1306
1307 prio = _get_prio(dev, ns, prio, priority,
1308 dev->num_ports * MAX_OPFC_RULES, 1, 0, 0);
1309 if (IS_ERR(prio)) {
1310 err = PTR_ERR(prio);
1311 goto put_prio;
1312 }
1313 }
1314
1315 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1316 dst.counter = opfc->fc;
1317
1318 flow_act.action =
1319 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1320
1321 for (i = 0; i < spec_num; i++) {
1322 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1323 &flow_act, &dst, 1);
1324 if (IS_ERR(opfc->rule[i])) {
1325 err = PTR_ERR(opfc->rule[i]);
1326 goto del_rules;
1327 }
1328 }
1329 prio->refcount += spec_num;
1330 kfree(spec);
1331
1332 return 0;
1333
1334 del_rules:
1335 for (i -= 1; i >= 0; i--)
1336 mlx5_del_flow_rules(opfc->rule[i]);
1337 put_flow_table(dev, prio, false);
1338 put_prio:
1339 put_per_qp_prio(dev, type);
1340 free:
1341 kfree(spec);
1342 return err;
1343 }
1344
mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev * dev,struct mlx5_ib_op_fc * opfc,enum mlx5_ib_optional_counter_type type)1345 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
1346 struct mlx5_ib_op_fc *opfc,
1347 enum mlx5_ib_optional_counter_type type)
1348 {
1349 struct mlx5_ib_flow_prio *prio;
1350 int i;
1351
1352 prio = get_opfc_prio(dev, type);
1353
1354 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) {
1355 mlx5_del_flow_rules(opfc->rule[i]);
1356 put_flow_table(dev, prio, true);
1357 }
1358
1359 put_per_qp_prio(dev, type);
1360 }
1361
mlx5r_fs_unbind_op_fc(struct ib_qp * qp,struct rdma_counter * counter)1362 void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct rdma_counter *counter)
1363 {
1364 struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
1365 struct mlx5_ib_dev *dev = to_mdev(counter->device);
1366 struct mlx5_per_qp_opfc *per_qp_opfc;
1367 struct mlx5_ib_op_fc *in_use_opfc;
1368 struct mlx5_ib_flow_prio *prio;
1369 int i, j;
1370
1371 per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp->qp_num);
1372 if (!per_qp_opfc)
1373 return;
1374
1375 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1376 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1377 if (!per_qp_opfc->opfcs[i].fc)
1378 continue;
1379
1380 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i,
1381 &in_use_opfc)) {
1382 per_qp_opfc->opfcs[i].fc = NULL;
1383 continue;
1384 }
1385
1386 for (j = 0; j < MAX_OPFC_RULES; j++) {
1387 if (!per_qp_opfc->opfcs[i].rule[j])
1388 continue;
1389 mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]);
1390 prio = get_opfc_prio(dev, i);
1391 put_flow_table(dev, prio, true);
1392 }
1393 per_qp_opfc->opfcs[i].fc = NULL;
1394 }
1395
1396 kfree(per_qp_opfc);
1397 xa_erase(&mcounter->qpn_opfc_xa, qp->qp_num);
1398 }
1399
mlx5r_fs_bind_op_fc(struct ib_qp * qp,struct rdma_counter * counter,u32 port)1400 int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter,
1401 u32 port)
1402 {
1403 struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
1404 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1405 struct mlx5_per_qp_opfc *per_qp_opfc;
1406 struct mlx5_ib_flow_prio *prio;
1407 struct mlx5_ib_counters *cnts;
1408 struct mlx5_ib_op_fc *opfc;
1409 struct mlx5_fc *in_use_fc;
1410 int i, err, per_qp_type;
1411 bool new;
1412
1413 if (!counter->mode.bind_opcnt)
1414 return 0;
1415
1416 cnts = &dev->port[port - 1].cnts;
1417
1418 for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) {
1419 opfc = &cnts->opfcs[i];
1420 if (!opfc->fc)
1421 continue;
1422
1423 per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1424 prio = get_opfc_prio(dev, per_qp_type);
1425 WARN_ON(!prio->flow_table);
1426
1427 if (is_fc_shared_and_in_use(mcounter, per_qp_type, &in_use_fc))
1428 mcounter->fc[per_qp_type] = in_use_fc;
1429
1430 if (!mcounter->fc[per_qp_type]) {
1431 mcounter->fc[per_qp_type] = mlx5_fc_create(dev->mdev,
1432 false);
1433 if (IS_ERR(mcounter->fc[per_qp_type]))
1434 return PTR_ERR(mcounter->fc[per_qp_type]);
1435 }
1436
1437 per_qp_opfc = get_per_qp_opfc(mcounter, qp->qp_num, &new);
1438 if (!per_qp_opfc) {
1439 err = -ENOMEM;
1440 goto free_fc;
1441 }
1442 err = add_op_fc_rules(dev, mcounter, per_qp_opfc, prio,
1443 per_qp_type, qp->qp_num, port);
1444 if (err)
1445 goto del_rules;
1446 }
1447
1448 return 0;
1449
1450 del_rules:
1451 mlx5r_fs_unbind_op_fc(qp, counter);
1452 if (new)
1453 kfree(per_qp_opfc);
1454 free_fc:
1455 if (xa_empty(&mcounter->qpn_opfc_xa))
1456 mlx5r_fs_destroy_fcs(dev, counter);
1457 return err;
1458 }
1459
set_underlay_qp(struct mlx5_ib_dev * dev,struct mlx5_flow_spec * spec,u32 underlay_qpn)1460 static void set_underlay_qp(struct mlx5_ib_dev *dev,
1461 struct mlx5_flow_spec *spec,
1462 u32 underlay_qpn)
1463 {
1464 void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
1465 spec->match_criteria,
1466 misc_parameters);
1467 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1468 misc_parameters);
1469
1470 if (underlay_qpn &&
1471 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1472 ft_field_support.bth_dst_qp)) {
1473 MLX5_SET(fte_match_set_misc,
1474 misc_params_v, bth_dst_qp, underlay_qpn);
1475 MLX5_SET(fte_match_set_misc,
1476 misc_params_c, bth_dst_qp, 0xffffff);
1477 }
1478 }
1479
mlx5_ib_set_rule_source_port(struct mlx5_ib_dev * dev,struct mlx5_flow_spec * spec,struct mlx5_eswitch_rep * rep)1480 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
1481 struct mlx5_flow_spec *spec,
1482 struct mlx5_eswitch_rep *rep)
1483 {
1484 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
1485 void *misc;
1486
1487 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1488 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1489 misc_parameters_2);
1490
1491 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1492 mlx5_eswitch_get_vport_metadata_for_match(rep->esw,
1493 rep->vport));
1494 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1495 misc_parameters_2);
1496
1497 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1498 mlx5_eswitch_get_vport_metadata_mask());
1499 } else {
1500 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1501 misc_parameters);
1502
1503 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
1504
1505 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1506 misc_parameters);
1507
1508 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1509 }
1510 }
1511
_create_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,const struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst,u32 underlay_qpn,struct mlx5_ib_create_flow * ucmd)1512 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
1513 struct mlx5_ib_flow_prio *ft_prio,
1514 const struct ib_flow_attr *flow_attr,
1515 struct mlx5_flow_destination *dst,
1516 u32 underlay_qpn,
1517 struct mlx5_ib_create_flow *ucmd)
1518 {
1519 struct mlx5_flow_table *ft = ft_prio->flow_table;
1520 struct mlx5_ib_flow_handler *handler;
1521 struct mlx5_flow_act flow_act = {};
1522 struct mlx5_flow_spec *spec;
1523 struct mlx5_flow_destination dest_arr[2] = {};
1524 struct mlx5_flow_destination *rule_dst = dest_arr;
1525 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
1526 unsigned int spec_index;
1527 u32 prev_type = 0;
1528 int err = 0;
1529 int dest_num = 0;
1530 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1531
1532 if (!is_valid_attr(dev->mdev, flow_attr))
1533 return ERR_PTR(-EINVAL);
1534
1535 if (dev->is_rep && is_egress)
1536 return ERR_PTR(-EINVAL);
1537
1538 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1539 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
1540 if (!handler || !spec) {
1541 err = -ENOMEM;
1542 goto free;
1543 }
1544
1545 INIT_LIST_HEAD(&handler->list);
1546
1547 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
1548 err = parse_flow_attr(dev->mdev, spec,
1549 ib_flow, flow_attr, &flow_act,
1550 prev_type);
1551 if (err < 0)
1552 goto free;
1553
1554 prev_type = ((union ib_flow_spec *)ib_flow)->type;
1555 ib_flow += ((union ib_flow_spec *)ib_flow)->size;
1556 }
1557
1558 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
1559 memcpy(&dest_arr[0], dst, sizeof(*dst));
1560 dest_num++;
1561 }
1562
1563 if (!flow_is_multicast_only(flow_attr))
1564 set_underlay_qp(dev, spec, underlay_qpn);
1565
1566 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) {
1567 struct mlx5_eswitch_rep *rep;
1568
1569 rep = dev->port[flow_attr->port - 1].rep;
1570 if (!rep) {
1571 err = -EINVAL;
1572 goto free;
1573 }
1574
1575 mlx5_ib_set_rule_source_port(dev, spec, rep);
1576 }
1577
1578 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
1579
1580 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1581 struct mlx5_ib_mcounters *mcounters;
1582
1583 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);
1584 if (err)
1585 goto free;
1586
1587 mcounters = to_mcounters(flow_act.counters);
1588 handler->ibcounters = flow_act.counters;
1589 dest_arr[dest_num].type =
1590 MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1591 dest_arr[dest_num].counter =
1592 mcounters->hw_cntrs_hndl;
1593 dest_num++;
1594 }
1595
1596 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
1597 if (!dest_num)
1598 rule_dst = NULL;
1599 } else {
1600 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
1601 flow_act.action |=
1602 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1603 if (is_egress)
1604 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1605 else if (dest_num)
1606 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1607 }
1608
1609 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
1610 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1611 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1612 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
1613 spec->flow_context.flow_tag, flow_attr->type);
1614 err = -EINVAL;
1615 goto free;
1616 }
1617 handler->rule = mlx5_add_flow_rules(ft, spec,
1618 &flow_act,
1619 rule_dst, dest_num);
1620
1621 if (IS_ERR(handler->rule)) {
1622 err = PTR_ERR(handler->rule);
1623 goto free;
1624 }
1625
1626 ft_prio->refcount++;
1627 handler->prio = ft_prio;
1628 handler->dev = dev;
1629
1630 ft_prio->flow_table = ft;
1631 free:
1632 if (err && handler) {
1633 mlx5_ib_counters_clear_description(handler->ibcounters);
1634 kfree(handler);
1635 }
1636 kvfree(spec);
1637 return err ? ERR_PTR(err) : handler;
1638 }
1639
create_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,const struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)1640 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
1641 struct mlx5_ib_flow_prio *ft_prio,
1642 const struct ib_flow_attr *flow_attr,
1643 struct mlx5_flow_destination *dst)
1644 {
1645 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
1646 }
1647
create_leftovers_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)1648 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
1649 struct mlx5_ib_flow_prio *ft_prio,
1650 struct ib_flow_attr *flow_attr,
1651 struct mlx5_flow_destination *dst)
1652 {
1653 struct mlx5_ib_flow_handler *handler_ucast = NULL;
1654 struct mlx5_ib_flow_handler *handler = NULL;
1655
1656 static struct {
1657 struct ib_flow_spec_eth eth_flow;
1658 struct ib_flow_attr flow_attr;
1659 } leftovers_wc = { .flow_attr = { .num_of_specs = 1,
1660 .size = sizeof(leftovers_wc) },
1661 .eth_flow = {
1662 .type = IB_FLOW_SPEC_ETH,
1663 .size = sizeof(struct ib_flow_spec_eth),
1664 .mask = { .dst_mac = { 0x1 } },
1665 .val = { .dst_mac = { 0x1 } } } };
1666
1667 static struct {
1668 struct ib_flow_spec_eth eth_flow;
1669 struct ib_flow_attr flow_attr;
1670 } leftovers_uc = { .flow_attr = { .num_of_specs = 1,
1671 .size = sizeof(leftovers_uc) },
1672 .eth_flow = {
1673 .type = IB_FLOW_SPEC_ETH,
1674 .size = sizeof(struct ib_flow_spec_eth),
1675 .mask = { .dst_mac = { 0x1 } },
1676 .val = { .dst_mac = {} } } };
1677
1678 handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst);
1679 if (!IS_ERR(handler) &&
1680 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
1681 handler_ucast = create_flow_rule(dev, ft_prio,
1682 &leftovers_uc.flow_attr, dst);
1683 if (IS_ERR(handler_ucast)) {
1684 mlx5_del_flow_rules(handler->rule);
1685 ft_prio->refcount--;
1686 kfree(handler);
1687 handler = handler_ucast;
1688 } else {
1689 list_add(&handler_ucast->list, &handler->list);
1690 }
1691 }
1692
1693 return handler;
1694 }
1695
create_sniffer_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_rx,struct mlx5_ib_flow_prio * ft_tx,struct mlx5_flow_destination * dst)1696 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
1697 struct mlx5_ib_flow_prio *ft_rx,
1698 struct mlx5_ib_flow_prio *ft_tx,
1699 struct mlx5_flow_destination *dst)
1700 {
1701 struct mlx5_ib_flow_handler *handler_rx;
1702 struct mlx5_ib_flow_handler *handler_tx;
1703 int err;
1704 static const struct ib_flow_attr flow_attr = {
1705 .num_of_specs = 0,
1706 .type = IB_FLOW_ATTR_SNIFFER,
1707 .size = sizeof(flow_attr)
1708 };
1709
1710 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
1711 if (IS_ERR(handler_rx)) {
1712 err = PTR_ERR(handler_rx);
1713 goto err;
1714 }
1715
1716 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
1717 if (IS_ERR(handler_tx)) {
1718 err = PTR_ERR(handler_tx);
1719 goto err_tx;
1720 }
1721
1722 list_add(&handler_tx->list, &handler_rx->list);
1723
1724 return handler_rx;
1725
1726 err_tx:
1727 mlx5_del_flow_rules(handler_rx->rule);
1728 ft_rx->refcount--;
1729 kfree(handler_rx);
1730 err:
1731 return ERR_PTR(err);
1732 }
1733
mlx5_ib_create_flow(struct ib_qp * qp,struct ib_flow_attr * flow_attr,struct ib_udata * udata)1734 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
1735 struct ib_flow_attr *flow_attr,
1736 struct ib_udata *udata)
1737 {
1738 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1739 struct mlx5_ib_qp *mqp = to_mqp(qp);
1740 struct mlx5_ib_flow_handler *handler = NULL;
1741 struct mlx5_flow_destination *dst = NULL;
1742 struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
1743 struct mlx5_ib_flow_prio *ft_prio;
1744 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1745 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
1746 size_t min_ucmd_sz, required_ucmd_sz;
1747 int err;
1748 int underlay_qpn;
1749
1750 if (udata && udata->inlen) {
1751 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);
1752 if (udata->inlen < min_ucmd_sz)
1753 return ERR_PTR(-EOPNOTSUPP);
1754
1755 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
1756 if (err)
1757 return ERR_PTR(err);
1758
1759 /* currently supports only one counters data */
1760 if (ucmd_hdr.ncounters_data > 1)
1761 return ERR_PTR(-EINVAL);
1762
1763 required_ucmd_sz = min_ucmd_sz +
1764 sizeof(struct mlx5_ib_flow_counters_data) *
1765 ucmd_hdr.ncounters_data;
1766 if (udata->inlen > required_ucmd_sz &&
1767 !ib_is_udata_cleared(udata, required_ucmd_sz,
1768 udata->inlen - required_ucmd_sz))
1769 return ERR_PTR(-EOPNOTSUPP);
1770
1771 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
1772 if (!ucmd)
1773 return ERR_PTR(-ENOMEM);
1774
1775 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
1776 if (err)
1777 goto free_ucmd;
1778 }
1779
1780 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
1781 err = -ENOMEM;
1782 goto free_ucmd;
1783 }
1784
1785 if (flow_attr->flags &
1786 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) {
1787 err = -EINVAL;
1788 goto free_ucmd;
1789 }
1790
1791 if (is_egress &&
1792 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1793 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1794 err = -EINVAL;
1795 goto free_ucmd;
1796 }
1797
1798 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1799 if (!dst) {
1800 err = -ENOMEM;
1801 goto free_ucmd;
1802 }
1803
1804 mutex_lock(&dev->flow_db->lock);
1805
1806 ft_prio = get_flow_table(dev, flow_attr,
1807 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
1808 if (IS_ERR(ft_prio)) {
1809 err = PTR_ERR(ft_prio);
1810 goto unlock;
1811 }
1812 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
1813 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
1814 if (IS_ERR(ft_prio_tx)) {
1815 err = PTR_ERR(ft_prio_tx);
1816 ft_prio_tx = NULL;
1817 goto destroy_ft;
1818 }
1819 }
1820
1821 if (is_egress) {
1822 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1823 } else {
1824 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1825 if (mqp->is_rss)
1826 dst->tir_num = mqp->rss_qp.tirn;
1827 else
1828 dst->tir_num = mqp->raw_packet_qp.rq.tirn;
1829 }
1830
1831 switch (flow_attr->type) {
1832 case IB_FLOW_ATTR_NORMAL:
1833 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
1834 mqp->underlay_qpn :
1835 0;
1836 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
1837 underlay_qpn, ucmd);
1838 break;
1839 case IB_FLOW_ATTR_ALL_DEFAULT:
1840 case IB_FLOW_ATTR_MC_DEFAULT:
1841 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);
1842 break;
1843 case IB_FLOW_ATTR_SNIFFER:
1844 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
1845 break;
1846 default:
1847 err = -EINVAL;
1848 goto destroy_ft;
1849 }
1850
1851 if (IS_ERR(handler)) {
1852 err = PTR_ERR(handler);
1853 handler = NULL;
1854 goto destroy_ft;
1855 }
1856
1857 mutex_unlock(&dev->flow_db->lock);
1858 kfree(dst);
1859 kfree(ucmd);
1860
1861 return &handler->ibflow;
1862
1863 destroy_ft:
1864 put_flow_table(dev, ft_prio, false);
1865 if (ft_prio_tx)
1866 put_flow_table(dev, ft_prio_tx, false);
1867 unlock:
1868 mutex_unlock(&dev->flow_db->lock);
1869 kfree(dst);
1870 free_ucmd:
1871 kfree(ucmd);
1872 return ERR_PTR(err);
1873 }
1874
mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev * dev,enum mlx5_flow_namespace_type type,u32 * flags,u16 * vport_idx,u16 * vport,struct mlx5_core_dev ** ft_mdev,u32 ib_port)1875 static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev,
1876 enum mlx5_flow_namespace_type type,
1877 u32 *flags, u16 *vport_idx,
1878 u16 *vport,
1879 struct mlx5_core_dev **ft_mdev,
1880 u32 ib_port)
1881 {
1882 struct mlx5_core_dev *esw_mdev;
1883
1884 if (!is_mdev_switchdev_mode(dev->mdev))
1885 return 0;
1886
1887 if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager))
1888 return -EOPNOTSUPP;
1889
1890 if (!dev->port[ib_port - 1].rep)
1891 return -EINVAL;
1892
1893 esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw);
1894 if (esw_mdev != dev->mdev)
1895 return -EOPNOTSUPP;
1896
1897 *flags |= MLX5_FLOW_TABLE_OTHER_VPORT;
1898 *ft_mdev = esw_mdev;
1899 *vport = dev->port[ib_port - 1].rep->vport;
1900 *vport_idx = dev->port[ib_port - 1].rep->vport_index;
1901
1902 return 0;
1903 }
1904
1905 static struct mlx5_ib_flow_prio *
_get_flow_table(struct mlx5_ib_dev * dev,u16 user_priority,enum mlx5_flow_namespace_type ns_type,bool mcast,u32 ib_port)1906 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
1907 enum mlx5_flow_namespace_type ns_type,
1908 bool mcast, u32 ib_port)
1909 {
1910 struct mlx5_core_dev *ft_mdev = dev->mdev;
1911 struct mlx5_flow_namespace *ns = NULL;
1912 struct mlx5_ib_flow_prio *prio = NULL;
1913 int max_table_size = 0;
1914 u16 vport_idx = 0;
1915 bool esw_encap;
1916 u32 flags = 0;
1917 u16 vport = 0;
1918 int priority;
1919 int ret;
1920
1921 if (mcast)
1922 priority = MLX5_IB_FLOW_MCAST_PRIO;
1923 else
1924 priority = ib_prio_to_core_prio(user_priority, false);
1925
1926 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
1927 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
1928 switch (ns_type) {
1929 case MLX5_FLOW_NAMESPACE_BYPASS:
1930 max_table_size = BIT(
1931 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size));
1932 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
1933 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1934 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1935 reformat_l3_tunnel_to_l2) &&
1936 !esw_encap)
1937 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1938 break;
1939 case MLX5_FLOW_NAMESPACE_EGRESS:
1940 max_table_size = BIT(
1941 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
1942 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) &&
1943 !esw_encap)
1944 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1945 break;
1946 case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
1947 max_table_size = BIT(
1948 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
1949 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
1950 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1951 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev,
1952 reformat_l3_tunnel_to_l2) &&
1953 esw_encap)
1954 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1955 priority = user_priority;
1956 break;
1957 case MLX5_FLOW_NAMESPACE_RDMA_RX:
1958 max_table_size = BIT(
1959 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size));
1960 priority = user_priority;
1961 break;
1962 case MLX5_FLOW_NAMESPACE_RDMA_TX:
1963 max_table_size = BIT(
1964 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size));
1965 priority = user_priority;
1966 break;
1967 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
1968 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
1969 if (ib_port == 0 || user_priority > MLX5_RDMA_TRANSPORT_BYPASS_PRIO)
1970 return ERR_PTR(-EINVAL);
1971 ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags,
1972 &vport_idx, &vport,
1973 &ft_mdev, ib_port);
1974 if (ret)
1975 return ERR_PTR(ret);
1976
1977 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX)
1978 max_table_size =
1979 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(
1980 ft_mdev, log_max_ft_size));
1981 else
1982 max_table_size =
1983 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(
1984 ft_mdev, log_max_ft_size));
1985 priority = user_priority;
1986 break;
1987 default:
1988 break;
1989 }
1990
1991 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
1992
1993 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX ||
1994 ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)
1995 ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx);
1996 else
1997 ns = mlx5_get_flow_namespace(ft_mdev, ns_type);
1998
1999 if (!ns)
2000 return ERR_PTR(-EOPNOTSUPP);
2001
2002 switch (ns_type) {
2003 case MLX5_FLOW_NAMESPACE_BYPASS:
2004 prio = &dev->flow_db->prios[priority];
2005 break;
2006 case MLX5_FLOW_NAMESPACE_EGRESS:
2007 prio = &dev->flow_db->egress_prios[priority];
2008 break;
2009 case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
2010 prio = &dev->flow_db->fdb[priority];
2011 break;
2012 case MLX5_FLOW_NAMESPACE_RDMA_RX:
2013 prio = &dev->flow_db->rdma_rx[priority];
2014 break;
2015 case MLX5_FLOW_NAMESPACE_RDMA_TX:
2016 prio = &dev->flow_db->rdma_tx[priority];
2017 break;
2018 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
2019 prio = &dev->flow_db->rdma_transport_rx[ib_port - 1];
2020 break;
2021 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
2022 prio = &dev->flow_db->rdma_transport_tx[ib_port - 1];
2023 break;
2024 default: return ERR_PTR(-EINVAL);
2025 }
2026
2027 if (!prio)
2028 return ERR_PTR(-EINVAL);
2029
2030 if (prio->flow_table)
2031 return prio;
2032
2033 return _get_prio(dev, ns, prio, priority, max_table_size,
2034 MLX5_FS_MAX_TYPES, flags, vport);
2035 }
2036
2037 static struct mlx5_ib_flow_handler *
_create_raw_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct mlx5_flow_destination * dst,struct mlx5_ib_flow_matcher * fs_matcher,struct mlx5_flow_context * flow_context,struct mlx5_flow_act * flow_act,void * cmd_in,int inlen,int dst_num)2038 _create_raw_flow_rule(struct mlx5_ib_dev *dev,
2039 struct mlx5_ib_flow_prio *ft_prio,
2040 struct mlx5_flow_destination *dst,
2041 struct mlx5_ib_flow_matcher *fs_matcher,
2042 struct mlx5_flow_context *flow_context,
2043 struct mlx5_flow_act *flow_act,
2044 void *cmd_in, int inlen,
2045 int dst_num)
2046 {
2047 struct mlx5_ib_flow_handler *handler;
2048 struct mlx5_flow_spec *spec;
2049 struct mlx5_flow_table *ft = ft_prio->flow_table;
2050 int err = 0;
2051
2052 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2053 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
2054 if (!handler || !spec) {
2055 err = -ENOMEM;
2056 goto free;
2057 }
2058
2059 INIT_LIST_HEAD(&handler->list);
2060
2061 memcpy(spec->match_value, cmd_in, inlen);
2062 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
2063 fs_matcher->mask_len);
2064 spec->match_criteria_enable = fs_matcher->match_criteria_enable;
2065 spec->flow_context = *flow_context;
2066
2067 handler->rule = mlx5_add_flow_rules(ft, spec,
2068 flow_act, dst, dst_num);
2069
2070 if (IS_ERR(handler->rule)) {
2071 err = PTR_ERR(handler->rule);
2072 goto free;
2073 }
2074
2075 ft_prio->refcount++;
2076 handler->prio = ft_prio;
2077 handler->dev = dev;
2078 ft_prio->flow_table = ft;
2079
2080 free:
2081 if (err)
2082 kfree(handler);
2083 kvfree(spec);
2084 return err ? ERR_PTR(err) : handler;
2085 }
2086
raw_fs_is_multicast(struct mlx5_ib_flow_matcher * fs_matcher,void * match_v)2087 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
2088 void *match_v)
2089 {
2090 void *match_c;
2091 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
2092 void *dmac, *dmac_mask;
2093 void *ipv4, *ipv4_mask;
2094
2095 if (!(fs_matcher->match_criteria_enable &
2096 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
2097 return false;
2098
2099 match_c = fs_matcher->matcher_mask.match_params;
2100 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
2101 outer_headers);
2102 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
2103 outer_headers);
2104
2105 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2106 dmac_47_16);
2107 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2108 dmac_47_16);
2109
2110 if (is_multicast_ether_addr(dmac) &&
2111 is_multicast_ether_addr(dmac_mask))
2112 return true;
2113
2114 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2115 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2116
2117 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2118 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2119
2120 if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
2121 ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
2122 return true;
2123
2124 return false;
2125 }
2126
raw_fs_rule_add(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_matcher * fs_matcher,struct mlx5_flow_context * flow_context,struct mlx5_flow_act * flow_act,struct mlx5_fc * counter,void * cmd_in,int inlen,int dest_id,int dest_type)2127 static struct mlx5_ib_flow_handler *raw_fs_rule_add(
2128 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
2129 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
2130 struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type)
2131 {
2132 struct mlx5_flow_destination *dst;
2133 struct mlx5_ib_flow_prio *ft_prio;
2134 struct mlx5_ib_flow_handler *handler;
2135 int dst_num = 0;
2136 bool mcast;
2137 int err;
2138
2139 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
2140 return ERR_PTR(-EOPNOTSUPP);
2141
2142 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
2143 return ERR_PTR(-ENOMEM);
2144
2145 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
2146 if (!dst)
2147 return ERR_PTR(-ENOMEM);
2148
2149 mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
2150 mutex_lock(&dev->flow_db->lock);
2151
2152 ft_prio = _get_flow_table(dev, fs_matcher->priority,
2153 fs_matcher->ns_type, mcast,
2154 fs_matcher->ib_port);
2155 if (IS_ERR(ft_prio)) {
2156 err = PTR_ERR(ft_prio);
2157 goto unlock;
2158 }
2159
2160 switch (dest_type) {
2161 case MLX5_FLOW_DESTINATION_TYPE_TIR:
2162 dst[dst_num].type = dest_type;
2163 dst[dst_num++].tir_num = dest_id;
2164 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2165 break;
2166 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
2167 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
2168 dst[dst_num++].ft_num = dest_id;
2169 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2170 break;
2171 case MLX5_FLOW_DESTINATION_TYPE_PORT:
2172 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2173 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
2174 break;
2175 default:
2176 break;
2177 }
2178
2179 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
2180 if (WARN_ON(!counter)) {
2181 err = -EINVAL;
2182 goto unlock;
2183 }
2184 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
2185 dst[dst_num].counter = counter;
2186 dst_num++;
2187 }
2188
2189 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL,
2190 fs_matcher, flow_context, flow_act,
2191 cmd_in, inlen, dst_num);
2192
2193 if (IS_ERR(handler)) {
2194 err = PTR_ERR(handler);
2195 goto destroy_ft;
2196 }
2197
2198 mutex_unlock(&dev->flow_db->lock);
2199 atomic_inc(&fs_matcher->usecnt);
2200 handler->flow_matcher = fs_matcher;
2201
2202 kfree(dst);
2203
2204 return handler;
2205
2206 destroy_ft:
2207 put_flow_table(dev, ft_prio, false);
2208 unlock:
2209 mutex_unlock(&dev->flow_db->lock);
2210 kfree(dst);
2211
2212 return ERR_PTR(err);
2213 }
2214
destroy_flow_action_raw(struct mlx5_ib_flow_action * maction)2215 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
2216 {
2217 switch (maction->flow_action_raw.sub_type) {
2218 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
2219 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
2220 maction->flow_action_raw.modify_hdr);
2221 break;
2222 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
2223 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
2224 maction->flow_action_raw.pkt_reformat);
2225 break;
2226 case MLX5_IB_FLOW_ACTION_DECAP:
2227 break;
2228 default:
2229 break;
2230 }
2231 }
2232
mlx5_ib_destroy_flow_action(struct ib_flow_action * action)2233 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
2234 {
2235 struct mlx5_ib_flow_action *maction = to_mflow_act(action);
2236
2237 switch (action->type) {
2238 case IB_FLOW_ACTION_UNSPECIFIED:
2239 destroy_flow_action_raw(maction);
2240 break;
2241 default:
2242 WARN_ON(true);
2243 break;
2244 }
2245
2246 kfree(maction);
2247 return 0;
2248 }
2249
2250 static int
mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,enum mlx5_flow_namespace_type * namespace)2251 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
2252 enum mlx5_flow_namespace_type *namespace)
2253 {
2254 switch (table_type) {
2255 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
2256 *namespace = MLX5_FLOW_NAMESPACE_BYPASS;
2257 break;
2258 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
2259 *namespace = MLX5_FLOW_NAMESPACE_EGRESS;
2260 break;
2261 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
2262 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS;
2263 break;
2264 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
2265 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
2266 break;
2267 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
2268 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
2269 break;
2270 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX:
2271 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX;
2272 break;
2273 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX:
2274 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX;
2275 break;
2276 default:
2277 return -EINVAL;
2278 }
2279
2280 return 0;
2281 }
2282
2283 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
2284 [MLX5_IB_FLOW_TYPE_NORMAL] = {
2285 .type = UVERBS_ATTR_TYPE_PTR_IN,
2286 .u.ptr = {
2287 .len = sizeof(u16), /* data is priority */
2288 .min_len = sizeof(u16),
2289 }
2290 },
2291 [MLX5_IB_FLOW_TYPE_SNIFFER] = {
2292 .type = UVERBS_ATTR_TYPE_PTR_IN,
2293 UVERBS_ATTR_NO_DATA(),
2294 },
2295 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
2296 .type = UVERBS_ATTR_TYPE_PTR_IN,
2297 UVERBS_ATTR_NO_DATA(),
2298 },
2299 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
2300 .type = UVERBS_ATTR_TYPE_PTR_IN,
2301 UVERBS_ATTR_NO_DATA(),
2302 },
2303 };
2304
is_flow_dest(void * obj,int * dest_id,int * dest_type)2305 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
2306 {
2307 struct devx_obj *devx_obj = obj;
2308 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2309
2310 switch (opcode) {
2311 case MLX5_CMD_OP_DESTROY_TIR:
2312 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2313 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
2314 obj_id);
2315 return true;
2316
2317 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
2318 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2319 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
2320 table_id);
2321 return true;
2322 default:
2323 return false;
2324 }
2325 }
2326
get_dests(struct uverbs_attr_bundle * attrs,struct mlx5_ib_flow_matcher * fs_matcher,int * dest_id,int * dest_type,struct ib_qp ** qp,u32 * flags)2327 static int get_dests(struct uverbs_attr_bundle *attrs,
2328 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
2329 int *dest_type, struct ib_qp **qp, u32 *flags)
2330 {
2331 bool dest_devx, dest_qp;
2332 void *devx_obj;
2333 int err;
2334
2335 dest_devx = uverbs_attr_is_valid(attrs,
2336 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2337 dest_qp = uverbs_attr_is_valid(attrs,
2338 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2339
2340 *flags = 0;
2341 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
2342 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
2343 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
2344 if (err)
2345 return err;
2346
2347 /* Both flags are not allowed */
2348 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
2349 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2350 return -EINVAL;
2351
2352 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
2353 if (dest_devx && (dest_qp || *flags))
2354 return -EINVAL;
2355 else if (dest_qp && *flags)
2356 return -EINVAL;
2357 }
2358
2359 /* Allow only DEVX object, drop as dest for FDB */
2360 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2361 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
2362 return -EINVAL;
2363
2364 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
2365 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2366 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2367 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
2368 return -EINVAL;
2369
2370 *qp = NULL;
2371 if (dest_devx) {
2372 devx_obj =
2373 uverbs_attr_get_obj(attrs,
2374 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2375
2376 /* Verify that the given DEVX object is a flow
2377 * steering destination.
2378 */
2379 if (!is_flow_dest(devx_obj, dest_id, dest_type))
2380 return -EINVAL;
2381 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */
2382 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS ||
2383 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2384 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2385 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
2386 return -EINVAL;
2387 } else if (dest_qp) {
2388 struct mlx5_ib_qp *mqp;
2389
2390 *qp = uverbs_attr_get_obj(attrs,
2391 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2392 if (IS_ERR(*qp))
2393 return PTR_ERR(*qp);
2394
2395 if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
2396 return -EINVAL;
2397
2398 mqp = to_mqp(*qp);
2399 if (mqp->is_rss)
2400 *dest_id = mqp->rss_qp.tirn;
2401 else
2402 *dest_id = mqp->raw_packet_qp.rq.tirn;
2403 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2404 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2405 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2406 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) &&
2407 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) {
2408 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2409 }
2410
2411 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
2412 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2413 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2414 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX))
2415 return -EINVAL;
2416
2417 return 0;
2418 }
2419
2420 static bool
is_flow_counter(void * obj,u32 offset,u32 * counter_id,u32 * fc_bulk_size)2421 is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size)
2422 {
2423 struct devx_obj *devx_obj = obj;
2424 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2425
2426 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
2427
2428 if (offset && offset >= devx_obj->flow_counter_bulk_size)
2429 return false;
2430
2431 *fc_bulk_size = devx_obj->flow_counter_bulk_size;
2432 *counter_id = MLX5_GET(dealloc_flow_counter_in,
2433 devx_obj->dinbox,
2434 flow_counter_id);
2435 *counter_id += offset;
2436 return true;
2437 }
2438
2439 return false;
2440 }
2441
2442 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)2443 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
2444 struct uverbs_attr_bundle *attrs)
2445 {
2446 struct mlx5_flow_context flow_context = {.flow_tag =
2447 MLX5_FS_DEFAULT_FLOW_TAG};
2448 int dest_id, dest_type = -1, inlen, len, ret, i;
2449 struct mlx5_ib_flow_handler *flow_handler;
2450 struct mlx5_ib_flow_matcher *fs_matcher;
2451 struct ib_uobject **arr_flow_actions;
2452 struct ib_uflow_resources *uflow_res;
2453 struct mlx5_flow_act flow_act = {};
2454 struct mlx5_fc *counter = NULL;
2455 struct ib_qp *qp = NULL;
2456 void *devx_obj, *cmd_in;
2457 struct ib_uobject *uobj;
2458 struct mlx5_ib_dev *dev;
2459 u32 flags;
2460
2461 if (!capable(CAP_NET_RAW))
2462 return -EPERM;
2463
2464 fs_matcher = uverbs_attr_get_obj(attrs,
2465 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
2466 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
2467 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2468
2469 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
2470 return -EINVAL;
2471
2472 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
2473 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
2474
2475 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2476 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
2477
2478 len = uverbs_attr_get_uobjs_arr(attrs,
2479 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
2480 if (len) {
2481 u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0;
2482 devx_obj = arr_flow_actions[0]->object;
2483
2484 if (uverbs_attr_is_valid(attrs,
2485 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
2486
2487 int num_offsets = uverbs_attr_ptr_get_array_size(
2488 attrs,
2489 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
2490 sizeof(u32));
2491
2492 if (num_offsets != 1)
2493 return -EINVAL;
2494
2495 offset_attr = uverbs_attr_get_alloced_ptr(
2496 attrs,
2497 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
2498 offset = *offset_attr;
2499 }
2500
2501 if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size))
2502 return -EINVAL;
2503 counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size);
2504 if (IS_ERR(counter))
2505 return PTR_ERR(counter);
2506
2507 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2508 }
2509
2510 cmd_in = uverbs_attr_get_alloced_ptr(
2511 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2512 inlen = uverbs_attr_get_len(attrs,
2513 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2514
2515 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
2516 if (!uflow_res) {
2517 ret = -ENOMEM;
2518 goto destroy_counter;
2519 }
2520
2521 len = uverbs_attr_get_uobjs_arr(attrs,
2522 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
2523 for (i = 0; i < len; i++) {
2524 struct mlx5_ib_flow_action *maction =
2525 to_mflow_act(arr_flow_actions[i]->object);
2526
2527 ret = parse_flow_flow_action(maction, false, &flow_act);
2528 if (ret)
2529 goto err_out;
2530 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
2531 arr_flow_actions[i]->object);
2532 }
2533
2534 ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
2535 MLX5_IB_ATTR_CREATE_FLOW_TAG);
2536 if (!ret) {
2537 if (flow_context.flow_tag >= BIT(24)) {
2538 ret = -EINVAL;
2539 goto err_out;
2540 }
2541 flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
2542 }
2543
2544 flow_handler =
2545 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act,
2546 counter, cmd_in, inlen, dest_id, dest_type);
2547 if (IS_ERR(flow_handler)) {
2548 ret = PTR_ERR(flow_handler);
2549 goto err_out;
2550 }
2551
2552 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
2553
2554 return 0;
2555 err_out:
2556 ib_uverbs_flow_resources_free(uflow_res);
2557 destroy_counter:
2558 if (counter)
2559 mlx5_fc_local_destroy(counter);
2560 return ret;
2561 }
2562
flow_matcher_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2563 static int flow_matcher_cleanup(struct ib_uobject *uobject,
2564 enum rdma_remove_reason why,
2565 struct uverbs_attr_bundle *attrs)
2566 {
2567 struct mlx5_ib_flow_matcher *obj = uobject->object;
2568
2569 if (atomic_read(&obj->usecnt))
2570 return -EBUSY;
2571
2572 kfree(obj);
2573 return 0;
2574 }
2575
steering_anchor_create_ft(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,enum mlx5_flow_namespace_type ns_type)2576 static int steering_anchor_create_ft(struct mlx5_ib_dev *dev,
2577 struct mlx5_ib_flow_prio *ft_prio,
2578 enum mlx5_flow_namespace_type ns_type)
2579 {
2580 struct mlx5_flow_table_attr ft_attr = {};
2581 struct mlx5_flow_namespace *ns;
2582 struct mlx5_flow_table *ft;
2583
2584 if (ft_prio->anchor.ft)
2585 return 0;
2586
2587 ns = mlx5_get_flow_namespace(dev->mdev, ns_type);
2588 if (!ns)
2589 return -EOPNOTSUPP;
2590
2591 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
2592 ft_attr.uid = MLX5_SHARED_RESOURCE_UID;
2593 ft_attr.prio = 0;
2594 ft_attr.max_fte = 2;
2595 ft_attr.level = 1;
2596
2597 ft = mlx5_create_flow_table(ns, &ft_attr);
2598 if (IS_ERR(ft))
2599 return PTR_ERR(ft);
2600
2601 ft_prio->anchor.ft = ft;
2602
2603 return 0;
2604 }
2605
steering_anchor_destroy_ft(struct mlx5_ib_flow_prio * ft_prio)2606 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)
2607 {
2608 if (ft_prio->anchor.ft) {
2609 mlx5_destroy_flow_table(ft_prio->anchor.ft);
2610 ft_prio->anchor.ft = NULL;
2611 }
2612 }
2613
2614 static int
steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio * ft_prio)2615 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2616 {
2617 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2618 struct mlx5_flow_group *fg;
2619 void *flow_group_in;
2620 int err = 0;
2621
2622 if (ft_prio->anchor.fg_drop)
2623 return 0;
2624
2625 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2626 if (!flow_group_in)
2627 return -ENOMEM;
2628
2629 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
2630 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
2631
2632 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2633 if (IS_ERR(fg)) {
2634 err = PTR_ERR(fg);
2635 goto out;
2636 }
2637
2638 ft_prio->anchor.fg_drop = fg;
2639
2640 out:
2641 kvfree(flow_group_in);
2642
2643 return err;
2644 }
2645
2646 static void
steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio * ft_prio)2647 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2648 {
2649 if (ft_prio->anchor.fg_drop) {
2650 mlx5_destroy_flow_group(ft_prio->anchor.fg_drop);
2651 ft_prio->anchor.fg_drop = NULL;
2652 }
2653 }
2654
2655 static int
steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio * ft_prio)2656 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2657 {
2658 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2659 struct mlx5_flow_group *fg;
2660 void *flow_group_in;
2661 int err = 0;
2662
2663 if (ft_prio->anchor.fg_goto_table)
2664 return 0;
2665
2666 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2667 if (!flow_group_in)
2668 return -ENOMEM;
2669
2670 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2671 if (IS_ERR(fg)) {
2672 err = PTR_ERR(fg);
2673 goto out;
2674 }
2675 ft_prio->anchor.fg_goto_table = fg;
2676
2677 out:
2678 kvfree(flow_group_in);
2679
2680 return err;
2681 }
2682
2683 static void
steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio * ft_prio)2684 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2685 {
2686 if (ft_prio->anchor.fg_goto_table) {
2687 mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table);
2688 ft_prio->anchor.fg_goto_table = NULL;
2689 }
2690 }
2691
2692 static int
steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio * ft_prio)2693 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2694 {
2695 struct mlx5_flow_act flow_act = {};
2696 struct mlx5_flow_handle *handle;
2697
2698 if (ft_prio->anchor.rule_drop)
2699 return 0;
2700
2701 flow_act.fg = ft_prio->anchor.fg_drop;
2702 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2703
2704 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2705 NULL, 0);
2706 if (IS_ERR(handle))
2707 return PTR_ERR(handle);
2708
2709 ft_prio->anchor.rule_drop = handle;
2710
2711 return 0;
2712 }
2713
steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio * ft_prio)2714 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2715 {
2716 if (ft_prio->anchor.rule_drop) {
2717 mlx5_del_flow_rules(ft_prio->anchor.rule_drop);
2718 ft_prio->anchor.rule_drop = NULL;
2719 }
2720 }
2721
2722 static int
steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio * ft_prio)2723 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2724 {
2725 struct mlx5_flow_destination dest = {};
2726 struct mlx5_flow_act flow_act = {};
2727 struct mlx5_flow_handle *handle;
2728
2729 if (ft_prio->anchor.rule_goto_table)
2730 return 0;
2731
2732 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2733 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
2734 flow_act.fg = ft_prio->anchor.fg_goto_table;
2735
2736 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2737 dest.ft = ft_prio->flow_table;
2738
2739 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2740 &dest, 1);
2741 if (IS_ERR(handle))
2742 return PTR_ERR(handle);
2743
2744 ft_prio->anchor.rule_goto_table = handle;
2745
2746 return 0;
2747 }
2748
2749 static void
steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio * ft_prio)2750 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2751 {
2752 if (ft_prio->anchor.rule_goto_table) {
2753 mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table);
2754 ft_prio->anchor.rule_goto_table = NULL;
2755 }
2756 }
2757
steering_anchor_create_res(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,enum mlx5_flow_namespace_type ns_type)2758 static int steering_anchor_create_res(struct mlx5_ib_dev *dev,
2759 struct mlx5_ib_flow_prio *ft_prio,
2760 enum mlx5_flow_namespace_type ns_type)
2761 {
2762 int err;
2763
2764 err = steering_anchor_create_ft(dev, ft_prio, ns_type);
2765 if (err)
2766 return err;
2767
2768 err = steering_anchor_create_fg_drop(ft_prio);
2769 if (err)
2770 goto destroy_ft;
2771
2772 err = steering_anchor_create_fg_goto_table(ft_prio);
2773 if (err)
2774 goto destroy_fg_drop;
2775
2776 err = steering_anchor_create_rule_drop(ft_prio);
2777 if (err)
2778 goto destroy_fg_goto_table;
2779
2780 err = steering_anchor_create_rule_goto_table(ft_prio);
2781 if (err)
2782 goto destroy_rule_drop;
2783
2784 return 0;
2785
2786 destroy_rule_drop:
2787 steering_anchor_destroy_rule_drop(ft_prio);
2788 destroy_fg_goto_table:
2789 steering_anchor_destroy_fg_goto_table(ft_prio);
2790 destroy_fg_drop:
2791 steering_anchor_destroy_fg_drop(ft_prio);
2792 destroy_ft:
2793 steering_anchor_destroy_ft(ft_prio);
2794
2795 return err;
2796 }
2797
mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio * ft_prio)2798 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)
2799 {
2800 steering_anchor_destroy_rule_goto_table(ft_prio);
2801 steering_anchor_destroy_rule_drop(ft_prio);
2802 steering_anchor_destroy_fg_goto_table(ft_prio);
2803 steering_anchor_destroy_fg_drop(ft_prio);
2804 steering_anchor_destroy_ft(ft_prio);
2805 }
2806
steering_anchor_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2807 static int steering_anchor_cleanup(struct ib_uobject *uobject,
2808 enum rdma_remove_reason why,
2809 struct uverbs_attr_bundle *attrs)
2810 {
2811 struct mlx5_ib_steering_anchor *obj = uobject->object;
2812
2813 if (atomic_read(&obj->usecnt))
2814 return -EBUSY;
2815
2816 mutex_lock(&obj->dev->flow_db->lock);
2817 if (!--obj->ft_prio->anchor.rule_goto_table_ref)
2818 steering_anchor_destroy_rule_goto_table(obj->ft_prio);
2819
2820 put_flow_table(obj->dev, obj->ft_prio, true);
2821 mutex_unlock(&obj->dev->flow_db->lock);
2822
2823 kfree(obj);
2824 return 0;
2825 }
2826
fs_cleanup_anchor(struct mlx5_ib_flow_prio * prio,int count)2827 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio,
2828 int count)
2829 {
2830 while (count--)
2831 mlx5_steering_anchor_destroy_res(&prio[count]);
2832 }
2833
mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev * dev)2834 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev)
2835 {
2836 fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT);
2837 fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT);
2838 fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS);
2839 fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS);
2840 fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS);
2841 fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT);
2842 fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT);
2843 }
2844
mlx5_ib_matcher_ns(struct uverbs_attr_bundle * attrs,struct mlx5_ib_flow_matcher * obj)2845 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
2846 struct mlx5_ib_flow_matcher *obj)
2847 {
2848 enum mlx5_ib_uapi_flow_table_type ft_type =
2849 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
2850 u32 flags;
2851 int err;
2852
2853 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
2854 * users should switch to it. We leave this to not break userspace
2855 */
2856 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
2857 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
2858 return -EINVAL;
2859
2860 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
2861 err = uverbs_get_const(&ft_type, attrs,
2862 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
2863 if (err)
2864 return err;
2865
2866 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
2867 if (err)
2868 return err;
2869
2870 return 0;
2871 }
2872
2873 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
2874 err = uverbs_get_flags32(&flags, attrs,
2875 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
2876 IB_FLOW_ATTR_FLAGS_EGRESS);
2877 if (err)
2878 return err;
2879
2880 if (flags)
2881 return mlx5_ib_ft_type_to_namespace(
2882 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
2883 &obj->ns_type);
2884 }
2885
2886 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
2887
2888 return 0;
2889 }
2890
verify_context_caps(struct mlx5_ib_dev * dev,u64 enabled_caps)2891 static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps)
2892 {
2893 if (is_mdev_switchdev_mode(dev->mdev))
2894 return UCAP_ENABLED(enabled_caps,
2895 RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
2896
2897 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL);
2898 }
2899
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)2900 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
2901 struct uverbs_attr_bundle *attrs)
2902 {
2903 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2904 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
2905 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2906 struct mlx5_ib_flow_matcher *obj;
2907 int err;
2908
2909 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
2910 if (!obj)
2911 return -ENOMEM;
2912
2913 obj->mask_len = uverbs_attr_get_len(
2914 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2915 err = uverbs_copy_from(&obj->matcher_mask,
2916 attrs,
2917 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2918 if (err)
2919 goto end;
2920
2921 obj->flow_type = uverbs_attr_get_enum_id(
2922 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2923
2924 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
2925 err = uverbs_copy_from(&obj->priority,
2926 attrs,
2927 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2928 if (err)
2929 goto end;
2930 }
2931
2932 err = uverbs_copy_from(&obj->match_criteria_enable,
2933 attrs,
2934 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
2935 if (err)
2936 goto end;
2937
2938 err = mlx5_ib_matcher_ns(attrs, obj);
2939 if (err)
2940 goto end;
2941
2942 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2943 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) {
2944 err = -EINVAL;
2945 goto end;
2946 }
2947
2948 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) {
2949 err = uverbs_copy_from(&obj->ib_port, attrs,
2950 MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT);
2951 if (err)
2952 goto end;
2953 if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) {
2954 err = -EINVAL;
2955 goto end;
2956 }
2957 if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX &&
2958 obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) {
2959 err = -EINVAL;
2960 goto end;
2961 }
2962 if (!verify_context_caps(dev, uobj->context->enabled_caps)) {
2963 err = -EOPNOTSUPP;
2964 goto end;
2965 }
2966 }
2967
2968 uobj->object = obj;
2969 obj->mdev = dev->mdev;
2970 atomic_set(&obj->usecnt, 0);
2971 return 0;
2972
2973 end:
2974 kfree(obj);
2975 return err;
2976 }
2977
UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)2978 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
2979 struct uverbs_attr_bundle *attrs)
2980 {
2981 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2982 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE);
2983 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2984 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type;
2985 enum mlx5_flow_namespace_type ns_type;
2986 struct mlx5_ib_steering_anchor *obj;
2987 struct mlx5_ib_flow_prio *ft_prio;
2988 u16 priority;
2989 u32 ft_id;
2990 int err;
2991
2992 if (!capable(CAP_NET_RAW))
2993 return -EPERM;
2994
2995 err = uverbs_get_const(&ib_uapi_ft_type, attrs,
2996 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE);
2997 if (err)
2998 return err;
2999
3000 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type);
3001 if (err)
3002 return err;
3003
3004 err = uverbs_copy_from(&priority, attrs,
3005 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY);
3006 if (err)
3007 return err;
3008
3009 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3010 if (!obj)
3011 return -ENOMEM;
3012
3013 mutex_lock(&dev->flow_db->lock);
3014
3015 ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0);
3016 if (IS_ERR(ft_prio)) {
3017 err = PTR_ERR(ft_prio);
3018 goto free_obj;
3019 }
3020
3021 ft_prio->refcount++;
3022
3023 if (!ft_prio->anchor.rule_goto_table_ref) {
3024 err = steering_anchor_create_res(dev, ft_prio, ns_type);
3025 if (err)
3026 goto put_flow_table;
3027 }
3028
3029 ft_prio->anchor.rule_goto_table_ref++;
3030
3031 ft_id = mlx5_flow_table_id(ft_prio->anchor.ft);
3032
3033 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3034 &ft_id, sizeof(ft_id));
3035 if (err)
3036 goto destroy_res;
3037
3038 mutex_unlock(&dev->flow_db->lock);
3039
3040 uobj->object = obj;
3041 obj->dev = dev;
3042 obj->ft_prio = ft_prio;
3043 atomic_set(&obj->usecnt, 0);
3044
3045 return 0;
3046
3047 destroy_res:
3048 --ft_prio->anchor.rule_goto_table_ref;
3049 mlx5_steering_anchor_destroy_res(ft_prio);
3050 put_flow_table:
3051 put_flow_table(dev, ft_prio, true);
3052 free_obj:
3053 mutex_unlock(&dev->flow_db->lock);
3054 kfree(obj);
3055
3056 return err;
3057 }
3058
3059 static struct ib_flow_action *
mlx5_ib_create_modify_header(struct mlx5_ib_dev * dev,enum mlx5_ib_uapi_flow_table_type ft_type,u8 num_actions,void * in)3060 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
3061 enum mlx5_ib_uapi_flow_table_type ft_type,
3062 u8 num_actions, void *in)
3063 {
3064 enum mlx5_flow_namespace_type namespace;
3065 struct mlx5_ib_flow_action *maction;
3066 int ret;
3067
3068 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3069 if (ret)
3070 return ERR_PTR(-EINVAL);
3071
3072 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3073 if (!maction)
3074 return ERR_PTR(-ENOMEM);
3075
3076 maction->flow_action_raw.modify_hdr =
3077 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
3078
3079 if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
3080 ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
3081 kfree(maction);
3082 return ERR_PTR(ret);
3083 }
3084 maction->flow_action_raw.sub_type =
3085 MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
3086 maction->flow_action_raw.dev = dev;
3087
3088 return &maction->ib_action;
3089 }
3090
mlx5_ib_modify_header_supported(struct mlx5_ib_dev * dev)3091 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
3092 {
3093 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3094 max_modify_header_actions) ||
3095 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
3096 max_modify_header_actions) ||
3097 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
3098 max_modify_header_actions);
3099 }
3100
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)3101 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
3102 struct uverbs_attr_bundle *attrs)
3103 {
3104 struct ib_uobject *uobj = uverbs_attr_get_uobject(
3105 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
3106 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3107 enum mlx5_ib_uapi_flow_table_type ft_type;
3108 struct ib_flow_action *action;
3109 int num_actions;
3110 void *in;
3111 int ret;
3112
3113 if (!mlx5_ib_modify_header_supported(mdev))
3114 return -EOPNOTSUPP;
3115
3116 in = uverbs_attr_get_alloced_ptr(attrs,
3117 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
3118
3119 num_actions = uverbs_attr_ptr_get_array_size(
3120 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3121 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
3122 if (num_actions < 0)
3123 return num_actions;
3124
3125 ret = uverbs_get_const(&ft_type, attrs,
3126 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
3127 if (ret)
3128 return ret;
3129 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
3130 if (IS_ERR(action))
3131 return PTR_ERR(action);
3132
3133 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
3134 IB_FLOW_ACTION_UNSPECIFIED);
3135
3136 return 0;
3137 }
3138
mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev * ibdev,u8 packet_reformat_type,u8 ft_type)3139 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
3140 u8 packet_reformat_type,
3141 u8 ft_type)
3142 {
3143 switch (packet_reformat_type) {
3144 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3145 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3146 return MLX5_CAP_FLOWTABLE(ibdev->mdev,
3147 encap_general_header);
3148 break;
3149 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3150 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3151 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
3152 reformat_l2_to_l3_tunnel);
3153 break;
3154 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3155 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3156 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
3157 reformat_l3_tunnel_to_l2);
3158 break;
3159 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
3160 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3161 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
3162 break;
3163 default:
3164 break;
3165 }
3166
3167 return false;
3168 }
3169
mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt,u8 * prm_prt)3170 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
3171 {
3172 switch (dv_prt) {
3173 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3174 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
3175 break;
3176 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3177 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
3178 break;
3179 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3180 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3181 break;
3182 default:
3183 return -EINVAL;
3184 }
3185
3186 return 0;
3187 }
3188
mlx5_ib_flow_action_create_packet_reformat_ctx(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_action * maction,u8 ft_type,u8 dv_prt,void * in,size_t len)3189 static int mlx5_ib_flow_action_create_packet_reformat_ctx(
3190 struct mlx5_ib_dev *dev,
3191 struct mlx5_ib_flow_action *maction,
3192 u8 ft_type, u8 dv_prt,
3193 void *in, size_t len)
3194 {
3195 struct mlx5_pkt_reformat_params reformat_params;
3196 enum mlx5_flow_namespace_type namespace;
3197 u8 prm_prt;
3198 int ret;
3199
3200 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3201 if (ret)
3202 return ret;
3203
3204 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
3205 if (ret)
3206 return ret;
3207
3208 memset(&reformat_params, 0, sizeof(reformat_params));
3209 reformat_params.type = prm_prt;
3210 reformat_params.size = len;
3211 reformat_params.data = in;
3212 maction->flow_action_raw.pkt_reformat =
3213 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params,
3214 namespace);
3215 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
3216 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
3217 return ret;
3218 }
3219
3220 maction->flow_action_raw.sub_type =
3221 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
3222 maction->flow_action_raw.dev = dev;
3223
3224 return 0;
3225 }
3226
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)3227 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
3228 struct uverbs_attr_bundle *attrs)
3229 {
3230 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
3231 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
3232 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3233 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
3234 enum mlx5_ib_uapi_flow_table_type ft_type;
3235 struct mlx5_ib_flow_action *maction;
3236 int ret;
3237
3238 ret = uverbs_get_const(&ft_type, attrs,
3239 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
3240 if (ret)
3241 return ret;
3242
3243 ret = uverbs_get_const(&dv_prt, attrs,
3244 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
3245 if (ret)
3246 return ret;
3247
3248 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
3249 return -EOPNOTSUPP;
3250
3251 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3252 if (!maction)
3253 return -ENOMEM;
3254
3255 if (dv_prt ==
3256 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
3257 maction->flow_action_raw.sub_type =
3258 MLX5_IB_FLOW_ACTION_DECAP;
3259 maction->flow_action_raw.dev = mdev;
3260 } else {
3261 void *in;
3262 int len;
3263
3264 in = uverbs_attr_get_alloced_ptr(attrs,
3265 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3266 if (IS_ERR(in)) {
3267 ret = PTR_ERR(in);
3268 goto free_maction;
3269 }
3270
3271 len = uverbs_attr_get_len(attrs,
3272 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3273
3274 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
3275 maction, ft_type, dv_prt, in, len);
3276 if (ret)
3277 goto free_maction;
3278 }
3279
3280 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
3281 IB_FLOW_ACTION_UNSPECIFIED);
3282 return 0;
3283
3284 free_maction:
3285 kfree(maction);
3286 return ret;
3287 }
3288
3289 DECLARE_UVERBS_NAMED_METHOD(
3290 MLX5_IB_METHOD_CREATE_FLOW,
3291 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3292 UVERBS_OBJECT_FLOW,
3293 UVERBS_ACCESS_NEW,
3294 UA_MANDATORY),
3295 UVERBS_ATTR_PTR_IN(
3296 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
3297 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3298 UA_MANDATORY,
3299 UA_ALLOC_AND_COPY),
3300 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
3301 MLX5_IB_OBJECT_FLOW_MATCHER,
3302 UVERBS_ACCESS_READ,
3303 UA_MANDATORY),
3304 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
3305 UVERBS_OBJECT_QP,
3306 UVERBS_ACCESS_READ),
3307 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
3308 MLX5_IB_OBJECT_DEVX_OBJ,
3309 UVERBS_ACCESS_READ),
3310 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
3311 UVERBS_OBJECT_FLOW_ACTION,
3312 UVERBS_ACCESS_READ, 1,
3313 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
3314 UA_OPTIONAL),
3315 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
3316 UVERBS_ATTR_TYPE(u32),
3317 UA_OPTIONAL),
3318 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
3319 MLX5_IB_OBJECT_DEVX_OBJ,
3320 UVERBS_ACCESS_READ, 1, 1,
3321 UA_OPTIONAL),
3322 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
3323 UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
3324 UA_OPTIONAL,
3325 UA_ALLOC_AND_COPY),
3326 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
3327 enum mlx5_ib_create_flow_flags,
3328 UA_OPTIONAL));
3329
3330 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3331 MLX5_IB_METHOD_DESTROY_FLOW,
3332 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3333 UVERBS_OBJECT_FLOW,
3334 UVERBS_ACCESS_DESTROY,
3335 UA_MANDATORY));
3336
3337 ADD_UVERBS_METHODS(mlx5_ib_fs,
3338 UVERBS_OBJECT_FLOW,
3339 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
3340 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
3341
3342 DECLARE_UVERBS_NAMED_METHOD(
3343 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
3344 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
3345 UVERBS_OBJECT_FLOW_ACTION,
3346 UVERBS_ACCESS_NEW,
3347 UA_MANDATORY),
3348 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3349 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
3350 set_add_copy_action_in_auto)),
3351 UA_MANDATORY,
3352 UA_ALLOC_AND_COPY),
3353 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
3354 enum mlx5_ib_uapi_flow_table_type,
3355 UA_MANDATORY));
3356
3357 DECLARE_UVERBS_NAMED_METHOD(
3358 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
3359 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
3360 UVERBS_OBJECT_FLOW_ACTION,
3361 UVERBS_ACCESS_NEW,
3362 UA_MANDATORY),
3363 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
3364 UVERBS_ATTR_MIN_SIZE(1),
3365 UA_ALLOC_AND_COPY,
3366 UA_OPTIONAL),
3367 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
3368 enum mlx5_ib_uapi_flow_action_packet_reformat_type,
3369 UA_MANDATORY),
3370 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
3371 enum mlx5_ib_uapi_flow_table_type,
3372 UA_MANDATORY));
3373
3374 ADD_UVERBS_METHODS(
3375 mlx5_ib_flow_actions,
3376 UVERBS_OBJECT_FLOW_ACTION,
3377 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
3378 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
3379
3380 DECLARE_UVERBS_NAMED_METHOD(
3381 MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
3382 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
3383 MLX5_IB_OBJECT_FLOW_MATCHER,
3384 UVERBS_ACCESS_NEW,
3385 UA_MANDATORY),
3386 UVERBS_ATTR_PTR_IN(
3387 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
3388 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3389 UA_MANDATORY),
3390 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
3391 mlx5_ib_flow_type,
3392 UA_MANDATORY),
3393 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
3394 UVERBS_ATTR_TYPE(u8),
3395 UA_MANDATORY),
3396 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
3397 enum ib_flow_flags,
3398 UA_OPTIONAL),
3399 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
3400 enum mlx5_ib_uapi_flow_table_type,
3401 UA_OPTIONAL),
3402 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT,
3403 UVERBS_ATTR_TYPE(u32),
3404 UA_OPTIONAL));
3405
3406 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3407 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
3408 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
3409 MLX5_IB_OBJECT_FLOW_MATCHER,
3410 UVERBS_ACCESS_DESTROY,
3411 UA_MANDATORY));
3412
3413 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
3414 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
3415 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
3416 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
3417
3418 DECLARE_UVERBS_NAMED_METHOD(
3419 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE,
3420 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE,
3421 MLX5_IB_OBJECT_STEERING_ANCHOR,
3422 UVERBS_ACCESS_NEW,
3423 UA_MANDATORY),
3424 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE,
3425 enum mlx5_ib_uapi_flow_table_type,
3426 UA_MANDATORY),
3427 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY,
3428 UVERBS_ATTR_TYPE(u16),
3429 UA_MANDATORY),
3430 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3431 UVERBS_ATTR_TYPE(u32),
3432 UA_MANDATORY));
3433
3434 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3435 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY,
3436 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE,
3437 MLX5_IB_OBJECT_STEERING_ANCHOR,
3438 UVERBS_ACCESS_DESTROY,
3439 UA_MANDATORY));
3440
3441 DECLARE_UVERBS_NAMED_OBJECT(
3442 MLX5_IB_OBJECT_STEERING_ANCHOR,
3443 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup),
3444 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE),
3445 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY));
3446
3447 const struct uapi_definition mlx5_ib_flow_defs[] = {
3448 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3449 MLX5_IB_OBJECT_FLOW_MATCHER),
3450 UAPI_DEF_CHAIN_OBJ_TREE(
3451 UVERBS_OBJECT_FLOW,
3452 &mlx5_ib_fs),
3453 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
3454 &mlx5_ib_flow_actions),
3455 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3456 MLX5_IB_OBJECT_STEERING_ANCHOR,
3457 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)),
3458 {},
3459 };
3460
3461 static const struct ib_device_ops flow_ops = {
3462 .create_flow = mlx5_ib_create_flow,
3463 .destroy_flow = mlx5_ib_destroy_flow,
3464 .destroy_flow_action = mlx5_ib_destroy_flow_action,
3465 };
3466
mlx5_ib_fs_init(struct mlx5_ib_dev * dev)3467 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
3468 {
3469 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
3470
3471 if (!dev->flow_db)
3472 return -ENOMEM;
3473
3474 dev->flow_db->rdma_transport_rx = kcalloc(dev->num_ports,
3475 sizeof(struct mlx5_ib_flow_prio),
3476 GFP_KERNEL);
3477 if (!dev->flow_db->rdma_transport_rx)
3478 goto free_flow_db;
3479
3480 dev->flow_db->rdma_transport_tx = kcalloc(dev->num_ports,
3481 sizeof(struct mlx5_ib_flow_prio),
3482 GFP_KERNEL);
3483 if (!dev->flow_db->rdma_transport_tx)
3484 goto free_rdma_transport_rx;
3485
3486 mutex_init(&dev->flow_db->lock);
3487
3488 ib_set_device_ops(&dev->ib_dev, &flow_ops);
3489 return 0;
3490
3491 free_rdma_transport_rx:
3492 kfree(dev->flow_db->rdma_transport_rx);
3493 free_flow_db:
3494 kfree(dev->flow_db);
3495 return -ENOMEM;
3496 }
3497