1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/uverbs_std_types.h> 11 #include <rdma/mlx5_user_ioctl_cmds.h> 12 #include <rdma/mlx5_user_ioctl_verbs.h> 13 #include <rdma/ib_hdrs.h> 14 #include <rdma/ib_umem.h> 15 #include <rdma/ib_ucaps.h> 16 #include <linux/mlx5/driver.h> 17 #include <linux/mlx5/fs.h> 18 #include <linux/mlx5/fs_helpers.h> 19 #include <linux/mlx5/eswitch.h> 20 #include <net/inet_ecn.h> 21 #include "mlx5_ib.h" 22 #include "counters.h" 23 #include "devx.h" 24 #include "fs.h" 25 26 #define UVERBS_MODULE_NAME mlx5_ib 27 #include <rdma/uverbs_named_ioctl.h> 28 29 enum { 30 MATCH_CRITERIA_ENABLE_OUTER_BIT, 31 MATCH_CRITERIA_ENABLE_MISC_BIT, 32 MATCH_CRITERIA_ENABLE_INNER_BIT, 33 MATCH_CRITERIA_ENABLE_MISC2_BIT 34 }; 35 36 37 struct mlx5_per_qp_opfc { 38 struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX]; 39 }; 40 41 #define HEADER_IS_ZERO(match_criteria, headers) \ 42 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 43 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 44 45 static u8 get_match_criteria_enable(u32 *match_criteria) 46 { 47 u8 match_criteria_enable; 48 49 match_criteria_enable = 50 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 51 MATCH_CRITERIA_ENABLE_OUTER_BIT; 52 match_criteria_enable |= 53 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 54 MATCH_CRITERIA_ENABLE_MISC_BIT; 55 match_criteria_enable |= 56 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 57 MATCH_CRITERIA_ENABLE_INNER_BIT; 58 match_criteria_enable |= 59 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 60 MATCH_CRITERIA_ENABLE_MISC2_BIT; 61 62 return match_criteria_enable; 63 } 64 65 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 66 { 67 u8 entry_mask; 68 u8 entry_val; 69 int err = 0; 70 71 if (!mask) 72 goto out; 73 74 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 75 ip_protocol); 76 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 77 ip_protocol); 78 if (!entry_mask) { 79 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 80 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 81 goto out; 82 } 83 /* Don't override existing ip protocol */ 84 if (mask != entry_mask || val != entry_val) 85 err = -EINVAL; 86 out: 87 return err; 88 } 89 90 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 91 bool inner) 92 { 93 if (inner) { 94 MLX5_SET(fte_match_set_misc, 95 misc_c, inner_ipv6_flow_label, mask); 96 MLX5_SET(fte_match_set_misc, 97 misc_v, inner_ipv6_flow_label, val); 98 } else { 99 MLX5_SET(fte_match_set_misc, 100 misc_c, outer_ipv6_flow_label, mask); 101 MLX5_SET(fte_match_set_misc, 102 misc_v, outer_ipv6_flow_label, val); 103 } 104 } 105 106 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 107 { 108 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 109 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 110 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 111 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 112 } 113 114 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 115 { 116 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 117 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 118 return -EOPNOTSUPP; 119 120 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 121 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 122 return -EOPNOTSUPP; 123 124 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 125 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 126 return -EOPNOTSUPP; 127 128 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 129 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 130 return -EOPNOTSUPP; 131 132 return 0; 133 } 134 135 #define LAST_ETH_FIELD vlan_tag 136 #define LAST_IPV4_FIELD tos 137 #define LAST_IPV6_FIELD traffic_class 138 #define LAST_TCP_UDP_FIELD src_port 139 #define LAST_TUNNEL_FIELD tunnel_id 140 #define LAST_FLOW_TAG_FIELD tag_id 141 #define LAST_DROP_FIELD size 142 #define LAST_COUNTERS_FIELD counters 143 144 /* Field is the last supported field */ 145 #define FIELDS_NOT_SUPPORTED(filter, field) \ 146 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ 147 sizeof(filter) - offsetofend(typeof(filter), field)) 148 149 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 150 bool is_egress, 151 struct mlx5_flow_act *action) 152 { 153 154 switch (maction->ib_action.type) { 155 case IB_FLOW_ACTION_UNSPECIFIED: 156 if (maction->flow_action_raw.sub_type == 157 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 158 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 159 return -EINVAL; 160 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 161 action->modify_hdr = 162 maction->flow_action_raw.modify_hdr; 163 return 0; 164 } 165 if (maction->flow_action_raw.sub_type == 166 MLX5_IB_FLOW_ACTION_DECAP) { 167 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 168 return -EINVAL; 169 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 170 return 0; 171 } 172 if (maction->flow_action_raw.sub_type == 173 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 174 if (action->action & 175 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 176 return -EINVAL; 177 action->action |= 178 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 179 action->pkt_reformat = 180 maction->flow_action_raw.pkt_reformat; 181 return 0; 182 } 183 fallthrough; 184 default: 185 return -EOPNOTSUPP; 186 } 187 } 188 189 static int parse_flow_attr(struct mlx5_core_dev *mdev, 190 struct mlx5_flow_spec *spec, 191 const union ib_flow_spec *ib_spec, 192 const struct ib_flow_attr *flow_attr, 193 struct mlx5_flow_act *action, u32 prev_type) 194 { 195 struct mlx5_flow_context *flow_context = &spec->flow_context; 196 u32 *match_c = spec->match_criteria; 197 u32 *match_v = spec->match_value; 198 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 199 misc_parameters); 200 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 201 misc_parameters); 202 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 203 misc_parameters_2); 204 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 205 misc_parameters_2); 206 void *headers_c; 207 void *headers_v; 208 int match_ipv; 209 int ret; 210 211 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 212 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 213 inner_headers); 214 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 215 inner_headers); 216 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 217 ft_field_support.inner_ip_version); 218 } else { 219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 220 outer_headers); 221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 222 outer_headers); 223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 224 ft_field_support.outer_ip_version); 225 } 226 227 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 228 case IB_FLOW_SPEC_ETH: 229 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 230 return -EOPNOTSUPP; 231 232 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 233 dmac_47_16), 234 ib_spec->eth.mask.dst_mac); 235 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 236 dmac_47_16), 237 ib_spec->eth.val.dst_mac); 238 239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 240 smac_47_16), 241 ib_spec->eth.mask.src_mac); 242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 243 smac_47_16), 244 ib_spec->eth.val.src_mac); 245 246 if (ib_spec->eth.mask.vlan_tag) { 247 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 248 cvlan_tag, 1); 249 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 250 cvlan_tag, 1); 251 252 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 253 first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 254 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 255 first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 256 257 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 258 first_cfi, 259 ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 260 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 261 first_cfi, 262 ntohs(ib_spec->eth.val.vlan_tag) >> 12); 263 264 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 265 first_prio, 266 ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 267 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 268 first_prio, 269 ntohs(ib_spec->eth.val.vlan_tag) >> 13); 270 } 271 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 272 ethertype, ntohs(ib_spec->eth.mask.ether_type)); 273 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 274 ethertype, ntohs(ib_spec->eth.val.ether_type)); 275 break; 276 case IB_FLOW_SPEC_IPV4: 277 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 278 return -EOPNOTSUPP; 279 280 if (match_ipv) { 281 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 282 ip_version, 0xf); 283 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 284 ip_version, MLX5_FS_IPV4_VERSION); 285 } else { 286 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 287 ethertype, 0xffff); 288 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 289 ethertype, ETH_P_IP); 290 } 291 292 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 293 src_ipv4_src_ipv6.ipv4_layout.ipv4), 294 &ib_spec->ipv4.mask.src_ip, 295 sizeof(ib_spec->ipv4.mask.src_ip)); 296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 297 src_ipv4_src_ipv6.ipv4_layout.ipv4), 298 &ib_spec->ipv4.val.src_ip, 299 sizeof(ib_spec->ipv4.val.src_ip)); 300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 301 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 302 &ib_spec->ipv4.mask.dst_ip, 303 sizeof(ib_spec->ipv4.mask.dst_ip)); 304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 306 &ib_spec->ipv4.val.dst_ip, 307 sizeof(ib_spec->ipv4.val.dst_ip)); 308 309 set_tos(headers_c, headers_v, 310 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 311 312 if (set_proto(headers_c, headers_v, 313 ib_spec->ipv4.mask.proto, 314 ib_spec->ipv4.val.proto)) 315 return -EINVAL; 316 break; 317 case IB_FLOW_SPEC_IPV6: 318 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 319 return -EOPNOTSUPP; 320 321 if (match_ipv) { 322 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 323 ip_version, 0xf); 324 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 325 ip_version, MLX5_FS_IPV6_VERSION); 326 } else { 327 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 328 ethertype, 0xffff); 329 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 330 ethertype, ETH_P_IPV6); 331 } 332 333 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 334 src_ipv4_src_ipv6.ipv6_layout.ipv6), 335 &ib_spec->ipv6.mask.src_ip, 336 sizeof(ib_spec->ipv6.mask.src_ip)); 337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 338 src_ipv4_src_ipv6.ipv6_layout.ipv6), 339 &ib_spec->ipv6.val.src_ip, 340 sizeof(ib_spec->ipv6.val.src_ip)); 341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 342 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 343 &ib_spec->ipv6.mask.dst_ip, 344 sizeof(ib_spec->ipv6.mask.dst_ip)); 345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 347 &ib_spec->ipv6.val.dst_ip, 348 sizeof(ib_spec->ipv6.val.dst_ip)); 349 350 set_tos(headers_c, headers_v, 351 ib_spec->ipv6.mask.traffic_class, 352 ib_spec->ipv6.val.traffic_class); 353 354 if (set_proto(headers_c, headers_v, 355 ib_spec->ipv6.mask.next_hdr, 356 ib_spec->ipv6.val.next_hdr)) 357 return -EINVAL; 358 359 set_flow_label(misc_params_c, misc_params_v, 360 ntohl(ib_spec->ipv6.mask.flow_label), 361 ntohl(ib_spec->ipv6.val.flow_label), 362 ib_spec->type & IB_FLOW_SPEC_INNER); 363 break; 364 case IB_FLOW_SPEC_ESP: 365 return -EOPNOTSUPP; 366 case IB_FLOW_SPEC_TCP: 367 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 368 LAST_TCP_UDP_FIELD)) 369 return -EOPNOTSUPP; 370 371 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 372 return -EINVAL; 373 374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 375 ntohs(ib_spec->tcp_udp.mask.src_port)); 376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 377 ntohs(ib_spec->tcp_udp.val.src_port)); 378 379 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 380 ntohs(ib_spec->tcp_udp.mask.dst_port)); 381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 382 ntohs(ib_spec->tcp_udp.val.dst_port)); 383 break; 384 case IB_FLOW_SPEC_UDP: 385 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 386 LAST_TCP_UDP_FIELD)) 387 return -EOPNOTSUPP; 388 389 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 390 return -EINVAL; 391 392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 393 ntohs(ib_spec->tcp_udp.mask.src_port)); 394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 395 ntohs(ib_spec->tcp_udp.val.src_port)); 396 397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 398 ntohs(ib_spec->tcp_udp.mask.dst_port)); 399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 400 ntohs(ib_spec->tcp_udp.val.dst_port)); 401 break; 402 case IB_FLOW_SPEC_GRE: 403 if (ib_spec->gre.mask.c_ks_res0_ver) 404 return -EOPNOTSUPP; 405 406 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 407 return -EINVAL; 408 409 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 410 0xff); 411 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 412 IPPROTO_GRE); 413 414 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 415 ntohs(ib_spec->gre.mask.protocol)); 416 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 417 ntohs(ib_spec->gre.val.protocol)); 418 419 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 420 gre_key.nvgre.hi), 421 &ib_spec->gre.mask.key, 422 sizeof(ib_spec->gre.mask.key)); 423 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 424 gre_key.nvgre.hi), 425 &ib_spec->gre.val.key, 426 sizeof(ib_spec->gre.val.key)); 427 break; 428 case IB_FLOW_SPEC_MPLS: 429 switch (prev_type) { 430 case IB_FLOW_SPEC_UDP: 431 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 432 ft_field_support.outer_first_mpls_over_udp), 433 &ib_spec->mpls.mask.tag)) 434 return -EOPNOTSUPP; 435 436 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 437 outer_first_mpls_over_udp), 438 &ib_spec->mpls.val.tag, 439 sizeof(ib_spec->mpls.val.tag)); 440 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 441 outer_first_mpls_over_udp), 442 &ib_spec->mpls.mask.tag, 443 sizeof(ib_spec->mpls.mask.tag)); 444 break; 445 case IB_FLOW_SPEC_GRE: 446 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 447 ft_field_support.outer_first_mpls_over_gre), 448 &ib_spec->mpls.mask.tag)) 449 return -EOPNOTSUPP; 450 451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 452 outer_first_mpls_over_gre), 453 &ib_spec->mpls.val.tag, 454 sizeof(ib_spec->mpls.val.tag)); 455 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 456 outer_first_mpls_over_gre), 457 &ib_spec->mpls.mask.tag, 458 sizeof(ib_spec->mpls.mask.tag)); 459 break; 460 default: 461 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 462 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 463 ft_field_support.inner_first_mpls), 464 &ib_spec->mpls.mask.tag)) 465 return -EOPNOTSUPP; 466 467 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 468 inner_first_mpls), 469 &ib_spec->mpls.val.tag, 470 sizeof(ib_spec->mpls.val.tag)); 471 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 472 inner_first_mpls), 473 &ib_spec->mpls.mask.tag, 474 sizeof(ib_spec->mpls.mask.tag)); 475 } else { 476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 477 ft_field_support.outer_first_mpls), 478 &ib_spec->mpls.mask.tag)) 479 return -EOPNOTSUPP; 480 481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 482 outer_first_mpls), 483 &ib_spec->mpls.val.tag, 484 sizeof(ib_spec->mpls.val.tag)); 485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 486 outer_first_mpls), 487 &ib_spec->mpls.mask.tag, 488 sizeof(ib_spec->mpls.mask.tag)); 489 } 490 } 491 break; 492 case IB_FLOW_SPEC_VXLAN_TUNNEL: 493 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 494 LAST_TUNNEL_FIELD)) 495 return -EOPNOTSUPP; 496 497 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 498 ntohl(ib_spec->tunnel.mask.tunnel_id)); 499 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 500 ntohl(ib_spec->tunnel.val.tunnel_id)); 501 break; 502 case IB_FLOW_SPEC_ACTION_TAG: 503 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 504 LAST_FLOW_TAG_FIELD)) 505 return -EOPNOTSUPP; 506 if (ib_spec->flow_tag.tag_id >= BIT(24)) 507 return -EINVAL; 508 509 flow_context->flow_tag = ib_spec->flow_tag.tag_id; 510 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 511 break; 512 case IB_FLOW_SPEC_ACTION_DROP: 513 if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 514 LAST_DROP_FIELD)) 515 return -EOPNOTSUPP; 516 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 517 break; 518 case IB_FLOW_SPEC_ACTION_HANDLE: 519 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 520 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 521 if (ret) 522 return ret; 523 break; 524 case IB_FLOW_SPEC_ACTION_COUNT: 525 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 526 LAST_COUNTERS_FIELD)) 527 return -EOPNOTSUPP; 528 529 /* for now support only one counters spec per flow */ 530 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 531 return -EINVAL; 532 533 action->counters = ib_spec->flow_count.counters; 534 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 535 break; 536 default: 537 return -EINVAL; 538 } 539 540 return 0; 541 } 542 543 /* If a flow could catch both multicast and unicast packets, 544 * it won't fall into the multicast flow steering table and this rule 545 * could steal other multicast packets. 546 */ 547 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 548 { 549 union ib_flow_spec *flow_spec; 550 551 if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 552 ib_attr->num_of_specs < 1) 553 return false; 554 555 flow_spec = (union ib_flow_spec *)(ib_attr + 1); 556 if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 557 struct ib_flow_spec_ipv4 *ipv4_spec; 558 559 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 560 if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 561 return true; 562 563 return false; 564 } 565 566 if (flow_spec->type == IB_FLOW_SPEC_ETH) { 567 struct ib_flow_spec_eth *eth_spec; 568 569 eth_spec = (struct ib_flow_spec_eth *)flow_spec; 570 return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 571 is_multicast_ether_addr(eth_spec->val.dst_mac); 572 } 573 574 return false; 575 } 576 577 static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 578 const struct ib_flow_attr *flow_attr, 579 bool check_inner) 580 { 581 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 582 int match_ipv = check_inner ? 583 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 584 ft_field_support.inner_ip_version) : 585 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 586 ft_field_support.outer_ip_version); 587 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 588 bool ipv4_spec_valid, ipv6_spec_valid; 589 unsigned int ip_spec_type = 0; 590 bool has_ethertype = false; 591 unsigned int spec_index; 592 bool mask_valid = true; 593 u16 eth_type = 0; 594 bool type_valid; 595 596 /* Validate that ethertype is correct */ 597 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 598 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 599 ib_spec->eth.mask.ether_type) { 600 mask_valid = (ib_spec->eth.mask.ether_type == 601 htons(0xffff)); 602 has_ethertype = true; 603 eth_type = ntohs(ib_spec->eth.val.ether_type); 604 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 605 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 606 ip_spec_type = ib_spec->type; 607 } 608 ib_spec = (void *)ib_spec + ib_spec->size; 609 } 610 611 type_valid = (!has_ethertype) || (!ip_spec_type); 612 if (!type_valid && mask_valid) { 613 ipv4_spec_valid = (eth_type == ETH_P_IP) && 614 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 615 ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 616 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 617 618 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 619 (((eth_type == ETH_P_MPLS_UC) || 620 (eth_type == ETH_P_MPLS_MC)) && match_ipv); 621 } 622 623 return type_valid; 624 } 625 626 static bool is_valid_attr(struct mlx5_core_dev *mdev, 627 const struct ib_flow_attr *flow_attr) 628 { 629 return is_valid_ethertype(mdev, flow_attr, false) && 630 is_valid_ethertype(mdev, flow_attr, true); 631 } 632 633 static void put_flow_table(struct mlx5_ib_dev *dev, 634 struct mlx5_ib_flow_prio *prio, bool ft_added) 635 { 636 prio->refcount -= !!ft_added; 637 if (!prio->refcount) { 638 mlx5_destroy_flow_table(prio->flow_table); 639 prio->flow_table = NULL; 640 } 641 } 642 643 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 644 { 645 struct mlx5_ib_flow_handler *handler = container_of(flow_id, 646 struct mlx5_ib_flow_handler, 647 ibflow); 648 struct mlx5_ib_flow_handler *iter, *tmp; 649 struct mlx5_ib_dev *dev = handler->dev; 650 651 mutex_lock(&dev->flow_db->lock); 652 653 list_for_each_entry_safe(iter, tmp, &handler->list, list) { 654 mlx5_del_flow_rules(iter->rule); 655 put_flow_table(dev, iter->prio, true); 656 list_del(&iter->list); 657 kfree(iter); 658 } 659 660 mlx5_del_flow_rules(handler->rule); 661 put_flow_table(dev, handler->prio, true); 662 mlx5_ib_counters_clear_description(handler->ibcounters); 663 mutex_unlock(&dev->flow_db->lock); 664 if (handler->flow_matcher) 665 atomic_dec(&handler->flow_matcher->usecnt); 666 kfree(handler); 667 668 return 0; 669 } 670 671 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 672 { 673 priority *= 2; 674 if (!dont_trap) 675 priority++; 676 return priority; 677 } 678 679 enum flow_table_type { 680 MLX5_IB_FT_RX, 681 MLX5_IB_FT_TX 682 }; 683 684 #define MLX5_FS_MAX_TYPES 6 685 #define MLX5_FS_MAX_ENTRIES BIT(16) 686 687 static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device) 688 { 689 struct mlx5_ib_dev *dev = to_mdev(device); 690 691 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed); 692 } 693 694 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_ib_dev *dev, 695 struct mlx5_flow_namespace *ns, 696 struct mlx5_ib_flow_prio *prio, 697 int priority, 698 int num_entries, int num_groups, 699 u32 flags, u16 vport) 700 { 701 struct mlx5_flow_table_attr ft_attr = {}; 702 struct mlx5_flow_table *ft; 703 704 ft_attr.prio = priority; 705 ft_attr.max_fte = num_entries; 706 ft_attr.flags = flags; 707 ft_attr.vport = vport; 708 ft_attr.autogroup.max_num_groups = num_groups; 709 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 710 if (IS_ERR(ft)) 711 return ERR_CAST(ft); 712 713 prio->flow_table = ft; 714 prio->refcount = 0; 715 return prio; 716 } 717 718 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 719 struct ib_flow_attr *flow_attr, 720 enum flow_table_type ft_type) 721 { 722 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 723 struct mlx5_flow_namespace *ns = NULL; 724 enum mlx5_flow_namespace_type fn_type; 725 struct mlx5_ib_flow_prio *prio; 726 struct mlx5_flow_table *ft; 727 int max_table_size; 728 int num_entries; 729 int num_groups; 730 bool esw_encap; 731 u32 flags = 0; 732 int priority; 733 734 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 735 log_max_ft_size)); 736 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 737 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 738 switch (flow_attr->type) { 739 case IB_FLOW_ATTR_NORMAL: 740 if (flow_is_multicast_only(flow_attr) && !dont_trap) 741 priority = MLX5_IB_FLOW_MCAST_PRIO; 742 else 743 priority = ib_prio_to_core_prio(flow_attr->priority, 744 dont_trap); 745 if (ft_type == MLX5_IB_FT_RX) { 746 fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 747 prio = &dev->flow_db->prios[priority]; 748 if (!dev->is_rep && !esw_encap && 749 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 750 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 751 if (!dev->is_rep && !esw_encap && 752 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 753 reformat_l3_tunnel_to_l2)) 754 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 755 } else { 756 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( 757 dev->mdev, log_max_ft_size)); 758 fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 759 prio = &dev->flow_db->egress_prios[priority]; 760 if (!dev->is_rep && !esw_encap && 761 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 762 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 763 } 764 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 765 num_entries = MLX5_FS_MAX_ENTRIES; 766 num_groups = MLX5_FS_MAX_TYPES; 767 break; 768 case IB_FLOW_ATTR_ALL_DEFAULT: 769 case IB_FLOW_ATTR_MC_DEFAULT: 770 ns = mlx5_get_flow_namespace(dev->mdev, 771 MLX5_FLOW_NAMESPACE_LEFTOVERS); 772 build_leftovers_ft_param(&priority, &num_entries, &num_groups); 773 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 774 break; 775 case IB_FLOW_ATTR_SNIFFER: 776 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 777 allow_sniffer_and_nic_rx_shared_tir)) 778 return ERR_PTR(-EOPNOTSUPP); 779 780 ns = mlx5_get_flow_namespace( 781 dev->mdev, ft_type == MLX5_IB_FT_RX ? 782 MLX5_FLOW_NAMESPACE_SNIFFER_RX : 783 MLX5_FLOW_NAMESPACE_SNIFFER_TX); 784 785 prio = &dev->flow_db->sniffer[ft_type]; 786 priority = 0; 787 num_entries = 1; 788 num_groups = 1; 789 break; 790 default: 791 break; 792 } 793 794 if (!ns) 795 return ERR_PTR(-EOPNOTSUPP); 796 797 max_table_size = min_t(int, num_entries, max_table_size); 798 799 ft = prio->flow_table; 800 if (!ft) 801 return _get_prio(dev, ns, prio, priority, max_table_size, 802 num_groups, flags, 0); 803 804 return prio; 805 } 806 807 enum { 808 RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO, 809 RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO, 810 RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 811 RDMA_RX_ECN_OPCOUNTER_PRIO, 812 RDMA_RX_CNP_OPCOUNTER_PRIO, 813 RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO, 814 }; 815 816 enum { 817 RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO, 818 RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 819 RDMA_TX_CNP_OPCOUNTER_PRIO, 820 RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO, 821 }; 822 823 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num, 824 struct mlx5_flow_spec *spec) 825 { 826 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 827 ft_field_support.source_vhca_port) || 828 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 829 ft_field_support.source_vhca_port)) 830 return -EOPNOTSUPP; 831 832 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria, 833 misc_parameters.source_vhca_port); 834 MLX5_SET(fte_match_param, &spec->match_value, 835 misc_parameters.source_vhca_port, port_num); 836 837 return 0; 838 } 839 840 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num, 841 struct mlx5_flow_spec *spec, int ipv) 842 { 843 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 844 ft_field_support.outer_ip_version)) 845 return -EOPNOTSUPP; 846 847 if (mlx5_core_mp_enabled(dev->mdev) && 848 set_vhca_port_spec(dev, port_num, spec)) 849 return -EOPNOTSUPP; 850 851 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 852 outer_headers.ip_ecn); 853 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn, 854 INET_ECN_CE); 855 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 856 outer_headers.ip_version); 857 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 858 ipv); 859 860 spec->match_criteria_enable = 861 get_match_criteria_enable(spec->match_criteria); 862 863 return 0; 864 } 865 866 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num, 867 struct mlx5_flow_spec *spec) 868 { 869 if (mlx5_core_mp_enabled(dev->mdev) && 870 set_vhca_port_spec(dev, port_num, spec)) 871 return -EOPNOTSUPP; 872 873 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 874 misc_parameters.bth_opcode); 875 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode, 876 IB_BTH_OPCODE_CNP); 877 878 spec->match_criteria_enable = 879 get_match_criteria_enable(spec->match_criteria); 880 881 return 0; 882 } 883 884 /* Returns the prio we should use for the given optional counter type, 885 * whereas for bytes type we use the packet type, since they share the same 886 * resources. 887 */ 888 static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev, 889 u32 type) 890 { 891 u32 prio_type; 892 893 switch (type) { 894 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 895 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS; 896 break; 897 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 898 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS; 899 break; 900 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 901 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 902 break; 903 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 904 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 905 break; 906 default: 907 prio_type = type; 908 } 909 910 return &dev->flow_db->opfcs[prio_type]; 911 } 912 913 static void put_per_qp_prio(struct mlx5_ib_dev *dev, 914 enum mlx5_ib_optional_counter_type type) 915 { 916 enum mlx5_ib_optional_counter_type per_qp_type; 917 struct mlx5_ib_flow_prio *prio; 918 919 switch (type) { 920 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 921 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 922 break; 923 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 924 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 925 break; 926 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 927 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 928 break; 929 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 930 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 931 break; 932 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 933 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 934 break; 935 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 936 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 937 break; 938 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 939 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 940 break; 941 default: 942 return; 943 } 944 945 prio = get_opfc_prio(dev, per_qp_type); 946 put_flow_table(dev, prio, true); 947 } 948 949 static int get_per_qp_prio(struct mlx5_ib_dev *dev, 950 enum mlx5_ib_optional_counter_type type) 951 { 952 enum mlx5_ib_optional_counter_type per_qp_type; 953 enum mlx5_flow_namespace_type fn_type; 954 struct mlx5_flow_namespace *ns; 955 struct mlx5_ib_flow_prio *prio; 956 int priority; 957 958 switch (type) { 959 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 960 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 961 priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO; 962 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 963 break; 964 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 965 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 966 priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO; 967 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 968 break; 969 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 970 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 971 priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO; 972 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 973 break; 974 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 975 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 976 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 977 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 978 break; 979 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 980 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 981 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 982 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 983 break; 984 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 985 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 986 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 987 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 988 break; 989 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 990 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 991 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 992 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 993 break; 994 default: 995 return -EINVAL; 996 } 997 998 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 999 if (!ns) 1000 return -EOPNOTSUPP; 1001 1002 prio = get_opfc_prio(dev, per_qp_type); 1003 if (prio->flow_table) 1004 return 0; 1005 1006 prio = _get_prio(dev, ns, prio, priority, MLX5_FS_MAX_POOL_SIZE, 1, 0, 0); 1007 if (IS_ERR(prio)) 1008 return PTR_ERR(prio); 1009 1010 prio->refcount = 1; 1011 1012 return 0; 1013 } 1014 1015 static struct mlx5_per_qp_opfc * 1016 get_per_qp_opfc(struct mlx5_rdma_counter *mcounter, u32 qp_num, bool *new) 1017 { 1018 struct mlx5_per_qp_opfc *per_qp_opfc; 1019 1020 *new = false; 1021 1022 per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp_num); 1023 if (per_qp_opfc) 1024 return per_qp_opfc; 1025 per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL); 1026 1027 if (!per_qp_opfc) 1028 return NULL; 1029 1030 *new = true; 1031 return per_qp_opfc; 1032 } 1033 1034 static int add_op_fc_rules(struct mlx5_ib_dev *dev, 1035 struct mlx5_rdma_counter *mcounter, 1036 struct mlx5_per_qp_opfc *per_qp_opfc, 1037 struct mlx5_ib_flow_prio *prio, 1038 enum mlx5_ib_optional_counter_type type, 1039 u32 qp_num, u32 port_num) 1040 { 1041 struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc; 1042 struct mlx5_flow_act flow_act = {}; 1043 struct mlx5_flow_destination dst; 1044 struct mlx5_flow_spec *spec; 1045 int i, err, spec_num; 1046 bool is_tx; 1047 1048 if (opfc->fc) 1049 return -EEXIST; 1050 1051 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type, 1052 &in_use_opfc)) { 1053 opfc->fc = in_use_opfc->fc; 1054 opfc->rule[0] = in_use_opfc->rule[0]; 1055 return 0; 1056 } 1057 1058 opfc->fc = mcounter->fc[type]; 1059 1060 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1061 if (!spec) { 1062 err = -ENOMEM; 1063 goto null_fc; 1064 } 1065 1066 switch (type) { 1067 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP: 1068 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1069 MLX5_FS_IPV4_VERSION) || 1070 set_ecn_ce_spec(dev, port_num, &spec[1], 1071 MLX5_FS_IPV6_VERSION)) { 1072 err = -EOPNOTSUPP; 1073 goto free_spec; 1074 } 1075 spec_num = 2; 1076 is_tx = false; 1077 1078 MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria, 1079 misc_parameters.bth_dst_qp); 1080 MLX5_SET(fte_match_param, spec[1].match_value, 1081 misc_parameters.bth_dst_qp, qp_num); 1082 spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1083 break; 1084 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP: 1085 if (!MLX5_CAP_FLOWTABLE( 1086 dev->mdev, 1087 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1088 set_cnp_spec(dev, port_num, &spec[0])) { 1089 err = -EOPNOTSUPP; 1090 goto free_spec; 1091 } 1092 spec_num = 1; 1093 is_tx = false; 1094 break; 1095 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP: 1096 if (!MLX5_CAP_FLOWTABLE( 1097 dev->mdev, 1098 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1099 set_cnp_spec(dev, port_num, &spec[0])) { 1100 err = -EOPNOTSUPP; 1101 goto free_spec; 1102 } 1103 spec_num = 1; 1104 is_tx = true; 1105 break; 1106 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1107 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1108 spec_num = 1; 1109 is_tx = true; 1110 break; 1111 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1112 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1113 spec_num = 1; 1114 is_tx = false; 1115 break; 1116 default: 1117 err = -EINVAL; 1118 goto free_spec; 1119 } 1120 1121 if (is_tx) { 1122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1123 misc_parameters.source_sqn); 1124 MLX5_SET(fte_match_param, spec->match_value, 1125 misc_parameters.source_sqn, qp_num); 1126 } else { 1127 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1128 misc_parameters.bth_dst_qp); 1129 MLX5_SET(fte_match_param, spec->match_value, 1130 misc_parameters.bth_dst_qp, qp_num); 1131 } 1132 1133 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1134 1135 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1136 dst.counter = opfc->fc; 1137 1138 flow_act.action = 1139 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1140 1141 for (i = 0; i < spec_num; i++) { 1142 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1143 &flow_act, &dst, 1); 1144 if (IS_ERR(opfc->rule[i])) { 1145 err = PTR_ERR(opfc->rule[i]); 1146 goto del_rules; 1147 } 1148 } 1149 prio->refcount += spec_num; 1150 1151 err = xa_err(xa_store(&mcounter->qpn_opfc_xa, qp_num, per_qp_opfc, 1152 GFP_KERNEL)); 1153 if (err) 1154 goto del_rules; 1155 1156 kfree(spec); 1157 1158 return 0; 1159 1160 del_rules: 1161 while (i--) 1162 mlx5_del_flow_rules(opfc->rule[i]); 1163 put_flow_table(dev, prio, false); 1164 free_spec: 1165 kfree(spec); 1166 null_fc: 1167 opfc->fc = NULL; 1168 return err; 1169 } 1170 1171 static bool is_fc_shared_and_in_use(struct mlx5_rdma_counter *mcounter, 1172 u32 type, struct mlx5_fc **fc) 1173 { 1174 u32 shared_fc_type; 1175 1176 switch (type) { 1177 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1178 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 1179 break; 1180 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1181 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 1182 break; 1183 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1184 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 1185 break; 1186 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1187 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 1188 break; 1189 default: 1190 return false; 1191 } 1192 1193 *fc = mcounter->fc[shared_fc_type]; 1194 if (!(*fc)) 1195 return false; 1196 1197 return true; 1198 } 1199 1200 void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev, 1201 struct rdma_counter *counter) 1202 { 1203 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 1204 struct mlx5_fc *in_use_fc; 1205 int i; 1206 1207 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1208 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1209 if (!mcounter->fc[i]) 1210 continue; 1211 1212 if (is_fc_shared_and_in_use(mcounter, i, &in_use_fc)) { 1213 mcounter->fc[i] = NULL; 1214 continue; 1215 } 1216 1217 mlx5_fc_destroy(dev->mdev, mcounter->fc[i]); 1218 mcounter->fc[i] = NULL; 1219 } 1220 } 1221 1222 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num, 1223 struct mlx5_ib_op_fc *opfc, 1224 enum mlx5_ib_optional_counter_type type) 1225 { 1226 enum mlx5_flow_namespace_type fn_type; 1227 int priority, i, err, spec_num; 1228 struct mlx5_flow_act flow_act = {}; 1229 struct mlx5_flow_destination dst; 1230 struct mlx5_flow_namespace *ns; 1231 struct mlx5_ib_flow_prio *prio; 1232 struct mlx5_flow_spec *spec; 1233 1234 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1235 if (!spec) 1236 return -ENOMEM; 1237 1238 switch (type) { 1239 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 1240 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1241 MLX5_FS_IPV4_VERSION) || 1242 set_ecn_ce_spec(dev, port_num, &spec[1], 1243 MLX5_FS_IPV6_VERSION)) { 1244 err = -EOPNOTSUPP; 1245 goto free; 1246 } 1247 spec_num = 2; 1248 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1249 priority = RDMA_RX_ECN_OPCOUNTER_PRIO; 1250 break; 1251 1252 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 1253 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1254 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1255 set_cnp_spec(dev, port_num, &spec[0])) { 1256 err = -EOPNOTSUPP; 1257 goto free; 1258 } 1259 spec_num = 1; 1260 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1261 priority = RDMA_RX_CNP_OPCOUNTER_PRIO; 1262 break; 1263 1264 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 1265 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1266 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1267 set_cnp_spec(dev, port_num, &spec[0])) { 1268 err = -EOPNOTSUPP; 1269 goto free; 1270 } 1271 spec_num = 1; 1272 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1273 priority = RDMA_TX_CNP_OPCOUNTER_PRIO; 1274 break; 1275 1276 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 1277 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 1278 spec_num = 1; 1279 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1280 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO; 1281 break; 1282 1283 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 1284 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 1285 spec_num = 1; 1286 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1287 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO; 1288 break; 1289 1290 default: 1291 err = -EOPNOTSUPP; 1292 goto free; 1293 } 1294 1295 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 1296 if (!ns) { 1297 err = -EOPNOTSUPP; 1298 goto free; 1299 } 1300 1301 prio = get_opfc_prio(dev, type); 1302 if (!prio->flow_table) { 1303 err = get_per_qp_prio(dev, type); 1304 if (err) 1305 goto free; 1306 1307 prio = _get_prio(dev, ns, prio, priority, 1308 dev->num_ports * MAX_OPFC_RULES, 1, 0, 0); 1309 if (IS_ERR(prio)) { 1310 err = PTR_ERR(prio); 1311 goto put_prio; 1312 } 1313 } 1314 1315 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1316 dst.counter = opfc->fc; 1317 1318 flow_act.action = 1319 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1320 1321 for (i = 0; i < spec_num; i++) { 1322 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1323 &flow_act, &dst, 1); 1324 if (IS_ERR(opfc->rule[i])) { 1325 err = PTR_ERR(opfc->rule[i]); 1326 goto del_rules; 1327 } 1328 } 1329 prio->refcount += spec_num; 1330 kfree(spec); 1331 1332 return 0; 1333 1334 del_rules: 1335 for (i -= 1; i >= 0; i--) 1336 mlx5_del_flow_rules(opfc->rule[i]); 1337 put_flow_table(dev, prio, false); 1338 put_prio: 1339 put_per_qp_prio(dev, type); 1340 free: 1341 kfree(spec); 1342 return err; 1343 } 1344 1345 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev, 1346 struct mlx5_ib_op_fc *opfc, 1347 enum mlx5_ib_optional_counter_type type) 1348 { 1349 struct mlx5_ib_flow_prio *prio; 1350 int i; 1351 1352 prio = get_opfc_prio(dev, type); 1353 1354 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) { 1355 mlx5_del_flow_rules(opfc->rule[i]); 1356 put_flow_table(dev, prio, true); 1357 } 1358 1359 put_per_qp_prio(dev, type); 1360 } 1361 1362 void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct rdma_counter *counter) 1363 { 1364 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 1365 struct mlx5_ib_dev *dev = to_mdev(counter->device); 1366 struct mlx5_per_qp_opfc *per_qp_opfc; 1367 struct mlx5_ib_op_fc *in_use_opfc; 1368 struct mlx5_ib_flow_prio *prio; 1369 int i, j; 1370 1371 per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp->qp_num); 1372 if (!per_qp_opfc) 1373 return; 1374 1375 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1376 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1377 if (!per_qp_opfc->opfcs[i].fc) 1378 continue; 1379 1380 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i, 1381 &in_use_opfc)) { 1382 per_qp_opfc->opfcs[i].fc = NULL; 1383 continue; 1384 } 1385 1386 for (j = 0; j < MAX_OPFC_RULES; j++) { 1387 if (!per_qp_opfc->opfcs[i].rule[j]) 1388 continue; 1389 mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]); 1390 prio = get_opfc_prio(dev, i); 1391 put_flow_table(dev, prio, true); 1392 } 1393 per_qp_opfc->opfcs[i].fc = NULL; 1394 } 1395 1396 kfree(per_qp_opfc); 1397 xa_erase(&mcounter->qpn_opfc_xa, qp->qp_num); 1398 } 1399 1400 int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter, 1401 u32 port) 1402 { 1403 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 1404 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1405 struct mlx5_per_qp_opfc *per_qp_opfc; 1406 struct mlx5_ib_flow_prio *prio; 1407 struct mlx5_ib_counters *cnts; 1408 struct mlx5_ib_op_fc *opfc; 1409 struct mlx5_fc *in_use_fc; 1410 int i, err, per_qp_type; 1411 bool new; 1412 1413 if (!counter->mode.bind_opcnt) 1414 return 0; 1415 1416 cnts = &dev->port[port - 1].cnts; 1417 1418 for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) { 1419 opfc = &cnts->opfcs[i]; 1420 if (!opfc->fc) 1421 continue; 1422 1423 per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1424 prio = get_opfc_prio(dev, per_qp_type); 1425 WARN_ON(!prio->flow_table); 1426 1427 if (is_fc_shared_and_in_use(mcounter, per_qp_type, &in_use_fc)) 1428 mcounter->fc[per_qp_type] = in_use_fc; 1429 1430 if (!mcounter->fc[per_qp_type]) { 1431 mcounter->fc[per_qp_type] = mlx5_fc_create(dev->mdev, 1432 false); 1433 if (IS_ERR(mcounter->fc[per_qp_type])) 1434 return PTR_ERR(mcounter->fc[per_qp_type]); 1435 } 1436 1437 per_qp_opfc = get_per_qp_opfc(mcounter, qp->qp_num, &new); 1438 if (!per_qp_opfc) { 1439 err = -ENOMEM; 1440 goto free_fc; 1441 } 1442 err = add_op_fc_rules(dev, mcounter, per_qp_opfc, prio, 1443 per_qp_type, qp->qp_num, port); 1444 if (err) 1445 goto del_rules; 1446 } 1447 1448 return 0; 1449 1450 del_rules: 1451 mlx5r_fs_unbind_op_fc(qp, counter); 1452 if (new) 1453 kfree(per_qp_opfc); 1454 free_fc: 1455 if (xa_empty(&mcounter->qpn_opfc_xa)) 1456 mlx5r_fs_destroy_fcs(dev, counter); 1457 return err; 1458 } 1459 1460 static void set_underlay_qp(struct mlx5_ib_dev *dev, 1461 struct mlx5_flow_spec *spec, 1462 u32 underlay_qpn) 1463 { 1464 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 1465 spec->match_criteria, 1466 misc_parameters); 1467 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1468 misc_parameters); 1469 1470 if (underlay_qpn && 1471 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1472 ft_field_support.bth_dst_qp)) { 1473 MLX5_SET(fte_match_set_misc, 1474 misc_params_v, bth_dst_qp, underlay_qpn); 1475 MLX5_SET(fte_match_set_misc, 1476 misc_params_c, bth_dst_qp, 0xffffff); 1477 } 1478 } 1479 1480 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 1481 struct mlx5_flow_spec *spec, 1482 struct mlx5_eswitch_rep *rep) 1483 { 1484 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 1485 void *misc; 1486 1487 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1488 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1489 misc_parameters_2); 1490 1491 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1492 mlx5_eswitch_get_vport_metadata_for_match(rep->esw, 1493 rep->vport)); 1494 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1495 misc_parameters_2); 1496 1497 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1498 mlx5_eswitch_get_vport_metadata_mask()); 1499 } else { 1500 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1501 misc_parameters); 1502 1503 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 1504 1505 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1506 misc_parameters); 1507 1508 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1509 } 1510 } 1511 1512 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 1513 struct mlx5_ib_flow_prio *ft_prio, 1514 const struct ib_flow_attr *flow_attr, 1515 struct mlx5_flow_destination *dst, 1516 u32 underlay_qpn, 1517 struct mlx5_ib_create_flow *ucmd) 1518 { 1519 struct mlx5_flow_table *ft = ft_prio->flow_table; 1520 struct mlx5_ib_flow_handler *handler; 1521 struct mlx5_flow_act flow_act = {}; 1522 struct mlx5_flow_spec *spec; 1523 struct mlx5_flow_destination dest_arr[2] = {}; 1524 struct mlx5_flow_destination *rule_dst = dest_arr; 1525 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 1526 unsigned int spec_index; 1527 u32 prev_type = 0; 1528 int err = 0; 1529 int dest_num = 0; 1530 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1531 1532 if (!is_valid_attr(dev->mdev, flow_attr)) 1533 return ERR_PTR(-EINVAL); 1534 1535 if (dev->is_rep && is_egress) 1536 return ERR_PTR(-EINVAL); 1537 1538 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1539 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1540 if (!handler || !spec) { 1541 err = -ENOMEM; 1542 goto free; 1543 } 1544 1545 INIT_LIST_HEAD(&handler->list); 1546 1547 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 1548 err = parse_flow_attr(dev->mdev, spec, 1549 ib_flow, flow_attr, &flow_act, 1550 prev_type); 1551 if (err < 0) 1552 goto free; 1553 1554 prev_type = ((union ib_flow_spec *)ib_flow)->type; 1555 ib_flow += ((union ib_flow_spec *)ib_flow)->size; 1556 } 1557 1558 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 1559 memcpy(&dest_arr[0], dst, sizeof(*dst)); 1560 dest_num++; 1561 } 1562 1563 if (!flow_is_multicast_only(flow_attr)) 1564 set_underlay_qp(dev, spec, underlay_qpn); 1565 1566 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { 1567 struct mlx5_eswitch_rep *rep; 1568 1569 rep = dev->port[flow_attr->port - 1].rep; 1570 if (!rep) { 1571 err = -EINVAL; 1572 goto free; 1573 } 1574 1575 mlx5_ib_set_rule_source_port(dev, spec, rep); 1576 } 1577 1578 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 1579 1580 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1581 struct mlx5_ib_mcounters *mcounters; 1582 1583 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 1584 if (err) 1585 goto free; 1586 1587 mcounters = to_mcounters(flow_act.counters); 1588 handler->ibcounters = flow_act.counters; 1589 dest_arr[dest_num].type = 1590 MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1591 dest_arr[dest_num].counter = 1592 mcounters->hw_cntrs_hndl; 1593 dest_num++; 1594 } 1595 1596 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 1597 if (!dest_num) 1598 rule_dst = NULL; 1599 } else { 1600 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 1601 flow_act.action |= 1602 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1603 if (is_egress) 1604 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1605 else if (dest_num) 1606 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1607 } 1608 1609 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1610 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1611 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1612 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1613 spec->flow_context.flow_tag, flow_attr->type); 1614 err = -EINVAL; 1615 goto free; 1616 } 1617 handler->rule = mlx5_add_flow_rules(ft, spec, 1618 &flow_act, 1619 rule_dst, dest_num); 1620 1621 if (IS_ERR(handler->rule)) { 1622 err = PTR_ERR(handler->rule); 1623 goto free; 1624 } 1625 1626 ft_prio->refcount++; 1627 handler->prio = ft_prio; 1628 handler->dev = dev; 1629 1630 ft_prio->flow_table = ft; 1631 free: 1632 if (err && handler) { 1633 mlx5_ib_counters_clear_description(handler->ibcounters); 1634 kfree(handler); 1635 } 1636 kvfree(spec); 1637 return err ? ERR_PTR(err) : handler; 1638 } 1639 1640 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1641 struct mlx5_ib_flow_prio *ft_prio, 1642 const struct ib_flow_attr *flow_attr, 1643 struct mlx5_flow_destination *dst) 1644 { 1645 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1646 } 1647 1648 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1649 struct mlx5_ib_flow_prio *ft_prio, 1650 struct ib_flow_attr *flow_attr, 1651 struct mlx5_flow_destination *dst) 1652 { 1653 struct mlx5_ib_flow_handler *handler_ucast = NULL; 1654 struct mlx5_ib_flow_handler *handler = NULL; 1655 1656 static struct { 1657 struct ib_flow_spec_eth eth_flow; 1658 struct ib_flow_attr flow_attr; 1659 } leftovers_wc = { .flow_attr = { .num_of_specs = 1, 1660 .size = sizeof(leftovers_wc) }, 1661 .eth_flow = { 1662 .type = IB_FLOW_SPEC_ETH, 1663 .size = sizeof(struct ib_flow_spec_eth), 1664 .mask = { .dst_mac = { 0x1 } }, 1665 .val = { .dst_mac = { 0x1 } } } }; 1666 1667 static struct { 1668 struct ib_flow_spec_eth eth_flow; 1669 struct ib_flow_attr flow_attr; 1670 } leftovers_uc = { .flow_attr = { .num_of_specs = 1, 1671 .size = sizeof(leftovers_uc) }, 1672 .eth_flow = { 1673 .type = IB_FLOW_SPEC_ETH, 1674 .size = sizeof(struct ib_flow_spec_eth), 1675 .mask = { .dst_mac = { 0x1 } }, 1676 .val = { .dst_mac = {} } } }; 1677 1678 handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst); 1679 if (!IS_ERR(handler) && 1680 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1681 handler_ucast = create_flow_rule(dev, ft_prio, 1682 &leftovers_uc.flow_attr, dst); 1683 if (IS_ERR(handler_ucast)) { 1684 mlx5_del_flow_rules(handler->rule); 1685 ft_prio->refcount--; 1686 kfree(handler); 1687 handler = handler_ucast; 1688 } else { 1689 list_add(&handler_ucast->list, &handler->list); 1690 } 1691 } 1692 1693 return handler; 1694 } 1695 1696 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1697 struct mlx5_ib_flow_prio *ft_rx, 1698 struct mlx5_ib_flow_prio *ft_tx, 1699 struct mlx5_flow_destination *dst) 1700 { 1701 struct mlx5_ib_flow_handler *handler_rx; 1702 struct mlx5_ib_flow_handler *handler_tx; 1703 int err; 1704 static const struct ib_flow_attr flow_attr = { 1705 .num_of_specs = 0, 1706 .type = IB_FLOW_ATTR_SNIFFER, 1707 .size = sizeof(flow_attr) 1708 }; 1709 1710 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1711 if (IS_ERR(handler_rx)) { 1712 err = PTR_ERR(handler_rx); 1713 goto err; 1714 } 1715 1716 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1717 if (IS_ERR(handler_tx)) { 1718 err = PTR_ERR(handler_tx); 1719 goto err_tx; 1720 } 1721 1722 list_add(&handler_tx->list, &handler_rx->list); 1723 1724 return handler_rx; 1725 1726 err_tx: 1727 mlx5_del_flow_rules(handler_rx->rule); 1728 ft_rx->refcount--; 1729 kfree(handler_rx); 1730 err: 1731 return ERR_PTR(err); 1732 } 1733 1734 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1735 struct ib_flow_attr *flow_attr, 1736 struct ib_udata *udata) 1737 { 1738 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1739 struct mlx5_ib_qp *mqp = to_mqp(qp); 1740 struct mlx5_ib_flow_handler *handler = NULL; 1741 struct mlx5_flow_destination *dst = NULL; 1742 struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1743 struct mlx5_ib_flow_prio *ft_prio; 1744 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1745 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1746 size_t min_ucmd_sz, required_ucmd_sz; 1747 int err; 1748 int underlay_qpn; 1749 1750 if (udata && udata->inlen) { 1751 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); 1752 if (udata->inlen < min_ucmd_sz) 1753 return ERR_PTR(-EOPNOTSUPP); 1754 1755 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1756 if (err) 1757 return ERR_PTR(err); 1758 1759 /* currently supports only one counters data */ 1760 if (ucmd_hdr.ncounters_data > 1) 1761 return ERR_PTR(-EINVAL); 1762 1763 required_ucmd_sz = min_ucmd_sz + 1764 sizeof(struct mlx5_ib_flow_counters_data) * 1765 ucmd_hdr.ncounters_data; 1766 if (udata->inlen > required_ucmd_sz && 1767 !ib_is_udata_cleared(udata, required_ucmd_sz, 1768 udata->inlen - required_ucmd_sz)) 1769 return ERR_PTR(-EOPNOTSUPP); 1770 1771 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1772 if (!ucmd) 1773 return ERR_PTR(-ENOMEM); 1774 1775 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1776 if (err) 1777 goto free_ucmd; 1778 } 1779 1780 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1781 err = -ENOMEM; 1782 goto free_ucmd; 1783 } 1784 1785 if (flow_attr->flags & 1786 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) { 1787 err = -EINVAL; 1788 goto free_ucmd; 1789 } 1790 1791 if (is_egress && 1792 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1793 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1794 err = -EINVAL; 1795 goto free_ucmd; 1796 } 1797 1798 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1799 if (!dst) { 1800 err = -ENOMEM; 1801 goto free_ucmd; 1802 } 1803 1804 mutex_lock(&dev->flow_db->lock); 1805 1806 ft_prio = get_flow_table(dev, flow_attr, 1807 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1808 if (IS_ERR(ft_prio)) { 1809 err = PTR_ERR(ft_prio); 1810 goto unlock; 1811 } 1812 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1813 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1814 if (IS_ERR(ft_prio_tx)) { 1815 err = PTR_ERR(ft_prio_tx); 1816 ft_prio_tx = NULL; 1817 goto destroy_ft; 1818 } 1819 } 1820 1821 if (is_egress) { 1822 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1823 } else { 1824 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1825 if (mqp->is_rss) 1826 dst->tir_num = mqp->rss_qp.tirn; 1827 else 1828 dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1829 } 1830 1831 switch (flow_attr->type) { 1832 case IB_FLOW_ATTR_NORMAL: 1833 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1834 mqp->underlay_qpn : 1835 0; 1836 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1837 underlay_qpn, ucmd); 1838 break; 1839 case IB_FLOW_ATTR_ALL_DEFAULT: 1840 case IB_FLOW_ATTR_MC_DEFAULT: 1841 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); 1842 break; 1843 case IB_FLOW_ATTR_SNIFFER: 1844 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1845 break; 1846 default: 1847 err = -EINVAL; 1848 goto destroy_ft; 1849 } 1850 1851 if (IS_ERR(handler)) { 1852 err = PTR_ERR(handler); 1853 handler = NULL; 1854 goto destroy_ft; 1855 } 1856 1857 mutex_unlock(&dev->flow_db->lock); 1858 kfree(dst); 1859 kfree(ucmd); 1860 1861 return &handler->ibflow; 1862 1863 destroy_ft: 1864 put_flow_table(dev, ft_prio, false); 1865 if (ft_prio_tx) 1866 put_flow_table(dev, ft_prio_tx, false); 1867 unlock: 1868 mutex_unlock(&dev->flow_db->lock); 1869 kfree(dst); 1870 free_ucmd: 1871 kfree(ucmd); 1872 return ERR_PTR(err); 1873 } 1874 1875 static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev, 1876 enum mlx5_flow_namespace_type type, 1877 u32 *flags, u16 *vport_idx, 1878 u16 *vport, 1879 struct mlx5_core_dev **ft_mdev, 1880 u32 ib_port) 1881 { 1882 struct mlx5_core_dev *esw_mdev; 1883 1884 if (!is_mdev_switchdev_mode(dev->mdev)) 1885 return 0; 1886 1887 if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager)) 1888 return -EOPNOTSUPP; 1889 1890 if (!dev->port[ib_port - 1].rep) 1891 return -EINVAL; 1892 1893 esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw); 1894 if (esw_mdev != dev->mdev) 1895 return -EOPNOTSUPP; 1896 1897 *flags |= MLX5_FLOW_TABLE_OTHER_VPORT; 1898 *ft_mdev = esw_mdev; 1899 *vport = dev->port[ib_port - 1].rep->vport; 1900 *vport_idx = dev->port[ib_port - 1].rep->vport_index; 1901 1902 return 0; 1903 } 1904 1905 static struct mlx5_ib_flow_prio * 1906 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority, 1907 enum mlx5_flow_namespace_type ns_type, 1908 bool mcast, u32 ib_port) 1909 { 1910 struct mlx5_core_dev *ft_mdev = dev->mdev; 1911 struct mlx5_flow_namespace *ns = NULL; 1912 struct mlx5_ib_flow_prio *prio = NULL; 1913 int max_table_size = 0; 1914 u16 vport_idx = 0; 1915 bool esw_encap; 1916 u32 flags = 0; 1917 u16 vport = 0; 1918 int priority; 1919 int ret; 1920 1921 if (mcast) 1922 priority = MLX5_IB_FLOW_MCAST_PRIO; 1923 else 1924 priority = ib_prio_to_core_prio(user_priority, false); 1925 1926 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1927 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1928 switch (ns_type) { 1929 case MLX5_FLOW_NAMESPACE_BYPASS: 1930 max_table_size = BIT( 1931 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); 1932 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1933 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1934 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1935 reformat_l3_tunnel_to_l2) && 1936 !esw_encap) 1937 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1938 break; 1939 case MLX5_FLOW_NAMESPACE_EGRESS: 1940 max_table_size = BIT( 1941 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1942 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && 1943 !esw_encap) 1944 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1945 break; 1946 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1947 max_table_size = BIT( 1948 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1949 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1950 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1951 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, 1952 reformat_l3_tunnel_to_l2) && 1953 esw_encap) 1954 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1955 priority = user_priority; 1956 break; 1957 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1958 max_table_size = BIT( 1959 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); 1960 priority = user_priority; 1961 break; 1962 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1963 max_table_size = BIT( 1964 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); 1965 priority = user_priority; 1966 break; 1967 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 1968 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 1969 if (ib_port == 0 || user_priority > MLX5_RDMA_TRANSPORT_BYPASS_PRIO) 1970 return ERR_PTR(-EINVAL); 1971 ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags, 1972 &vport_idx, &vport, 1973 &ft_mdev, ib_port); 1974 if (ret) 1975 return ERR_PTR(ret); 1976 1977 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) 1978 max_table_size = 1979 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX( 1980 ft_mdev, log_max_ft_size)); 1981 else 1982 max_table_size = 1983 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX( 1984 ft_mdev, log_max_ft_size)); 1985 priority = user_priority; 1986 break; 1987 default: 1988 break; 1989 } 1990 1991 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1992 1993 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX || 1994 ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) 1995 ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx); 1996 else 1997 ns = mlx5_get_flow_namespace(ft_mdev, ns_type); 1998 1999 if (!ns) 2000 return ERR_PTR(-EOPNOTSUPP); 2001 2002 switch (ns_type) { 2003 case MLX5_FLOW_NAMESPACE_BYPASS: 2004 prio = &dev->flow_db->prios[priority]; 2005 break; 2006 case MLX5_FLOW_NAMESPACE_EGRESS: 2007 prio = &dev->flow_db->egress_prios[priority]; 2008 break; 2009 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 2010 prio = &dev->flow_db->fdb[priority]; 2011 break; 2012 case MLX5_FLOW_NAMESPACE_RDMA_RX: 2013 prio = &dev->flow_db->rdma_rx[priority]; 2014 break; 2015 case MLX5_FLOW_NAMESPACE_RDMA_TX: 2016 prio = &dev->flow_db->rdma_tx[priority]; 2017 break; 2018 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 2019 prio = &dev->flow_db->rdma_transport_rx[ib_port - 1]; 2020 break; 2021 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 2022 prio = &dev->flow_db->rdma_transport_tx[ib_port - 1]; 2023 break; 2024 default: return ERR_PTR(-EINVAL); 2025 } 2026 2027 if (!prio) 2028 return ERR_PTR(-EINVAL); 2029 2030 if (prio->flow_table) 2031 return prio; 2032 2033 return _get_prio(dev, ns, prio, priority, max_table_size, 2034 MLX5_FS_MAX_TYPES, flags, vport); 2035 } 2036 2037 static struct mlx5_ib_flow_handler * 2038 _create_raw_flow_rule(struct mlx5_ib_dev *dev, 2039 struct mlx5_ib_flow_prio *ft_prio, 2040 struct mlx5_flow_destination *dst, 2041 struct mlx5_ib_flow_matcher *fs_matcher, 2042 struct mlx5_flow_context *flow_context, 2043 struct mlx5_flow_act *flow_act, 2044 void *cmd_in, int inlen, 2045 int dst_num) 2046 { 2047 struct mlx5_ib_flow_handler *handler; 2048 struct mlx5_flow_spec *spec; 2049 struct mlx5_flow_table *ft = ft_prio->flow_table; 2050 int err = 0; 2051 2052 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2053 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 2054 if (!handler || !spec) { 2055 err = -ENOMEM; 2056 goto free; 2057 } 2058 2059 INIT_LIST_HEAD(&handler->list); 2060 2061 memcpy(spec->match_value, cmd_in, inlen); 2062 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 2063 fs_matcher->mask_len); 2064 spec->match_criteria_enable = fs_matcher->match_criteria_enable; 2065 spec->flow_context = *flow_context; 2066 2067 handler->rule = mlx5_add_flow_rules(ft, spec, 2068 flow_act, dst, dst_num); 2069 2070 if (IS_ERR(handler->rule)) { 2071 err = PTR_ERR(handler->rule); 2072 goto free; 2073 } 2074 2075 ft_prio->refcount++; 2076 handler->prio = ft_prio; 2077 handler->dev = dev; 2078 ft_prio->flow_table = ft; 2079 2080 free: 2081 if (err) 2082 kfree(handler); 2083 kvfree(spec); 2084 return err ? ERR_PTR(err) : handler; 2085 } 2086 2087 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 2088 void *match_v) 2089 { 2090 void *match_c; 2091 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 2092 void *dmac, *dmac_mask; 2093 void *ipv4, *ipv4_mask; 2094 2095 if (!(fs_matcher->match_criteria_enable & 2096 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 2097 return false; 2098 2099 match_c = fs_matcher->matcher_mask.match_params; 2100 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 2101 outer_headers); 2102 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 2103 outer_headers); 2104 2105 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2106 dmac_47_16); 2107 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2108 dmac_47_16); 2109 2110 if (is_multicast_ether_addr(dmac) && 2111 is_multicast_ether_addr(dmac_mask)) 2112 return true; 2113 2114 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2115 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2116 2117 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2118 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2119 2120 if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 2121 ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 2122 return true; 2123 2124 return false; 2125 } 2126 2127 static struct mlx5_ib_flow_handler *raw_fs_rule_add( 2128 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 2129 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 2130 struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type) 2131 { 2132 struct mlx5_flow_destination *dst; 2133 struct mlx5_ib_flow_prio *ft_prio; 2134 struct mlx5_ib_flow_handler *handler; 2135 int dst_num = 0; 2136 bool mcast; 2137 int err; 2138 2139 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 2140 return ERR_PTR(-EOPNOTSUPP); 2141 2142 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 2143 return ERR_PTR(-ENOMEM); 2144 2145 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 2146 if (!dst) 2147 return ERR_PTR(-ENOMEM); 2148 2149 mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 2150 mutex_lock(&dev->flow_db->lock); 2151 2152 ft_prio = _get_flow_table(dev, fs_matcher->priority, 2153 fs_matcher->ns_type, mcast, 2154 fs_matcher->ib_port); 2155 if (IS_ERR(ft_prio)) { 2156 err = PTR_ERR(ft_prio); 2157 goto unlock; 2158 } 2159 2160 switch (dest_type) { 2161 case MLX5_FLOW_DESTINATION_TYPE_TIR: 2162 dst[dst_num].type = dest_type; 2163 dst[dst_num++].tir_num = dest_id; 2164 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2165 break; 2166 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: 2167 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 2168 dst[dst_num++].ft_num = dest_id; 2169 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2170 break; 2171 case MLX5_FLOW_DESTINATION_TYPE_PORT: 2172 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2173 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 2174 break; 2175 default: 2176 break; 2177 } 2178 2179 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 2180 if (WARN_ON(!counter)) { 2181 err = -EINVAL; 2182 goto unlock; 2183 } 2184 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2185 dst[dst_num].counter = counter; 2186 dst_num++; 2187 } 2188 2189 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, 2190 fs_matcher, flow_context, flow_act, 2191 cmd_in, inlen, dst_num); 2192 2193 if (IS_ERR(handler)) { 2194 err = PTR_ERR(handler); 2195 goto destroy_ft; 2196 } 2197 2198 mutex_unlock(&dev->flow_db->lock); 2199 atomic_inc(&fs_matcher->usecnt); 2200 handler->flow_matcher = fs_matcher; 2201 2202 kfree(dst); 2203 2204 return handler; 2205 2206 destroy_ft: 2207 put_flow_table(dev, ft_prio, false); 2208 unlock: 2209 mutex_unlock(&dev->flow_db->lock); 2210 kfree(dst); 2211 2212 return ERR_PTR(err); 2213 } 2214 2215 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 2216 { 2217 switch (maction->flow_action_raw.sub_type) { 2218 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 2219 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 2220 maction->flow_action_raw.modify_hdr); 2221 break; 2222 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 2223 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 2224 maction->flow_action_raw.pkt_reformat); 2225 break; 2226 case MLX5_IB_FLOW_ACTION_DECAP: 2227 break; 2228 default: 2229 break; 2230 } 2231 } 2232 2233 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 2234 { 2235 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 2236 2237 switch (action->type) { 2238 case IB_FLOW_ACTION_UNSPECIFIED: 2239 destroy_flow_action_raw(maction); 2240 break; 2241 default: 2242 WARN_ON(true); 2243 break; 2244 } 2245 2246 kfree(maction); 2247 return 0; 2248 } 2249 2250 static int 2251 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 2252 enum mlx5_flow_namespace_type *namespace) 2253 { 2254 switch (table_type) { 2255 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 2256 *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 2257 break; 2258 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 2259 *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 2260 break; 2261 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 2262 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS; 2263 break; 2264 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 2265 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 2266 break; 2267 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 2268 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 2269 break; 2270 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX: 2271 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX; 2272 break; 2273 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX: 2274 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX; 2275 break; 2276 default: 2277 return -EINVAL; 2278 } 2279 2280 return 0; 2281 } 2282 2283 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 2284 [MLX5_IB_FLOW_TYPE_NORMAL] = { 2285 .type = UVERBS_ATTR_TYPE_PTR_IN, 2286 .u.ptr = { 2287 .len = sizeof(u16), /* data is priority */ 2288 .min_len = sizeof(u16), 2289 } 2290 }, 2291 [MLX5_IB_FLOW_TYPE_SNIFFER] = { 2292 .type = UVERBS_ATTR_TYPE_PTR_IN, 2293 UVERBS_ATTR_NO_DATA(), 2294 }, 2295 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 2296 .type = UVERBS_ATTR_TYPE_PTR_IN, 2297 UVERBS_ATTR_NO_DATA(), 2298 }, 2299 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 2300 .type = UVERBS_ATTR_TYPE_PTR_IN, 2301 UVERBS_ATTR_NO_DATA(), 2302 }, 2303 }; 2304 2305 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 2306 { 2307 struct devx_obj *devx_obj = obj; 2308 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2309 2310 switch (opcode) { 2311 case MLX5_CMD_OP_DESTROY_TIR: 2312 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2313 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 2314 obj_id); 2315 return true; 2316 2317 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 2318 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2319 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 2320 table_id); 2321 return true; 2322 default: 2323 return false; 2324 } 2325 } 2326 2327 static int get_dests(struct uverbs_attr_bundle *attrs, 2328 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 2329 int *dest_type, struct ib_qp **qp, u32 *flags) 2330 { 2331 bool dest_devx, dest_qp; 2332 void *devx_obj; 2333 int err; 2334 2335 dest_devx = uverbs_attr_is_valid(attrs, 2336 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2337 dest_qp = uverbs_attr_is_valid(attrs, 2338 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2339 2340 *flags = 0; 2341 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2342 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 2343 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 2344 if (err) 2345 return err; 2346 2347 /* Both flags are not allowed */ 2348 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 2349 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2350 return -EINVAL; 2351 2352 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 2353 if (dest_devx && (dest_qp || *flags)) 2354 return -EINVAL; 2355 else if (dest_qp && *flags) 2356 return -EINVAL; 2357 } 2358 2359 /* Allow only DEVX object, drop as dest for FDB */ 2360 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2361 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 2362 return -EINVAL; 2363 2364 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 2365 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2366 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2367 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 2368 return -EINVAL; 2369 2370 *qp = NULL; 2371 if (dest_devx) { 2372 devx_obj = 2373 uverbs_attr_get_obj(attrs, 2374 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2375 2376 /* Verify that the given DEVX object is a flow 2377 * steering destination. 2378 */ 2379 if (!is_flow_dest(devx_obj, dest_id, dest_type)) 2380 return -EINVAL; 2381 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 2382 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS || 2383 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2384 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2385 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 2386 return -EINVAL; 2387 } else if (dest_qp) { 2388 struct mlx5_ib_qp *mqp; 2389 2390 *qp = uverbs_attr_get_obj(attrs, 2391 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2392 if (IS_ERR(*qp)) 2393 return PTR_ERR(*qp); 2394 2395 if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 2396 return -EINVAL; 2397 2398 mqp = to_mqp(*qp); 2399 if (mqp->is_rss) 2400 *dest_id = mqp->rss_qp.tirn; 2401 else 2402 *dest_id = mqp->raw_packet_qp.rq.tirn; 2403 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2404 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2405 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2406 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) && 2407 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { 2408 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2409 } 2410 2411 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 2412 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2413 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2414 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)) 2415 return -EINVAL; 2416 2417 return 0; 2418 } 2419 2420 static bool 2421 is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size) 2422 { 2423 struct devx_obj *devx_obj = obj; 2424 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2425 2426 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 2427 2428 if (offset && offset >= devx_obj->flow_counter_bulk_size) 2429 return false; 2430 2431 *fc_bulk_size = devx_obj->flow_counter_bulk_size; 2432 *counter_id = MLX5_GET(dealloc_flow_counter_in, 2433 devx_obj->dinbox, 2434 flow_counter_id); 2435 *counter_id += offset; 2436 return true; 2437 } 2438 2439 return false; 2440 } 2441 2442 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 2443 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 2444 struct uverbs_attr_bundle *attrs) 2445 { 2446 struct mlx5_flow_context flow_context = {.flow_tag = 2447 MLX5_FS_DEFAULT_FLOW_TAG}; 2448 int dest_id, dest_type = -1, inlen, len, ret, i; 2449 struct mlx5_ib_flow_handler *flow_handler; 2450 struct mlx5_ib_flow_matcher *fs_matcher; 2451 struct ib_uobject **arr_flow_actions; 2452 struct ib_uflow_resources *uflow_res; 2453 struct mlx5_flow_act flow_act = {}; 2454 struct mlx5_fc *counter = NULL; 2455 struct ib_qp *qp = NULL; 2456 void *devx_obj, *cmd_in; 2457 struct ib_uobject *uobj; 2458 struct mlx5_ib_dev *dev; 2459 u32 flags; 2460 2461 if (!capable(CAP_NET_RAW)) 2462 return -EPERM; 2463 2464 fs_matcher = uverbs_attr_get_obj(attrs, 2465 MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 2466 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 2467 dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2468 2469 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 2470 return -EINVAL; 2471 2472 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 2473 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 2474 2475 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2476 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 2477 2478 len = uverbs_attr_get_uobjs_arr(attrs, 2479 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 2480 if (len) { 2481 u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0; 2482 devx_obj = arr_flow_actions[0]->object; 2483 2484 if (uverbs_attr_is_valid(attrs, 2485 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 2486 2487 int num_offsets = uverbs_attr_ptr_get_array_size( 2488 attrs, 2489 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2490 sizeof(u32)); 2491 2492 if (num_offsets != 1) 2493 return -EINVAL; 2494 2495 offset_attr = uverbs_attr_get_alloced_ptr( 2496 attrs, 2497 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 2498 offset = *offset_attr; 2499 } 2500 2501 if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size)) 2502 return -EINVAL; 2503 counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size); 2504 if (IS_ERR(counter)) 2505 return PTR_ERR(counter); 2506 2507 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 2508 } 2509 2510 cmd_in = uverbs_attr_get_alloced_ptr( 2511 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2512 inlen = uverbs_attr_get_len(attrs, 2513 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2514 2515 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 2516 if (!uflow_res) { 2517 ret = -ENOMEM; 2518 goto destroy_counter; 2519 } 2520 2521 len = uverbs_attr_get_uobjs_arr(attrs, 2522 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 2523 for (i = 0; i < len; i++) { 2524 struct mlx5_ib_flow_action *maction = 2525 to_mflow_act(arr_flow_actions[i]->object); 2526 2527 ret = parse_flow_flow_action(maction, false, &flow_act); 2528 if (ret) 2529 goto err_out; 2530 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 2531 arr_flow_actions[i]->object); 2532 } 2533 2534 ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 2535 MLX5_IB_ATTR_CREATE_FLOW_TAG); 2536 if (!ret) { 2537 if (flow_context.flow_tag >= BIT(24)) { 2538 ret = -EINVAL; 2539 goto err_out; 2540 } 2541 flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 2542 } 2543 2544 flow_handler = 2545 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 2546 counter, cmd_in, inlen, dest_id, dest_type); 2547 if (IS_ERR(flow_handler)) { 2548 ret = PTR_ERR(flow_handler); 2549 goto err_out; 2550 } 2551 2552 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2553 2554 return 0; 2555 err_out: 2556 ib_uverbs_flow_resources_free(uflow_res); 2557 destroy_counter: 2558 if (counter) 2559 mlx5_fc_local_destroy(counter); 2560 return ret; 2561 } 2562 2563 static int flow_matcher_cleanup(struct ib_uobject *uobject, 2564 enum rdma_remove_reason why, 2565 struct uverbs_attr_bundle *attrs) 2566 { 2567 struct mlx5_ib_flow_matcher *obj = uobject->object; 2568 2569 if (atomic_read(&obj->usecnt)) 2570 return -EBUSY; 2571 2572 kfree(obj); 2573 return 0; 2574 } 2575 2576 static int steering_anchor_create_ft(struct mlx5_ib_dev *dev, 2577 struct mlx5_ib_flow_prio *ft_prio, 2578 enum mlx5_flow_namespace_type ns_type) 2579 { 2580 struct mlx5_flow_table_attr ft_attr = {}; 2581 struct mlx5_flow_namespace *ns; 2582 struct mlx5_flow_table *ft; 2583 2584 if (ft_prio->anchor.ft) 2585 return 0; 2586 2587 ns = mlx5_get_flow_namespace(dev->mdev, ns_type); 2588 if (!ns) 2589 return -EOPNOTSUPP; 2590 2591 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 2592 ft_attr.uid = MLX5_SHARED_RESOURCE_UID; 2593 ft_attr.prio = 0; 2594 ft_attr.max_fte = 2; 2595 ft_attr.level = 1; 2596 2597 ft = mlx5_create_flow_table(ns, &ft_attr); 2598 if (IS_ERR(ft)) 2599 return PTR_ERR(ft); 2600 2601 ft_prio->anchor.ft = ft; 2602 2603 return 0; 2604 } 2605 2606 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio) 2607 { 2608 if (ft_prio->anchor.ft) { 2609 mlx5_destroy_flow_table(ft_prio->anchor.ft); 2610 ft_prio->anchor.ft = NULL; 2611 } 2612 } 2613 2614 static int 2615 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2616 { 2617 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2618 struct mlx5_flow_group *fg; 2619 void *flow_group_in; 2620 int err = 0; 2621 2622 if (ft_prio->anchor.fg_drop) 2623 return 0; 2624 2625 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2626 if (!flow_group_in) 2627 return -ENOMEM; 2628 2629 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1); 2630 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); 2631 2632 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2633 if (IS_ERR(fg)) { 2634 err = PTR_ERR(fg); 2635 goto out; 2636 } 2637 2638 ft_prio->anchor.fg_drop = fg; 2639 2640 out: 2641 kvfree(flow_group_in); 2642 2643 return err; 2644 } 2645 2646 static void 2647 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2648 { 2649 if (ft_prio->anchor.fg_drop) { 2650 mlx5_destroy_flow_group(ft_prio->anchor.fg_drop); 2651 ft_prio->anchor.fg_drop = NULL; 2652 } 2653 } 2654 2655 static int 2656 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2657 { 2658 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2659 struct mlx5_flow_group *fg; 2660 void *flow_group_in; 2661 int err = 0; 2662 2663 if (ft_prio->anchor.fg_goto_table) 2664 return 0; 2665 2666 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2667 if (!flow_group_in) 2668 return -ENOMEM; 2669 2670 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2671 if (IS_ERR(fg)) { 2672 err = PTR_ERR(fg); 2673 goto out; 2674 } 2675 ft_prio->anchor.fg_goto_table = fg; 2676 2677 out: 2678 kvfree(flow_group_in); 2679 2680 return err; 2681 } 2682 2683 static void 2684 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2685 { 2686 if (ft_prio->anchor.fg_goto_table) { 2687 mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table); 2688 ft_prio->anchor.fg_goto_table = NULL; 2689 } 2690 } 2691 2692 static int 2693 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2694 { 2695 struct mlx5_flow_act flow_act = {}; 2696 struct mlx5_flow_handle *handle; 2697 2698 if (ft_prio->anchor.rule_drop) 2699 return 0; 2700 2701 flow_act.fg = ft_prio->anchor.fg_drop; 2702 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2703 2704 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2705 NULL, 0); 2706 if (IS_ERR(handle)) 2707 return PTR_ERR(handle); 2708 2709 ft_prio->anchor.rule_drop = handle; 2710 2711 return 0; 2712 } 2713 2714 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2715 { 2716 if (ft_prio->anchor.rule_drop) { 2717 mlx5_del_flow_rules(ft_prio->anchor.rule_drop); 2718 ft_prio->anchor.rule_drop = NULL; 2719 } 2720 } 2721 2722 static int 2723 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2724 { 2725 struct mlx5_flow_destination dest = {}; 2726 struct mlx5_flow_act flow_act = {}; 2727 struct mlx5_flow_handle *handle; 2728 2729 if (ft_prio->anchor.rule_goto_table) 2730 return 0; 2731 2732 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2733 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 2734 flow_act.fg = ft_prio->anchor.fg_goto_table; 2735 2736 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2737 dest.ft = ft_prio->flow_table; 2738 2739 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2740 &dest, 1); 2741 if (IS_ERR(handle)) 2742 return PTR_ERR(handle); 2743 2744 ft_prio->anchor.rule_goto_table = handle; 2745 2746 return 0; 2747 } 2748 2749 static void 2750 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2751 { 2752 if (ft_prio->anchor.rule_goto_table) { 2753 mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table); 2754 ft_prio->anchor.rule_goto_table = NULL; 2755 } 2756 } 2757 2758 static int steering_anchor_create_res(struct mlx5_ib_dev *dev, 2759 struct mlx5_ib_flow_prio *ft_prio, 2760 enum mlx5_flow_namespace_type ns_type) 2761 { 2762 int err; 2763 2764 err = steering_anchor_create_ft(dev, ft_prio, ns_type); 2765 if (err) 2766 return err; 2767 2768 err = steering_anchor_create_fg_drop(ft_prio); 2769 if (err) 2770 goto destroy_ft; 2771 2772 err = steering_anchor_create_fg_goto_table(ft_prio); 2773 if (err) 2774 goto destroy_fg_drop; 2775 2776 err = steering_anchor_create_rule_drop(ft_prio); 2777 if (err) 2778 goto destroy_fg_goto_table; 2779 2780 err = steering_anchor_create_rule_goto_table(ft_prio); 2781 if (err) 2782 goto destroy_rule_drop; 2783 2784 return 0; 2785 2786 destroy_rule_drop: 2787 steering_anchor_destroy_rule_drop(ft_prio); 2788 destroy_fg_goto_table: 2789 steering_anchor_destroy_fg_goto_table(ft_prio); 2790 destroy_fg_drop: 2791 steering_anchor_destroy_fg_drop(ft_prio); 2792 destroy_ft: 2793 steering_anchor_destroy_ft(ft_prio); 2794 2795 return err; 2796 } 2797 2798 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio) 2799 { 2800 steering_anchor_destroy_rule_goto_table(ft_prio); 2801 steering_anchor_destroy_rule_drop(ft_prio); 2802 steering_anchor_destroy_fg_goto_table(ft_prio); 2803 steering_anchor_destroy_fg_drop(ft_prio); 2804 steering_anchor_destroy_ft(ft_prio); 2805 } 2806 2807 static int steering_anchor_cleanup(struct ib_uobject *uobject, 2808 enum rdma_remove_reason why, 2809 struct uverbs_attr_bundle *attrs) 2810 { 2811 struct mlx5_ib_steering_anchor *obj = uobject->object; 2812 2813 if (atomic_read(&obj->usecnt)) 2814 return -EBUSY; 2815 2816 mutex_lock(&obj->dev->flow_db->lock); 2817 if (!--obj->ft_prio->anchor.rule_goto_table_ref) 2818 steering_anchor_destroy_rule_goto_table(obj->ft_prio); 2819 2820 put_flow_table(obj->dev, obj->ft_prio, true); 2821 mutex_unlock(&obj->dev->flow_db->lock); 2822 2823 kfree(obj); 2824 return 0; 2825 } 2826 2827 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio, 2828 int count) 2829 { 2830 while (count--) 2831 mlx5_steering_anchor_destroy_res(&prio[count]); 2832 } 2833 2834 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev) 2835 { 2836 fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT); 2837 fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT); 2838 fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS); 2839 fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS); 2840 fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS); 2841 fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT); 2842 fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT); 2843 } 2844 2845 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2846 struct mlx5_ib_flow_matcher *obj) 2847 { 2848 enum mlx5_ib_uapi_flow_table_type ft_type = 2849 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2850 u32 flags; 2851 int err; 2852 2853 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2854 * users should switch to it. We leave this to not break userspace 2855 */ 2856 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2857 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2858 return -EINVAL; 2859 2860 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2861 err = uverbs_get_const(&ft_type, attrs, 2862 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2863 if (err) 2864 return err; 2865 2866 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2867 if (err) 2868 return err; 2869 2870 return 0; 2871 } 2872 2873 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2874 err = uverbs_get_flags32(&flags, attrs, 2875 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2876 IB_FLOW_ATTR_FLAGS_EGRESS); 2877 if (err) 2878 return err; 2879 2880 if (flags) 2881 return mlx5_ib_ft_type_to_namespace( 2882 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2883 &obj->ns_type); 2884 } 2885 2886 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2887 2888 return 0; 2889 } 2890 2891 static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps) 2892 { 2893 if (is_mdev_switchdev_mode(dev->mdev)) 2894 return UCAP_ENABLED(enabled_caps, 2895 RDMA_UCAP_MLX5_CTRL_OTHER_VHCA); 2896 2897 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL); 2898 } 2899 2900 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2901 struct uverbs_attr_bundle *attrs) 2902 { 2903 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2904 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2905 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2906 struct mlx5_ib_flow_matcher *obj; 2907 int err; 2908 2909 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2910 if (!obj) 2911 return -ENOMEM; 2912 2913 obj->mask_len = uverbs_attr_get_len( 2914 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2915 err = uverbs_copy_from(&obj->matcher_mask, 2916 attrs, 2917 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2918 if (err) 2919 goto end; 2920 2921 obj->flow_type = uverbs_attr_get_enum_id( 2922 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2923 2924 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2925 err = uverbs_copy_from(&obj->priority, 2926 attrs, 2927 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2928 if (err) 2929 goto end; 2930 } 2931 2932 err = uverbs_copy_from(&obj->match_criteria_enable, 2933 attrs, 2934 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2935 if (err) 2936 goto end; 2937 2938 err = mlx5_ib_matcher_ns(attrs, obj); 2939 if (err) 2940 goto end; 2941 2942 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2943 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { 2944 err = -EINVAL; 2945 goto end; 2946 } 2947 2948 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) { 2949 err = uverbs_copy_from(&obj->ib_port, attrs, 2950 MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT); 2951 if (err) 2952 goto end; 2953 if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) { 2954 err = -EINVAL; 2955 goto end; 2956 } 2957 if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX && 2958 obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) { 2959 err = -EINVAL; 2960 goto end; 2961 } 2962 if (!verify_context_caps(dev, uobj->context->enabled_caps)) { 2963 err = -EOPNOTSUPP; 2964 goto end; 2965 } 2966 } 2967 2968 uobj->object = obj; 2969 obj->mdev = dev->mdev; 2970 atomic_set(&obj->usecnt, 0); 2971 return 0; 2972 2973 end: 2974 kfree(obj); 2975 return err; 2976 } 2977 2978 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( 2979 struct uverbs_attr_bundle *attrs) 2980 { 2981 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2982 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE); 2983 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2984 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type; 2985 enum mlx5_flow_namespace_type ns_type; 2986 struct mlx5_ib_steering_anchor *obj; 2987 struct mlx5_ib_flow_prio *ft_prio; 2988 u16 priority; 2989 u32 ft_id; 2990 int err; 2991 2992 if (!capable(CAP_NET_RAW)) 2993 return -EPERM; 2994 2995 err = uverbs_get_const(&ib_uapi_ft_type, attrs, 2996 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE); 2997 if (err) 2998 return err; 2999 3000 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type); 3001 if (err) 3002 return err; 3003 3004 err = uverbs_copy_from(&priority, attrs, 3005 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY); 3006 if (err) 3007 return err; 3008 3009 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 3010 if (!obj) 3011 return -ENOMEM; 3012 3013 mutex_lock(&dev->flow_db->lock); 3014 3015 ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0); 3016 if (IS_ERR(ft_prio)) { 3017 err = PTR_ERR(ft_prio); 3018 goto free_obj; 3019 } 3020 3021 ft_prio->refcount++; 3022 3023 if (!ft_prio->anchor.rule_goto_table_ref) { 3024 err = steering_anchor_create_res(dev, ft_prio, ns_type); 3025 if (err) 3026 goto put_flow_table; 3027 } 3028 3029 ft_prio->anchor.rule_goto_table_ref++; 3030 3031 ft_id = mlx5_flow_table_id(ft_prio->anchor.ft); 3032 3033 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3034 &ft_id, sizeof(ft_id)); 3035 if (err) 3036 goto destroy_res; 3037 3038 mutex_unlock(&dev->flow_db->lock); 3039 3040 uobj->object = obj; 3041 obj->dev = dev; 3042 obj->ft_prio = ft_prio; 3043 atomic_set(&obj->usecnt, 0); 3044 3045 return 0; 3046 3047 destroy_res: 3048 --ft_prio->anchor.rule_goto_table_ref; 3049 mlx5_steering_anchor_destroy_res(ft_prio); 3050 put_flow_table: 3051 put_flow_table(dev, ft_prio, true); 3052 free_obj: 3053 mutex_unlock(&dev->flow_db->lock); 3054 kfree(obj); 3055 3056 return err; 3057 } 3058 3059 static struct ib_flow_action * 3060 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 3061 enum mlx5_ib_uapi_flow_table_type ft_type, 3062 u8 num_actions, void *in) 3063 { 3064 enum mlx5_flow_namespace_type namespace; 3065 struct mlx5_ib_flow_action *maction; 3066 int ret; 3067 3068 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3069 if (ret) 3070 return ERR_PTR(-EINVAL); 3071 3072 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3073 if (!maction) 3074 return ERR_PTR(-ENOMEM); 3075 3076 maction->flow_action_raw.modify_hdr = 3077 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 3078 3079 if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 3080 ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 3081 kfree(maction); 3082 return ERR_PTR(ret); 3083 } 3084 maction->flow_action_raw.sub_type = 3085 MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 3086 maction->flow_action_raw.dev = dev; 3087 3088 return &maction->ib_action; 3089 } 3090 3091 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 3092 { 3093 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3094 max_modify_header_actions) || 3095 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 3096 max_modify_header_actions) || 3097 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 3098 max_modify_header_actions); 3099 } 3100 3101 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 3102 struct uverbs_attr_bundle *attrs) 3103 { 3104 struct ib_uobject *uobj = uverbs_attr_get_uobject( 3105 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 3106 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3107 enum mlx5_ib_uapi_flow_table_type ft_type; 3108 struct ib_flow_action *action; 3109 int num_actions; 3110 void *in; 3111 int ret; 3112 3113 if (!mlx5_ib_modify_header_supported(mdev)) 3114 return -EOPNOTSUPP; 3115 3116 in = uverbs_attr_get_alloced_ptr(attrs, 3117 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 3118 3119 num_actions = uverbs_attr_ptr_get_array_size( 3120 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3121 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 3122 if (num_actions < 0) 3123 return num_actions; 3124 3125 ret = uverbs_get_const(&ft_type, attrs, 3126 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 3127 if (ret) 3128 return ret; 3129 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 3130 if (IS_ERR(action)) 3131 return PTR_ERR(action); 3132 3133 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 3134 IB_FLOW_ACTION_UNSPECIFIED); 3135 3136 return 0; 3137 } 3138 3139 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 3140 u8 packet_reformat_type, 3141 u8 ft_type) 3142 { 3143 switch (packet_reformat_type) { 3144 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3145 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3146 return MLX5_CAP_FLOWTABLE(ibdev->mdev, 3147 encap_general_header); 3148 break; 3149 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3150 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3151 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 3152 reformat_l2_to_l3_tunnel); 3153 break; 3154 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3155 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3156 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 3157 reformat_l3_tunnel_to_l2); 3158 break; 3159 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 3160 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3161 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 3162 break; 3163 default: 3164 break; 3165 } 3166 3167 return false; 3168 } 3169 3170 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 3171 { 3172 switch (dv_prt) { 3173 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3174 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 3175 break; 3176 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3177 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 3178 break; 3179 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3180 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 3181 break; 3182 default: 3183 return -EINVAL; 3184 } 3185 3186 return 0; 3187 } 3188 3189 static int mlx5_ib_flow_action_create_packet_reformat_ctx( 3190 struct mlx5_ib_dev *dev, 3191 struct mlx5_ib_flow_action *maction, 3192 u8 ft_type, u8 dv_prt, 3193 void *in, size_t len) 3194 { 3195 struct mlx5_pkt_reformat_params reformat_params; 3196 enum mlx5_flow_namespace_type namespace; 3197 u8 prm_prt; 3198 int ret; 3199 3200 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3201 if (ret) 3202 return ret; 3203 3204 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 3205 if (ret) 3206 return ret; 3207 3208 memset(&reformat_params, 0, sizeof(reformat_params)); 3209 reformat_params.type = prm_prt; 3210 reformat_params.size = len; 3211 reformat_params.data = in; 3212 maction->flow_action_raw.pkt_reformat = 3213 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params, 3214 namespace); 3215 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 3216 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 3217 return ret; 3218 } 3219 3220 maction->flow_action_raw.sub_type = 3221 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 3222 maction->flow_action_raw.dev = dev; 3223 3224 return 0; 3225 } 3226 3227 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 3228 struct uverbs_attr_bundle *attrs) 3229 { 3230 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 3231 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 3232 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3233 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 3234 enum mlx5_ib_uapi_flow_table_type ft_type; 3235 struct mlx5_ib_flow_action *maction; 3236 int ret; 3237 3238 ret = uverbs_get_const(&ft_type, attrs, 3239 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 3240 if (ret) 3241 return ret; 3242 3243 ret = uverbs_get_const(&dv_prt, attrs, 3244 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 3245 if (ret) 3246 return ret; 3247 3248 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 3249 return -EOPNOTSUPP; 3250 3251 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3252 if (!maction) 3253 return -ENOMEM; 3254 3255 if (dv_prt == 3256 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 3257 maction->flow_action_raw.sub_type = 3258 MLX5_IB_FLOW_ACTION_DECAP; 3259 maction->flow_action_raw.dev = mdev; 3260 } else { 3261 void *in; 3262 int len; 3263 3264 in = uverbs_attr_get_alloced_ptr(attrs, 3265 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3266 if (IS_ERR(in)) { 3267 ret = PTR_ERR(in); 3268 goto free_maction; 3269 } 3270 3271 len = uverbs_attr_get_len(attrs, 3272 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3273 3274 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 3275 maction, ft_type, dv_prt, in, len); 3276 if (ret) 3277 goto free_maction; 3278 } 3279 3280 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 3281 IB_FLOW_ACTION_UNSPECIFIED); 3282 return 0; 3283 3284 free_maction: 3285 kfree(maction); 3286 return ret; 3287 } 3288 3289 DECLARE_UVERBS_NAMED_METHOD( 3290 MLX5_IB_METHOD_CREATE_FLOW, 3291 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3292 UVERBS_OBJECT_FLOW, 3293 UVERBS_ACCESS_NEW, 3294 UA_MANDATORY), 3295 UVERBS_ATTR_PTR_IN( 3296 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 3297 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3298 UA_MANDATORY, 3299 UA_ALLOC_AND_COPY), 3300 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 3301 MLX5_IB_OBJECT_FLOW_MATCHER, 3302 UVERBS_ACCESS_READ, 3303 UA_MANDATORY), 3304 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 3305 UVERBS_OBJECT_QP, 3306 UVERBS_ACCESS_READ), 3307 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 3308 MLX5_IB_OBJECT_DEVX_OBJ, 3309 UVERBS_ACCESS_READ), 3310 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 3311 UVERBS_OBJECT_FLOW_ACTION, 3312 UVERBS_ACCESS_READ, 1, 3313 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 3314 UA_OPTIONAL), 3315 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 3316 UVERBS_ATTR_TYPE(u32), 3317 UA_OPTIONAL), 3318 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 3319 MLX5_IB_OBJECT_DEVX_OBJ, 3320 UVERBS_ACCESS_READ, 1, 1, 3321 UA_OPTIONAL), 3322 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 3323 UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 3324 UA_OPTIONAL, 3325 UA_ALLOC_AND_COPY), 3326 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 3327 enum mlx5_ib_create_flow_flags, 3328 UA_OPTIONAL)); 3329 3330 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3331 MLX5_IB_METHOD_DESTROY_FLOW, 3332 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3333 UVERBS_OBJECT_FLOW, 3334 UVERBS_ACCESS_DESTROY, 3335 UA_MANDATORY)); 3336 3337 ADD_UVERBS_METHODS(mlx5_ib_fs, 3338 UVERBS_OBJECT_FLOW, 3339 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 3340 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 3341 3342 DECLARE_UVERBS_NAMED_METHOD( 3343 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 3344 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 3345 UVERBS_OBJECT_FLOW_ACTION, 3346 UVERBS_ACCESS_NEW, 3347 UA_MANDATORY), 3348 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3349 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 3350 set_add_copy_action_in_auto)), 3351 UA_MANDATORY, 3352 UA_ALLOC_AND_COPY), 3353 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 3354 enum mlx5_ib_uapi_flow_table_type, 3355 UA_MANDATORY)); 3356 3357 DECLARE_UVERBS_NAMED_METHOD( 3358 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 3359 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 3360 UVERBS_OBJECT_FLOW_ACTION, 3361 UVERBS_ACCESS_NEW, 3362 UA_MANDATORY), 3363 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 3364 UVERBS_ATTR_MIN_SIZE(1), 3365 UA_ALLOC_AND_COPY, 3366 UA_OPTIONAL), 3367 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 3368 enum mlx5_ib_uapi_flow_action_packet_reformat_type, 3369 UA_MANDATORY), 3370 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 3371 enum mlx5_ib_uapi_flow_table_type, 3372 UA_MANDATORY)); 3373 3374 ADD_UVERBS_METHODS( 3375 mlx5_ib_flow_actions, 3376 UVERBS_OBJECT_FLOW_ACTION, 3377 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 3378 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 3379 3380 DECLARE_UVERBS_NAMED_METHOD( 3381 MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 3382 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 3383 MLX5_IB_OBJECT_FLOW_MATCHER, 3384 UVERBS_ACCESS_NEW, 3385 UA_MANDATORY), 3386 UVERBS_ATTR_PTR_IN( 3387 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 3388 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3389 UA_MANDATORY), 3390 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 3391 mlx5_ib_flow_type, 3392 UA_MANDATORY), 3393 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 3394 UVERBS_ATTR_TYPE(u8), 3395 UA_MANDATORY), 3396 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 3397 enum ib_flow_flags, 3398 UA_OPTIONAL), 3399 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 3400 enum mlx5_ib_uapi_flow_table_type, 3401 UA_OPTIONAL), 3402 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT, 3403 UVERBS_ATTR_TYPE(u32), 3404 UA_OPTIONAL)); 3405 3406 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3407 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 3408 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 3409 MLX5_IB_OBJECT_FLOW_MATCHER, 3410 UVERBS_ACCESS_DESTROY, 3411 UA_MANDATORY)); 3412 3413 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 3414 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 3415 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 3416 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 3417 3418 DECLARE_UVERBS_NAMED_METHOD( 3419 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE, 3420 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE, 3421 MLX5_IB_OBJECT_STEERING_ANCHOR, 3422 UVERBS_ACCESS_NEW, 3423 UA_MANDATORY), 3424 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE, 3425 enum mlx5_ib_uapi_flow_table_type, 3426 UA_MANDATORY), 3427 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY, 3428 UVERBS_ATTR_TYPE(u16), 3429 UA_MANDATORY), 3430 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3431 UVERBS_ATTR_TYPE(u32), 3432 UA_MANDATORY)); 3433 3434 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3435 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY, 3436 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE, 3437 MLX5_IB_OBJECT_STEERING_ANCHOR, 3438 UVERBS_ACCESS_DESTROY, 3439 UA_MANDATORY)); 3440 3441 DECLARE_UVERBS_NAMED_OBJECT( 3442 MLX5_IB_OBJECT_STEERING_ANCHOR, 3443 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup), 3444 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE), 3445 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY)); 3446 3447 const struct uapi_definition mlx5_ib_flow_defs[] = { 3448 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3449 MLX5_IB_OBJECT_FLOW_MATCHER), 3450 UAPI_DEF_CHAIN_OBJ_TREE( 3451 UVERBS_OBJECT_FLOW, 3452 &mlx5_ib_fs), 3453 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 3454 &mlx5_ib_flow_actions), 3455 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3456 MLX5_IB_OBJECT_STEERING_ANCHOR, 3457 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)), 3458 {}, 3459 }; 3460 3461 static const struct ib_device_ops flow_ops = { 3462 .create_flow = mlx5_ib_create_flow, 3463 .destroy_flow = mlx5_ib_destroy_flow, 3464 .destroy_flow_action = mlx5_ib_destroy_flow_action, 3465 }; 3466 3467 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 3468 { 3469 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 3470 3471 if (!dev->flow_db) 3472 return -ENOMEM; 3473 3474 dev->flow_db->rdma_transport_rx = kcalloc(dev->num_ports, 3475 sizeof(struct mlx5_ib_flow_prio), 3476 GFP_KERNEL); 3477 if (!dev->flow_db->rdma_transport_rx) 3478 goto free_flow_db; 3479 3480 dev->flow_db->rdma_transport_tx = kcalloc(dev->num_ports, 3481 sizeof(struct mlx5_ib_flow_prio), 3482 GFP_KERNEL); 3483 if (!dev->flow_db->rdma_transport_tx) 3484 goto free_rdma_transport_rx; 3485 3486 mutex_init(&dev->flow_db->lock); 3487 3488 ib_set_device_ops(&dev->ib_dev, &flow_ops); 3489 return 0; 3490 3491 free_rdma_transport_rx: 3492 kfree(dev->flow_db->rdma_transport_rx); 3493 free_flow_db: 3494 kfree(dev->flow_db); 3495 return -ENOMEM; 3496 } 3497