1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/uverbs_std_types.h> 11 #include <rdma/mlx5_user_ioctl_cmds.h> 12 #include <rdma/mlx5_user_ioctl_verbs.h> 13 #include <rdma/ib_hdrs.h> 14 #include <rdma/ib_umem.h> 15 #include <rdma/ib_ucaps.h> 16 #include <linux/mlx5/driver.h> 17 #include <linux/mlx5/fs.h> 18 #include <linux/mlx5/fs_helpers.h> 19 #include <linux/mlx5/eswitch.h> 20 #include <net/inet_ecn.h> 21 #include "mlx5_ib.h" 22 #include "counters.h" 23 #include "devx.h" 24 #include "fs.h" 25 26 #define UVERBS_MODULE_NAME mlx5_ib 27 #include <rdma/uverbs_named_ioctl.h> 28 29 enum { 30 MATCH_CRITERIA_ENABLE_OUTER_BIT, 31 MATCH_CRITERIA_ENABLE_MISC_BIT, 32 MATCH_CRITERIA_ENABLE_INNER_BIT, 33 MATCH_CRITERIA_ENABLE_MISC2_BIT 34 }; 35 36 37 struct mlx5_per_qp_opfc { 38 struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX]; 39 }; 40 41 #define HEADER_IS_ZERO(match_criteria, headers) \ 42 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 43 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 44 45 static u8 get_match_criteria_enable(u32 *match_criteria) 46 { 47 u8 match_criteria_enable; 48 49 match_criteria_enable = 50 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 51 MATCH_CRITERIA_ENABLE_OUTER_BIT; 52 match_criteria_enable |= 53 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 54 MATCH_CRITERIA_ENABLE_MISC_BIT; 55 match_criteria_enable |= 56 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 57 MATCH_CRITERIA_ENABLE_INNER_BIT; 58 match_criteria_enable |= 59 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 60 MATCH_CRITERIA_ENABLE_MISC2_BIT; 61 62 return match_criteria_enable; 63 } 64 65 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 66 { 67 u8 entry_mask; 68 u8 entry_val; 69 int err = 0; 70 71 if (!mask) 72 goto out; 73 74 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 75 ip_protocol); 76 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 77 ip_protocol); 78 if (!entry_mask) { 79 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 80 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 81 goto out; 82 } 83 /* Don't override existing ip protocol */ 84 if (mask != entry_mask || val != entry_val) 85 err = -EINVAL; 86 out: 87 return err; 88 } 89 90 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 91 bool inner) 92 { 93 if (inner) { 94 MLX5_SET(fte_match_set_misc, 95 misc_c, inner_ipv6_flow_label, mask); 96 MLX5_SET(fte_match_set_misc, 97 misc_v, inner_ipv6_flow_label, val); 98 } else { 99 MLX5_SET(fte_match_set_misc, 100 misc_c, outer_ipv6_flow_label, mask); 101 MLX5_SET(fte_match_set_misc, 102 misc_v, outer_ipv6_flow_label, val); 103 } 104 } 105 106 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 107 { 108 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 109 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 110 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 111 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 112 } 113 114 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 115 { 116 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 117 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 118 return -EOPNOTSUPP; 119 120 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 121 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 122 return -EOPNOTSUPP; 123 124 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 125 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 126 return -EOPNOTSUPP; 127 128 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 129 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 130 return -EOPNOTSUPP; 131 132 return 0; 133 } 134 135 #define LAST_ETH_FIELD vlan_tag 136 #define LAST_IPV4_FIELD tos 137 #define LAST_IPV6_FIELD traffic_class 138 #define LAST_TCP_UDP_FIELD src_port 139 #define LAST_TUNNEL_FIELD tunnel_id 140 #define LAST_FLOW_TAG_FIELD tag_id 141 #define LAST_DROP_FIELD size 142 #define LAST_COUNTERS_FIELD counters 143 144 /* Field is the last supported field */ 145 #define FIELDS_NOT_SUPPORTED(filter, field) \ 146 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ 147 sizeof(filter) - offsetofend(typeof(filter), field)) 148 149 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 150 bool is_egress, 151 struct mlx5_flow_act *action) 152 { 153 154 switch (maction->ib_action.type) { 155 case IB_FLOW_ACTION_UNSPECIFIED: 156 if (maction->flow_action_raw.sub_type == 157 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 158 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 159 return -EINVAL; 160 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 161 action->modify_hdr = 162 maction->flow_action_raw.modify_hdr; 163 return 0; 164 } 165 if (maction->flow_action_raw.sub_type == 166 MLX5_IB_FLOW_ACTION_DECAP) { 167 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 168 return -EINVAL; 169 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 170 return 0; 171 } 172 if (maction->flow_action_raw.sub_type == 173 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 174 if (action->action & 175 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 176 return -EINVAL; 177 action->action |= 178 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 179 action->pkt_reformat = 180 maction->flow_action_raw.pkt_reformat; 181 return 0; 182 } 183 fallthrough; 184 default: 185 return -EOPNOTSUPP; 186 } 187 } 188 189 static int parse_flow_attr(struct mlx5_core_dev *mdev, 190 struct mlx5_flow_spec *spec, 191 const union ib_flow_spec *ib_spec, 192 const struct ib_flow_attr *flow_attr, 193 struct mlx5_flow_act *action, u32 prev_type) 194 { 195 struct mlx5_flow_context *flow_context = &spec->flow_context; 196 u32 *match_c = spec->match_criteria; 197 u32 *match_v = spec->match_value; 198 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 199 misc_parameters); 200 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 201 misc_parameters); 202 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 203 misc_parameters_2); 204 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 205 misc_parameters_2); 206 void *headers_c; 207 void *headers_v; 208 int match_ipv; 209 int ret; 210 211 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 212 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 213 inner_headers); 214 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 215 inner_headers); 216 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 217 ft_field_support.inner_ip_version); 218 } else { 219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 220 outer_headers); 221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 222 outer_headers); 223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 224 ft_field_support.outer_ip_version); 225 } 226 227 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 228 case IB_FLOW_SPEC_ETH: 229 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 230 return -EOPNOTSUPP; 231 232 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 233 dmac_47_16), 234 ib_spec->eth.mask.dst_mac); 235 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 236 dmac_47_16), 237 ib_spec->eth.val.dst_mac); 238 239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 240 smac_47_16), 241 ib_spec->eth.mask.src_mac); 242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 243 smac_47_16), 244 ib_spec->eth.val.src_mac); 245 246 if (ib_spec->eth.mask.vlan_tag) { 247 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 248 cvlan_tag, 1); 249 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 250 cvlan_tag, 1); 251 252 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 253 first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 254 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 255 first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 256 257 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 258 first_cfi, 259 ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 260 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 261 first_cfi, 262 ntohs(ib_spec->eth.val.vlan_tag) >> 12); 263 264 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 265 first_prio, 266 ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 267 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 268 first_prio, 269 ntohs(ib_spec->eth.val.vlan_tag) >> 13); 270 } 271 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 272 ethertype, ntohs(ib_spec->eth.mask.ether_type)); 273 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 274 ethertype, ntohs(ib_spec->eth.val.ether_type)); 275 break; 276 case IB_FLOW_SPEC_IPV4: 277 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 278 return -EOPNOTSUPP; 279 280 if (match_ipv) { 281 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 282 ip_version, 0xf); 283 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 284 ip_version, MLX5_FS_IPV4_VERSION); 285 } else { 286 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 287 ethertype, 0xffff); 288 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 289 ethertype, ETH_P_IP); 290 } 291 292 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 293 src_ipv4_src_ipv6.ipv4_layout.ipv4), 294 &ib_spec->ipv4.mask.src_ip, 295 sizeof(ib_spec->ipv4.mask.src_ip)); 296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 297 src_ipv4_src_ipv6.ipv4_layout.ipv4), 298 &ib_spec->ipv4.val.src_ip, 299 sizeof(ib_spec->ipv4.val.src_ip)); 300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 301 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 302 &ib_spec->ipv4.mask.dst_ip, 303 sizeof(ib_spec->ipv4.mask.dst_ip)); 304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 306 &ib_spec->ipv4.val.dst_ip, 307 sizeof(ib_spec->ipv4.val.dst_ip)); 308 309 set_tos(headers_c, headers_v, 310 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 311 312 if (set_proto(headers_c, headers_v, 313 ib_spec->ipv4.mask.proto, 314 ib_spec->ipv4.val.proto)) 315 return -EINVAL; 316 break; 317 case IB_FLOW_SPEC_IPV6: 318 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 319 return -EOPNOTSUPP; 320 321 if (match_ipv) { 322 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 323 ip_version, 0xf); 324 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 325 ip_version, MLX5_FS_IPV6_VERSION); 326 } else { 327 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 328 ethertype, 0xffff); 329 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 330 ethertype, ETH_P_IPV6); 331 } 332 333 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 334 src_ipv4_src_ipv6.ipv6_layout.ipv6), 335 &ib_spec->ipv6.mask.src_ip, 336 sizeof(ib_spec->ipv6.mask.src_ip)); 337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 338 src_ipv4_src_ipv6.ipv6_layout.ipv6), 339 &ib_spec->ipv6.val.src_ip, 340 sizeof(ib_spec->ipv6.val.src_ip)); 341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 342 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 343 &ib_spec->ipv6.mask.dst_ip, 344 sizeof(ib_spec->ipv6.mask.dst_ip)); 345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 347 &ib_spec->ipv6.val.dst_ip, 348 sizeof(ib_spec->ipv6.val.dst_ip)); 349 350 set_tos(headers_c, headers_v, 351 ib_spec->ipv6.mask.traffic_class, 352 ib_spec->ipv6.val.traffic_class); 353 354 if (set_proto(headers_c, headers_v, 355 ib_spec->ipv6.mask.next_hdr, 356 ib_spec->ipv6.val.next_hdr)) 357 return -EINVAL; 358 359 set_flow_label(misc_params_c, misc_params_v, 360 ntohl(ib_spec->ipv6.mask.flow_label), 361 ntohl(ib_spec->ipv6.val.flow_label), 362 ib_spec->type & IB_FLOW_SPEC_INNER); 363 break; 364 case IB_FLOW_SPEC_ESP: 365 return -EOPNOTSUPP; 366 case IB_FLOW_SPEC_TCP: 367 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 368 LAST_TCP_UDP_FIELD)) 369 return -EOPNOTSUPP; 370 371 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 372 return -EINVAL; 373 374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 375 ntohs(ib_spec->tcp_udp.mask.src_port)); 376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 377 ntohs(ib_spec->tcp_udp.val.src_port)); 378 379 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 380 ntohs(ib_spec->tcp_udp.mask.dst_port)); 381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 382 ntohs(ib_spec->tcp_udp.val.dst_port)); 383 break; 384 case IB_FLOW_SPEC_UDP: 385 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 386 LAST_TCP_UDP_FIELD)) 387 return -EOPNOTSUPP; 388 389 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 390 return -EINVAL; 391 392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 393 ntohs(ib_spec->tcp_udp.mask.src_port)); 394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 395 ntohs(ib_spec->tcp_udp.val.src_port)); 396 397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 398 ntohs(ib_spec->tcp_udp.mask.dst_port)); 399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 400 ntohs(ib_spec->tcp_udp.val.dst_port)); 401 break; 402 case IB_FLOW_SPEC_GRE: 403 if (ib_spec->gre.mask.c_ks_res0_ver) 404 return -EOPNOTSUPP; 405 406 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 407 return -EINVAL; 408 409 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 410 0xff); 411 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 412 IPPROTO_GRE); 413 414 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 415 ntohs(ib_spec->gre.mask.protocol)); 416 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 417 ntohs(ib_spec->gre.val.protocol)); 418 419 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 420 gre_key.nvgre.hi), 421 &ib_spec->gre.mask.key, 422 sizeof(ib_spec->gre.mask.key)); 423 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 424 gre_key.nvgre.hi), 425 &ib_spec->gre.val.key, 426 sizeof(ib_spec->gre.val.key)); 427 break; 428 case IB_FLOW_SPEC_MPLS: 429 switch (prev_type) { 430 case IB_FLOW_SPEC_UDP: 431 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 432 ft_field_support.outer_first_mpls_over_udp), 433 &ib_spec->mpls.mask.tag)) 434 return -EOPNOTSUPP; 435 436 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 437 outer_first_mpls_over_udp), 438 &ib_spec->mpls.val.tag, 439 sizeof(ib_spec->mpls.val.tag)); 440 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 441 outer_first_mpls_over_udp), 442 &ib_spec->mpls.mask.tag, 443 sizeof(ib_spec->mpls.mask.tag)); 444 break; 445 case IB_FLOW_SPEC_GRE: 446 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 447 ft_field_support.outer_first_mpls_over_gre), 448 &ib_spec->mpls.mask.tag)) 449 return -EOPNOTSUPP; 450 451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 452 outer_first_mpls_over_gre), 453 &ib_spec->mpls.val.tag, 454 sizeof(ib_spec->mpls.val.tag)); 455 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 456 outer_first_mpls_over_gre), 457 &ib_spec->mpls.mask.tag, 458 sizeof(ib_spec->mpls.mask.tag)); 459 break; 460 default: 461 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 462 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 463 ft_field_support.inner_first_mpls), 464 &ib_spec->mpls.mask.tag)) 465 return -EOPNOTSUPP; 466 467 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 468 inner_first_mpls), 469 &ib_spec->mpls.val.tag, 470 sizeof(ib_spec->mpls.val.tag)); 471 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 472 inner_first_mpls), 473 &ib_spec->mpls.mask.tag, 474 sizeof(ib_spec->mpls.mask.tag)); 475 } else { 476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 477 ft_field_support.outer_first_mpls), 478 &ib_spec->mpls.mask.tag)) 479 return -EOPNOTSUPP; 480 481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 482 outer_first_mpls), 483 &ib_spec->mpls.val.tag, 484 sizeof(ib_spec->mpls.val.tag)); 485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 486 outer_first_mpls), 487 &ib_spec->mpls.mask.tag, 488 sizeof(ib_spec->mpls.mask.tag)); 489 } 490 } 491 break; 492 case IB_FLOW_SPEC_VXLAN_TUNNEL: 493 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 494 LAST_TUNNEL_FIELD)) 495 return -EOPNOTSUPP; 496 497 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 498 ntohl(ib_spec->tunnel.mask.tunnel_id)); 499 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 500 ntohl(ib_spec->tunnel.val.tunnel_id)); 501 break; 502 case IB_FLOW_SPEC_ACTION_TAG: 503 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 504 LAST_FLOW_TAG_FIELD)) 505 return -EOPNOTSUPP; 506 if (ib_spec->flow_tag.tag_id >= BIT(24)) 507 return -EINVAL; 508 509 flow_context->flow_tag = ib_spec->flow_tag.tag_id; 510 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 511 break; 512 case IB_FLOW_SPEC_ACTION_DROP: 513 if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 514 LAST_DROP_FIELD)) 515 return -EOPNOTSUPP; 516 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 517 break; 518 case IB_FLOW_SPEC_ACTION_HANDLE: 519 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 520 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 521 if (ret) 522 return ret; 523 break; 524 case IB_FLOW_SPEC_ACTION_COUNT: 525 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 526 LAST_COUNTERS_FIELD)) 527 return -EOPNOTSUPP; 528 529 /* for now support only one counters spec per flow */ 530 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 531 return -EINVAL; 532 533 action->counters = ib_spec->flow_count.counters; 534 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 535 break; 536 default: 537 return -EINVAL; 538 } 539 540 return 0; 541 } 542 543 /* If a flow could catch both multicast and unicast packets, 544 * it won't fall into the multicast flow steering table and this rule 545 * could steal other multicast packets. 546 */ 547 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 548 { 549 union ib_flow_spec *flow_spec; 550 551 if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 552 ib_attr->num_of_specs < 1) 553 return false; 554 555 flow_spec = (union ib_flow_spec *)(ib_attr + 1); 556 if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 557 struct ib_flow_spec_ipv4 *ipv4_spec; 558 559 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 560 if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 561 return true; 562 563 return false; 564 } 565 566 if (flow_spec->type == IB_FLOW_SPEC_ETH) { 567 struct ib_flow_spec_eth *eth_spec; 568 569 eth_spec = (struct ib_flow_spec_eth *)flow_spec; 570 return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 571 is_multicast_ether_addr(eth_spec->val.dst_mac); 572 } 573 574 return false; 575 } 576 577 static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 578 const struct ib_flow_attr *flow_attr, 579 bool check_inner) 580 { 581 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 582 int match_ipv = check_inner ? 583 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 584 ft_field_support.inner_ip_version) : 585 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 586 ft_field_support.outer_ip_version); 587 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 588 bool ipv4_spec_valid, ipv6_spec_valid; 589 unsigned int ip_spec_type = 0; 590 bool has_ethertype = false; 591 unsigned int spec_index; 592 bool mask_valid = true; 593 u16 eth_type = 0; 594 bool type_valid; 595 596 /* Validate that ethertype is correct */ 597 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 598 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 599 ib_spec->eth.mask.ether_type) { 600 mask_valid = (ib_spec->eth.mask.ether_type == 601 htons(0xffff)); 602 has_ethertype = true; 603 eth_type = ntohs(ib_spec->eth.val.ether_type); 604 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 605 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 606 ip_spec_type = ib_spec->type; 607 } 608 ib_spec = (void *)ib_spec + ib_spec->size; 609 } 610 611 type_valid = (!has_ethertype) || (!ip_spec_type); 612 if (!type_valid && mask_valid) { 613 ipv4_spec_valid = (eth_type == ETH_P_IP) && 614 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 615 ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 616 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 617 618 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 619 (((eth_type == ETH_P_MPLS_UC) || 620 (eth_type == ETH_P_MPLS_MC)) && match_ipv); 621 } 622 623 return type_valid; 624 } 625 626 static bool is_valid_attr(struct mlx5_core_dev *mdev, 627 const struct ib_flow_attr *flow_attr) 628 { 629 return is_valid_ethertype(mdev, flow_attr, false) && 630 is_valid_ethertype(mdev, flow_attr, true); 631 } 632 633 static void put_flow_table(struct mlx5_ib_dev *dev, 634 struct mlx5_ib_flow_prio *prio, bool ft_added) 635 { 636 prio->refcount -= !!ft_added; 637 if (!prio->refcount) { 638 mlx5_destroy_flow_table(prio->flow_table); 639 prio->flow_table = NULL; 640 } 641 } 642 643 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 644 { 645 struct mlx5_ib_flow_handler *handler = container_of(flow_id, 646 struct mlx5_ib_flow_handler, 647 ibflow); 648 struct mlx5_ib_flow_handler *iter, *tmp; 649 struct mlx5_ib_dev *dev = handler->dev; 650 651 mutex_lock(&dev->flow_db->lock); 652 653 list_for_each_entry_safe(iter, tmp, &handler->list, list) { 654 mlx5_del_flow_rules(iter->rule); 655 put_flow_table(dev, iter->prio, true); 656 list_del(&iter->list); 657 kfree(iter); 658 } 659 660 mlx5_del_flow_rules(handler->rule); 661 put_flow_table(dev, handler->prio, true); 662 mlx5_ib_counters_clear_description(handler->ibcounters); 663 mutex_unlock(&dev->flow_db->lock); 664 if (handler->flow_matcher) 665 atomic_dec(&handler->flow_matcher->usecnt); 666 kfree(handler); 667 668 return 0; 669 } 670 671 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 672 { 673 priority *= 2; 674 if (!dont_trap) 675 priority++; 676 return priority; 677 } 678 679 enum flow_table_type { 680 MLX5_IB_FT_RX, 681 MLX5_IB_FT_TX 682 }; 683 684 #define MLX5_FS_MAX_TYPES 6 685 #define MLX5_FS_MAX_ENTRIES BIT(16) 686 687 static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device) 688 { 689 struct mlx5_ib_dev *dev = to_mdev(device); 690 691 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed); 692 } 693 694 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_ib_dev *dev, 695 struct mlx5_flow_namespace *ns, 696 struct mlx5_ib_flow_prio *prio, 697 int priority, 698 int num_entries, int num_groups, 699 u32 flags, u16 vport) 700 { 701 struct mlx5_flow_table_attr ft_attr = {}; 702 struct mlx5_flow_table *ft; 703 704 ft_attr.prio = priority; 705 ft_attr.max_fte = num_entries; 706 ft_attr.flags = flags; 707 ft_attr.vport = vport; 708 ft_attr.autogroup.max_num_groups = num_groups; 709 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 710 if (IS_ERR(ft)) 711 return ERR_CAST(ft); 712 713 prio->flow_table = ft; 714 prio->refcount = 0; 715 return prio; 716 } 717 718 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 719 struct ib_flow_attr *flow_attr, 720 enum flow_table_type ft_type) 721 { 722 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 723 struct mlx5_flow_namespace *ns = NULL; 724 enum mlx5_flow_namespace_type fn_type; 725 struct mlx5_ib_flow_prio *prio; 726 struct mlx5_flow_table *ft; 727 int max_table_size; 728 int num_entries; 729 int num_groups; 730 bool esw_encap; 731 u32 flags = 0; 732 int priority; 733 734 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 735 log_max_ft_size)); 736 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 737 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 738 switch (flow_attr->type) { 739 case IB_FLOW_ATTR_NORMAL: 740 if (flow_is_multicast_only(flow_attr) && !dont_trap) 741 priority = MLX5_IB_FLOW_MCAST_PRIO; 742 else 743 priority = ib_prio_to_core_prio(flow_attr->priority, 744 dont_trap); 745 if (ft_type == MLX5_IB_FT_RX) { 746 fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 747 prio = &dev->flow_db->prios[priority]; 748 if (!dev->is_rep && !esw_encap && 749 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 750 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 751 if (!dev->is_rep && !esw_encap && 752 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 753 reformat_l3_tunnel_to_l2)) 754 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 755 } else { 756 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( 757 dev->mdev, log_max_ft_size)); 758 fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 759 prio = &dev->flow_db->egress_prios[priority]; 760 if (!dev->is_rep && !esw_encap && 761 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 762 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 763 } 764 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 765 num_entries = MLX5_FS_MAX_ENTRIES; 766 num_groups = MLX5_FS_MAX_TYPES; 767 break; 768 case IB_FLOW_ATTR_ALL_DEFAULT: 769 case IB_FLOW_ATTR_MC_DEFAULT: 770 ns = mlx5_get_flow_namespace(dev->mdev, 771 MLX5_FLOW_NAMESPACE_LEFTOVERS); 772 build_leftovers_ft_param(&priority, &num_entries, &num_groups); 773 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 774 break; 775 case IB_FLOW_ATTR_SNIFFER: 776 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 777 allow_sniffer_and_nic_rx_shared_tir)) 778 return ERR_PTR(-EOPNOTSUPP); 779 780 ns = mlx5_get_flow_namespace( 781 dev->mdev, ft_type == MLX5_IB_FT_RX ? 782 MLX5_FLOW_NAMESPACE_SNIFFER_RX : 783 MLX5_FLOW_NAMESPACE_SNIFFER_TX); 784 785 prio = &dev->flow_db->sniffer[ft_type]; 786 priority = 0; 787 num_entries = 1; 788 num_groups = 1; 789 break; 790 default: 791 break; 792 } 793 794 if (!ns) 795 return ERR_PTR(-EOPNOTSUPP); 796 797 max_table_size = min_t(int, num_entries, max_table_size); 798 799 ft = prio->flow_table; 800 if (!ft) 801 return _get_prio(dev, ns, prio, priority, max_table_size, 802 num_groups, flags, 0); 803 804 return prio; 805 } 806 807 enum { 808 RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO, 809 RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO, 810 RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 811 RDMA_RX_ECN_OPCOUNTER_PRIO, 812 RDMA_RX_CNP_OPCOUNTER_PRIO, 813 RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO, 814 }; 815 816 enum { 817 RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO, 818 RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 819 RDMA_TX_CNP_OPCOUNTER_PRIO, 820 RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO, 821 }; 822 823 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num, 824 struct mlx5_flow_spec *spec) 825 { 826 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 827 ft_field_support.source_vhca_port) || 828 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 829 ft_field_support.source_vhca_port)) 830 return -EOPNOTSUPP; 831 832 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria, 833 misc_parameters.source_vhca_port); 834 MLX5_SET(fte_match_param, &spec->match_value, 835 misc_parameters.source_vhca_port, port_num); 836 837 return 0; 838 } 839 840 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num, 841 struct mlx5_flow_spec *spec, int ipv) 842 { 843 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 844 ft_field_support.outer_ip_version)) 845 return -EOPNOTSUPP; 846 847 if (mlx5_core_mp_enabled(dev->mdev) && 848 set_vhca_port_spec(dev, port_num, spec)) 849 return -EOPNOTSUPP; 850 851 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 852 outer_headers.ip_ecn); 853 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn, 854 INET_ECN_CE); 855 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 856 outer_headers.ip_version); 857 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 858 ipv); 859 860 spec->match_criteria_enable = 861 get_match_criteria_enable(spec->match_criteria); 862 863 return 0; 864 } 865 866 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num, 867 struct mlx5_flow_spec *spec) 868 { 869 if (mlx5_core_mp_enabled(dev->mdev) && 870 set_vhca_port_spec(dev, port_num, spec)) 871 return -EOPNOTSUPP; 872 873 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 874 misc_parameters.bth_opcode); 875 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode, 876 IB_BTH_OPCODE_CNP); 877 878 spec->match_criteria_enable = 879 get_match_criteria_enable(spec->match_criteria); 880 881 return 0; 882 } 883 884 /* Returns the prio we should use for the given optional counter type, 885 * whereas for bytes type we use the packet type, since they share the same 886 * resources. 887 */ 888 static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev, 889 u32 type) 890 { 891 u32 prio_type; 892 893 switch (type) { 894 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 895 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS; 896 break; 897 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 898 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS; 899 break; 900 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 901 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 902 break; 903 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 904 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 905 break; 906 default: 907 prio_type = type; 908 } 909 910 return &dev->flow_db->opfcs[prio_type]; 911 } 912 913 static void put_per_qp_prio(struct mlx5_ib_dev *dev, 914 enum mlx5_ib_optional_counter_type type) 915 { 916 enum mlx5_ib_optional_counter_type per_qp_type; 917 struct mlx5_ib_flow_prio *prio; 918 919 switch (type) { 920 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 921 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 922 break; 923 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 924 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 925 break; 926 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 927 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 928 break; 929 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 930 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 931 break; 932 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 933 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 934 break; 935 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 936 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 937 break; 938 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 939 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 940 break; 941 default: 942 return; 943 } 944 945 prio = get_opfc_prio(dev, per_qp_type); 946 put_flow_table(dev, prio, true); 947 } 948 949 static int get_per_qp_prio(struct mlx5_ib_dev *dev, 950 enum mlx5_ib_optional_counter_type type) 951 { 952 enum mlx5_ib_optional_counter_type per_qp_type; 953 enum mlx5_flow_namespace_type fn_type; 954 struct mlx5_flow_namespace *ns; 955 struct mlx5_ib_flow_prio *prio; 956 int priority; 957 958 switch (type) { 959 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 960 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 961 priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO; 962 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 963 break; 964 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 965 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 966 priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO; 967 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 968 break; 969 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 970 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 971 priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO; 972 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 973 break; 974 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 975 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 976 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 977 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 978 break; 979 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 980 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 981 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 982 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 983 break; 984 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 985 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 986 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 987 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 988 break; 989 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 990 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 991 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 992 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 993 break; 994 default: 995 return -EINVAL; 996 } 997 998 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 999 if (!ns) 1000 return -EOPNOTSUPP; 1001 1002 prio = get_opfc_prio(dev, per_qp_type); 1003 if (prio->flow_table) 1004 return 0; 1005 1006 prio = _get_prio(dev, ns, prio, priority, MLX5_FS_MAX_POOL_SIZE, 1, 0, 0); 1007 if (IS_ERR(prio)) 1008 return PTR_ERR(prio); 1009 1010 prio->refcount = 1; 1011 1012 return 0; 1013 } 1014 1015 static struct mlx5_per_qp_opfc * 1016 get_per_qp_opfc(struct mlx5_rdma_counter *mcounter, u32 qp_num, bool *new) 1017 { 1018 struct mlx5_per_qp_opfc *per_qp_opfc; 1019 1020 *new = false; 1021 1022 per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp_num); 1023 if (per_qp_opfc) 1024 return per_qp_opfc; 1025 per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL); 1026 1027 if (!per_qp_opfc) 1028 return NULL; 1029 1030 *new = true; 1031 return per_qp_opfc; 1032 } 1033 1034 static int add_op_fc_rules(struct mlx5_ib_dev *dev, 1035 struct mlx5_rdma_counter *mcounter, 1036 struct mlx5_per_qp_opfc *per_qp_opfc, 1037 struct mlx5_ib_flow_prio *prio, 1038 enum mlx5_ib_optional_counter_type type, 1039 u32 qp_num, u32 port_num) 1040 { 1041 struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc; 1042 struct mlx5_flow_act flow_act = {}; 1043 struct mlx5_flow_destination dst; 1044 struct mlx5_flow_spec *spec; 1045 int i, err, spec_num; 1046 bool is_tx; 1047 1048 if (opfc->fc) 1049 return -EEXIST; 1050 1051 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type, 1052 &in_use_opfc)) { 1053 opfc->fc = in_use_opfc->fc; 1054 opfc->rule[0] = in_use_opfc->rule[0]; 1055 return 0; 1056 } 1057 1058 opfc->fc = mcounter->fc[type]; 1059 1060 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1061 if (!spec) { 1062 err = -ENOMEM; 1063 goto null_fc; 1064 } 1065 1066 switch (type) { 1067 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP: 1068 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1069 MLX5_FS_IPV4_VERSION) || 1070 set_ecn_ce_spec(dev, port_num, &spec[1], 1071 MLX5_FS_IPV6_VERSION)) { 1072 err = -EOPNOTSUPP; 1073 goto free_spec; 1074 } 1075 spec_num = 2; 1076 is_tx = false; 1077 1078 MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria, 1079 misc_parameters.bth_dst_qp); 1080 MLX5_SET(fte_match_param, spec[1].match_value, 1081 misc_parameters.bth_dst_qp, qp_num); 1082 spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1083 break; 1084 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP: 1085 if (!MLX5_CAP_FLOWTABLE( 1086 dev->mdev, 1087 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1088 set_cnp_spec(dev, port_num, &spec[0])) { 1089 err = -EOPNOTSUPP; 1090 goto free_spec; 1091 } 1092 spec_num = 1; 1093 is_tx = false; 1094 break; 1095 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP: 1096 if (!MLX5_CAP_FLOWTABLE( 1097 dev->mdev, 1098 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1099 set_cnp_spec(dev, port_num, &spec[0])) { 1100 err = -EOPNOTSUPP; 1101 goto free_spec; 1102 } 1103 spec_num = 1; 1104 is_tx = true; 1105 break; 1106 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1107 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1108 spec_num = 1; 1109 is_tx = true; 1110 break; 1111 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1112 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1113 spec_num = 1; 1114 is_tx = false; 1115 break; 1116 default: 1117 err = -EINVAL; 1118 goto free_spec; 1119 } 1120 1121 if (is_tx) { 1122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1123 misc_parameters.source_sqn); 1124 MLX5_SET(fte_match_param, spec->match_value, 1125 misc_parameters.source_sqn, qp_num); 1126 } else { 1127 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1128 misc_parameters.bth_dst_qp); 1129 MLX5_SET(fte_match_param, spec->match_value, 1130 misc_parameters.bth_dst_qp, qp_num); 1131 } 1132 1133 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1134 1135 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1136 dst.counter = opfc->fc; 1137 1138 flow_act.action = 1139 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1140 1141 for (i = 0; i < spec_num; i++) { 1142 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1143 &flow_act, &dst, 1); 1144 if (IS_ERR(opfc->rule[i])) { 1145 err = PTR_ERR(opfc->rule[i]); 1146 goto del_rules; 1147 } 1148 } 1149 prio->refcount += spec_num; 1150 1151 err = xa_err(xa_store(&mcounter->qpn_opfc_xa, qp_num, per_qp_opfc, 1152 GFP_KERNEL)); 1153 if (err) 1154 goto del_rules; 1155 1156 kfree(spec); 1157 1158 return 0; 1159 1160 del_rules: 1161 while (i--) 1162 mlx5_del_flow_rules(opfc->rule[i]); 1163 put_flow_table(dev, prio, false); 1164 free_spec: 1165 kfree(spec); 1166 null_fc: 1167 opfc->fc = NULL; 1168 return err; 1169 } 1170 1171 static bool is_fc_shared_and_in_use(struct mlx5_rdma_counter *mcounter, 1172 u32 type, struct mlx5_fc **fc) 1173 { 1174 u32 shared_fc_type; 1175 1176 switch (type) { 1177 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1178 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 1179 break; 1180 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1181 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 1182 break; 1183 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1184 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 1185 break; 1186 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1187 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 1188 break; 1189 default: 1190 return false; 1191 } 1192 1193 *fc = mcounter->fc[shared_fc_type]; 1194 if (!(*fc)) 1195 return false; 1196 1197 return true; 1198 } 1199 1200 void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev, 1201 struct rdma_counter *counter) 1202 { 1203 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 1204 struct mlx5_fc *in_use_fc; 1205 int i; 1206 1207 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1208 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1209 if (!mcounter->fc[i]) 1210 continue; 1211 1212 if (is_fc_shared_and_in_use(mcounter, i, &in_use_fc)) { 1213 mcounter->fc[i] = NULL; 1214 continue; 1215 } 1216 1217 mlx5_fc_destroy(dev->mdev, mcounter->fc[i]); 1218 mcounter->fc[i] = NULL; 1219 } 1220 } 1221 1222 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num, 1223 struct mlx5_ib_op_fc *opfc, 1224 enum mlx5_ib_optional_counter_type type) 1225 { 1226 enum mlx5_flow_namespace_type fn_type; 1227 int priority, i, err, spec_num; 1228 struct mlx5_flow_act flow_act = {}; 1229 struct mlx5_flow_destination dst; 1230 struct mlx5_flow_namespace *ns; 1231 struct mlx5_ib_flow_prio *prio; 1232 struct mlx5_flow_spec *spec; 1233 1234 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1235 if (!spec) 1236 return -ENOMEM; 1237 1238 switch (type) { 1239 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 1240 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1241 MLX5_FS_IPV4_VERSION) || 1242 set_ecn_ce_spec(dev, port_num, &spec[1], 1243 MLX5_FS_IPV6_VERSION)) { 1244 err = -EOPNOTSUPP; 1245 goto free; 1246 } 1247 spec_num = 2; 1248 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1249 priority = RDMA_RX_ECN_OPCOUNTER_PRIO; 1250 break; 1251 1252 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 1253 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1254 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1255 set_cnp_spec(dev, port_num, &spec[0])) { 1256 err = -EOPNOTSUPP; 1257 goto free; 1258 } 1259 spec_num = 1; 1260 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1261 priority = RDMA_RX_CNP_OPCOUNTER_PRIO; 1262 break; 1263 1264 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 1265 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1266 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1267 set_cnp_spec(dev, port_num, &spec[0])) { 1268 err = -EOPNOTSUPP; 1269 goto free; 1270 } 1271 spec_num = 1; 1272 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1273 priority = RDMA_TX_CNP_OPCOUNTER_PRIO; 1274 break; 1275 1276 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 1277 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 1278 spec_num = 1; 1279 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1280 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO; 1281 break; 1282 1283 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 1284 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 1285 spec_num = 1; 1286 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1287 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO; 1288 break; 1289 1290 default: 1291 err = -EOPNOTSUPP; 1292 goto free; 1293 } 1294 1295 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 1296 if (!ns) { 1297 err = -EOPNOTSUPP; 1298 goto free; 1299 } 1300 1301 prio = get_opfc_prio(dev, type); 1302 if (!prio->flow_table) { 1303 err = get_per_qp_prio(dev, type); 1304 if (err) 1305 goto free; 1306 1307 prio = _get_prio(dev, ns, prio, priority, 1308 dev->num_ports * MAX_OPFC_RULES, 1, 0, 0); 1309 if (IS_ERR(prio)) { 1310 err = PTR_ERR(prio); 1311 goto put_prio; 1312 } 1313 } 1314 1315 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1316 dst.counter = opfc->fc; 1317 1318 flow_act.action = 1319 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1320 1321 for (i = 0; i < spec_num; i++) { 1322 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1323 &flow_act, &dst, 1); 1324 if (IS_ERR(opfc->rule[i])) { 1325 err = PTR_ERR(opfc->rule[i]); 1326 goto del_rules; 1327 } 1328 } 1329 prio->refcount += spec_num; 1330 kfree(spec); 1331 1332 return 0; 1333 1334 del_rules: 1335 for (i -= 1; i >= 0; i--) 1336 mlx5_del_flow_rules(opfc->rule[i]); 1337 put_flow_table(dev, prio, false); 1338 put_prio: 1339 put_per_qp_prio(dev, type); 1340 free: 1341 kfree(spec); 1342 return err; 1343 } 1344 1345 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev, 1346 struct mlx5_ib_op_fc *opfc, 1347 enum mlx5_ib_optional_counter_type type) 1348 { 1349 struct mlx5_ib_flow_prio *prio; 1350 int i; 1351 1352 prio = get_opfc_prio(dev, type); 1353 1354 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) { 1355 mlx5_del_flow_rules(opfc->rule[i]); 1356 put_flow_table(dev, prio, true); 1357 } 1358 1359 put_per_qp_prio(dev, type); 1360 } 1361 1362 void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct rdma_counter *counter) 1363 { 1364 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 1365 struct mlx5_ib_dev *dev = to_mdev(counter->device); 1366 struct mlx5_per_qp_opfc *per_qp_opfc; 1367 struct mlx5_ib_op_fc *in_use_opfc; 1368 struct mlx5_ib_flow_prio *prio; 1369 int i, j; 1370 1371 per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp->qp_num); 1372 if (!per_qp_opfc) 1373 return; 1374 1375 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1376 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1377 if (!per_qp_opfc->opfcs[i].fc) 1378 continue; 1379 1380 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i, 1381 &in_use_opfc)) { 1382 per_qp_opfc->opfcs[i].fc = NULL; 1383 continue; 1384 } 1385 1386 for (j = 0; j < MAX_OPFC_RULES; j++) { 1387 if (!per_qp_opfc->opfcs[i].rule[j]) 1388 continue; 1389 mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]); 1390 prio = get_opfc_prio(dev, i); 1391 put_flow_table(dev, prio, true); 1392 } 1393 per_qp_opfc->opfcs[i].fc = NULL; 1394 } 1395 1396 kfree(per_qp_opfc); 1397 xa_erase(&mcounter->qpn_opfc_xa, qp->qp_num); 1398 } 1399 1400 int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter, 1401 u32 port) 1402 { 1403 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 1404 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1405 struct mlx5_per_qp_opfc *per_qp_opfc; 1406 struct mlx5_ib_flow_prio *prio; 1407 struct mlx5_ib_counters *cnts; 1408 struct mlx5_ib_op_fc *opfc; 1409 struct mlx5_fc *in_use_fc; 1410 int i, err, per_qp_type; 1411 bool new; 1412 1413 if (!counter->mode.bind_opcnt) 1414 return 0; 1415 1416 cnts = &dev->port[port - 1].cnts; 1417 1418 for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) { 1419 opfc = &cnts->opfcs[i]; 1420 if (!opfc->fc) 1421 continue; 1422 1423 per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1424 prio = get_opfc_prio(dev, per_qp_type); 1425 WARN_ON(!prio->flow_table); 1426 1427 if (is_fc_shared_and_in_use(mcounter, per_qp_type, &in_use_fc)) 1428 mcounter->fc[per_qp_type] = in_use_fc; 1429 1430 if (!mcounter->fc[per_qp_type]) { 1431 mcounter->fc[per_qp_type] = mlx5_fc_create(dev->mdev, 1432 false); 1433 if (IS_ERR(mcounter->fc[per_qp_type])) 1434 return PTR_ERR(mcounter->fc[per_qp_type]); 1435 } 1436 1437 per_qp_opfc = get_per_qp_opfc(mcounter, qp->qp_num, &new); 1438 if (!per_qp_opfc) { 1439 err = -ENOMEM; 1440 goto free_fc; 1441 } 1442 err = add_op_fc_rules(dev, mcounter, per_qp_opfc, prio, 1443 per_qp_type, qp->qp_num, port); 1444 if (err) 1445 goto del_rules; 1446 } 1447 1448 return 0; 1449 1450 del_rules: 1451 mlx5r_fs_unbind_op_fc(qp, counter); 1452 if (new) 1453 kfree(per_qp_opfc); 1454 free_fc: 1455 if (xa_empty(&mcounter->qpn_opfc_xa)) 1456 mlx5r_fs_destroy_fcs(dev, counter); 1457 return err; 1458 } 1459 1460 static void set_underlay_qp(struct mlx5_ib_dev *dev, 1461 struct mlx5_flow_spec *spec, 1462 u32 underlay_qpn) 1463 { 1464 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 1465 spec->match_criteria, 1466 misc_parameters); 1467 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1468 misc_parameters); 1469 1470 if (underlay_qpn && 1471 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1472 ft_field_support.bth_dst_qp)) { 1473 MLX5_SET(fte_match_set_misc, 1474 misc_params_v, bth_dst_qp, underlay_qpn); 1475 MLX5_SET(fte_match_set_misc, 1476 misc_params_c, bth_dst_qp, 0xffffff); 1477 } 1478 } 1479 1480 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 1481 struct mlx5_flow_spec *spec, 1482 struct mlx5_eswitch_rep *rep) 1483 { 1484 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 1485 void *misc; 1486 1487 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1488 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1489 misc_parameters_2); 1490 1491 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1492 mlx5_eswitch_get_vport_metadata_for_match(rep->esw, 1493 rep->vport)); 1494 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1495 misc_parameters_2); 1496 1497 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1498 mlx5_eswitch_get_vport_metadata_mask()); 1499 } else { 1500 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1501 misc_parameters); 1502 1503 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 1504 1505 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1506 misc_parameters); 1507 1508 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1509 } 1510 } 1511 1512 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 1513 struct mlx5_ib_flow_prio *ft_prio, 1514 const struct ib_flow_attr *flow_attr, 1515 struct mlx5_flow_destination *dst, 1516 u32 underlay_qpn, 1517 struct mlx5_ib_create_flow *ucmd) 1518 { 1519 struct mlx5_flow_table *ft = ft_prio->flow_table; 1520 struct mlx5_ib_flow_handler *handler; 1521 struct mlx5_flow_act flow_act = {}; 1522 struct mlx5_flow_spec *spec; 1523 struct mlx5_flow_destination dest_arr[2] = {}; 1524 struct mlx5_flow_destination *rule_dst = dest_arr; 1525 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 1526 unsigned int spec_index; 1527 u32 prev_type = 0; 1528 int err = 0; 1529 int dest_num = 0; 1530 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1531 1532 if (!is_valid_attr(dev->mdev, flow_attr)) 1533 return ERR_PTR(-EINVAL); 1534 1535 if (dev->is_rep && is_egress) 1536 return ERR_PTR(-EINVAL); 1537 1538 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1539 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1540 if (!handler || !spec) { 1541 err = -ENOMEM; 1542 goto free; 1543 } 1544 1545 INIT_LIST_HEAD(&handler->list); 1546 1547 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 1548 err = parse_flow_attr(dev->mdev, spec, 1549 ib_flow, flow_attr, &flow_act, 1550 prev_type); 1551 if (err < 0) 1552 goto free; 1553 1554 prev_type = ((union ib_flow_spec *)ib_flow)->type; 1555 ib_flow += ((union ib_flow_spec *)ib_flow)->size; 1556 } 1557 1558 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 1559 memcpy(&dest_arr[0], dst, sizeof(*dst)); 1560 dest_num++; 1561 } 1562 1563 if (!flow_is_multicast_only(flow_attr)) 1564 set_underlay_qp(dev, spec, underlay_qpn); 1565 1566 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { 1567 struct mlx5_eswitch_rep *rep; 1568 1569 rep = dev->port[flow_attr->port - 1].rep; 1570 if (!rep) { 1571 err = -EINVAL; 1572 goto free; 1573 } 1574 1575 mlx5_ib_set_rule_source_port(dev, spec, rep); 1576 } 1577 1578 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 1579 1580 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1581 struct mlx5_ib_mcounters *mcounters; 1582 1583 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 1584 if (err) 1585 goto free; 1586 1587 mcounters = to_mcounters(flow_act.counters); 1588 handler->ibcounters = flow_act.counters; 1589 dest_arr[dest_num].type = 1590 MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1591 dest_arr[dest_num].counter = 1592 mcounters->hw_cntrs_hndl; 1593 dest_num++; 1594 } 1595 1596 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 1597 if (!dest_num) 1598 rule_dst = NULL; 1599 } else { 1600 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 1601 flow_act.action |= 1602 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1603 if (is_egress) 1604 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1605 else if (dest_num) 1606 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1607 } 1608 1609 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1610 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1611 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1612 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1613 spec->flow_context.flow_tag, flow_attr->type); 1614 err = -EINVAL; 1615 goto free; 1616 } 1617 handler->rule = mlx5_add_flow_rules(ft, spec, 1618 &flow_act, 1619 rule_dst, dest_num); 1620 1621 if (IS_ERR(handler->rule)) { 1622 err = PTR_ERR(handler->rule); 1623 goto free; 1624 } 1625 1626 ft_prio->refcount++; 1627 handler->prio = ft_prio; 1628 handler->dev = dev; 1629 1630 ft_prio->flow_table = ft; 1631 free: 1632 if (err && handler) { 1633 mlx5_ib_counters_clear_description(handler->ibcounters); 1634 kfree(handler); 1635 } 1636 kvfree(spec); 1637 return err ? ERR_PTR(err) : handler; 1638 } 1639 1640 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1641 struct mlx5_ib_flow_prio *ft_prio, 1642 const struct ib_flow_attr *flow_attr, 1643 struct mlx5_flow_destination *dst) 1644 { 1645 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1646 } 1647 1648 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1649 struct mlx5_ib_flow_prio *ft_prio, 1650 struct ib_flow_attr *flow_attr, 1651 struct mlx5_flow_destination *dst) 1652 { 1653 struct mlx5_ib_flow_handler *handler_ucast = NULL; 1654 struct mlx5_ib_flow_handler *handler = NULL; 1655 1656 static struct { 1657 struct ib_flow_spec_eth eth_flow; 1658 struct ib_flow_attr flow_attr; 1659 } leftovers_wc = { .flow_attr = { .num_of_specs = 1, 1660 .size = sizeof(leftovers_wc) }, 1661 .eth_flow = { 1662 .type = IB_FLOW_SPEC_ETH, 1663 .size = sizeof(struct ib_flow_spec_eth), 1664 .mask = { .dst_mac = { 0x1 } }, 1665 .val = { .dst_mac = { 0x1 } } } }; 1666 1667 static struct { 1668 struct ib_flow_spec_eth eth_flow; 1669 struct ib_flow_attr flow_attr; 1670 } leftovers_uc = { .flow_attr = { .num_of_specs = 1, 1671 .size = sizeof(leftovers_uc) }, 1672 .eth_flow = { 1673 .type = IB_FLOW_SPEC_ETH, 1674 .size = sizeof(struct ib_flow_spec_eth), 1675 .mask = { .dst_mac = { 0x1 } }, 1676 .val = { .dst_mac = {} } } }; 1677 1678 handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst); 1679 if (!IS_ERR(handler) && 1680 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1681 handler_ucast = create_flow_rule(dev, ft_prio, 1682 &leftovers_uc.flow_attr, dst); 1683 if (IS_ERR(handler_ucast)) { 1684 mlx5_del_flow_rules(handler->rule); 1685 ft_prio->refcount--; 1686 kfree(handler); 1687 handler = handler_ucast; 1688 } else { 1689 list_add(&handler_ucast->list, &handler->list); 1690 } 1691 } 1692 1693 return handler; 1694 } 1695 1696 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1697 struct mlx5_ib_flow_prio *ft_rx, 1698 struct mlx5_ib_flow_prio *ft_tx, 1699 struct mlx5_flow_destination *dst) 1700 { 1701 struct mlx5_ib_flow_handler *handler_rx; 1702 struct mlx5_ib_flow_handler *handler_tx; 1703 int err; 1704 static const struct ib_flow_attr flow_attr = { 1705 .num_of_specs = 0, 1706 .type = IB_FLOW_ATTR_SNIFFER, 1707 .size = sizeof(flow_attr) 1708 }; 1709 1710 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1711 if (IS_ERR(handler_rx)) { 1712 err = PTR_ERR(handler_rx); 1713 goto err; 1714 } 1715 1716 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1717 if (IS_ERR(handler_tx)) { 1718 err = PTR_ERR(handler_tx); 1719 goto err_tx; 1720 } 1721 1722 list_add(&handler_tx->list, &handler_rx->list); 1723 1724 return handler_rx; 1725 1726 err_tx: 1727 mlx5_del_flow_rules(handler_rx->rule); 1728 ft_rx->refcount--; 1729 kfree(handler_rx); 1730 err: 1731 return ERR_PTR(err); 1732 } 1733 1734 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1735 struct ib_flow_attr *flow_attr, 1736 struct ib_udata *udata) 1737 { 1738 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1739 struct mlx5_ib_qp *mqp = to_mqp(qp); 1740 struct mlx5_ib_flow_handler *handler = NULL; 1741 struct mlx5_flow_destination *dst = NULL; 1742 struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1743 struct mlx5_ib_flow_prio *ft_prio; 1744 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1745 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1746 size_t min_ucmd_sz, required_ucmd_sz; 1747 int err; 1748 int underlay_qpn; 1749 1750 if (udata && udata->inlen) { 1751 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); 1752 if (udata->inlen < min_ucmd_sz) 1753 return ERR_PTR(-EOPNOTSUPP); 1754 1755 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1756 if (err) 1757 return ERR_PTR(err); 1758 1759 /* currently supports only one counters data */ 1760 if (ucmd_hdr.ncounters_data > 1) 1761 return ERR_PTR(-EINVAL); 1762 1763 required_ucmd_sz = min_ucmd_sz + 1764 sizeof(struct mlx5_ib_flow_counters_data) * 1765 ucmd_hdr.ncounters_data; 1766 if (udata->inlen > required_ucmd_sz && 1767 !ib_is_udata_cleared(udata, required_ucmd_sz, 1768 udata->inlen - required_ucmd_sz)) 1769 return ERR_PTR(-EOPNOTSUPP); 1770 1771 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1772 if (!ucmd) 1773 return ERR_PTR(-ENOMEM); 1774 1775 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1776 if (err) 1777 goto free_ucmd; 1778 } 1779 1780 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1781 err = -ENOMEM; 1782 goto free_ucmd; 1783 } 1784 1785 if (flow_attr->flags & 1786 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) { 1787 err = -EINVAL; 1788 goto free_ucmd; 1789 } 1790 1791 if (is_egress && 1792 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1793 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1794 err = -EINVAL; 1795 goto free_ucmd; 1796 } 1797 1798 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1799 if (!dst) { 1800 err = -ENOMEM; 1801 goto free_ucmd; 1802 } 1803 1804 mutex_lock(&dev->flow_db->lock); 1805 1806 ft_prio = get_flow_table(dev, flow_attr, 1807 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1808 if (IS_ERR(ft_prio)) { 1809 err = PTR_ERR(ft_prio); 1810 goto unlock; 1811 } 1812 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1813 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1814 if (IS_ERR(ft_prio_tx)) { 1815 err = PTR_ERR(ft_prio_tx); 1816 ft_prio_tx = NULL; 1817 goto destroy_ft; 1818 } 1819 } 1820 1821 if (is_egress) { 1822 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1823 } else { 1824 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1825 if (mqp->is_rss) 1826 dst->tir_num = mqp->rss_qp.tirn; 1827 else 1828 dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1829 } 1830 1831 switch (flow_attr->type) { 1832 case IB_FLOW_ATTR_NORMAL: 1833 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1834 mqp->underlay_qpn : 1835 0; 1836 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1837 underlay_qpn, ucmd); 1838 break; 1839 case IB_FLOW_ATTR_ALL_DEFAULT: 1840 case IB_FLOW_ATTR_MC_DEFAULT: 1841 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); 1842 break; 1843 case IB_FLOW_ATTR_SNIFFER: 1844 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1845 break; 1846 default: 1847 err = -EINVAL; 1848 goto destroy_ft; 1849 } 1850 1851 if (IS_ERR(handler)) { 1852 err = PTR_ERR(handler); 1853 handler = NULL; 1854 goto destroy_ft; 1855 } 1856 1857 mutex_unlock(&dev->flow_db->lock); 1858 kfree(dst); 1859 kfree(ucmd); 1860 1861 return &handler->ibflow; 1862 1863 destroy_ft: 1864 put_flow_table(dev, ft_prio, false); 1865 if (ft_prio_tx) 1866 put_flow_table(dev, ft_prio_tx, false); 1867 unlock: 1868 mutex_unlock(&dev->flow_db->lock); 1869 kfree(dst); 1870 free_ucmd: 1871 kfree(ucmd); 1872 return ERR_PTR(err); 1873 } 1874 1875 static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev, 1876 enum mlx5_flow_namespace_type type, 1877 u32 *flags, u16 *vport_idx, 1878 u16 *vport, 1879 struct mlx5_core_dev **ft_mdev, 1880 u32 ib_port) 1881 { 1882 struct mlx5_core_dev *esw_mdev; 1883 1884 if (!is_mdev_switchdev_mode(dev->mdev)) 1885 return 0; 1886 1887 if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager)) 1888 return -EOPNOTSUPP; 1889 1890 if (!dev->port[ib_port - 1].rep) 1891 return -EINVAL; 1892 1893 esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw); 1894 if (esw_mdev != dev->mdev) 1895 return -EOPNOTSUPP; 1896 1897 *flags |= MLX5_FLOW_TABLE_OTHER_VPORT; 1898 *ft_mdev = esw_mdev; 1899 *vport = dev->port[ib_port - 1].rep->vport; 1900 *vport_idx = dev->port[ib_port - 1].rep->vport_index; 1901 1902 return 0; 1903 } 1904 1905 static struct mlx5_ib_flow_prio * 1906 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority, 1907 enum mlx5_flow_namespace_type ns_type, 1908 bool mcast, u32 ib_port) 1909 { 1910 struct mlx5_core_dev *ft_mdev = dev->mdev; 1911 struct mlx5_flow_namespace *ns = NULL; 1912 struct mlx5_ib_flow_prio *prio = NULL; 1913 int max_table_size = 0; 1914 u16 vport_idx = 0; 1915 bool esw_encap; 1916 u32 flags = 0; 1917 u16 vport = 0; 1918 int priority; 1919 int ret; 1920 1921 if (mcast) 1922 priority = MLX5_IB_FLOW_MCAST_PRIO; 1923 else 1924 priority = ib_prio_to_core_prio(user_priority, false); 1925 1926 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1927 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1928 switch (ns_type) { 1929 case MLX5_FLOW_NAMESPACE_BYPASS: 1930 max_table_size = BIT( 1931 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); 1932 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1933 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1934 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1935 reformat_l3_tunnel_to_l2) && 1936 !esw_encap) 1937 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1938 break; 1939 case MLX5_FLOW_NAMESPACE_EGRESS: 1940 max_table_size = BIT( 1941 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1942 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && 1943 !esw_encap) 1944 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1945 break; 1946 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1947 max_table_size = BIT( 1948 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1949 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1950 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1951 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, 1952 reformat_l3_tunnel_to_l2) && 1953 esw_encap) 1954 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1955 priority = user_priority; 1956 break; 1957 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1958 max_table_size = BIT( 1959 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); 1960 priority = user_priority; 1961 break; 1962 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1963 max_table_size = BIT( 1964 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); 1965 priority = user_priority; 1966 break; 1967 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 1968 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 1969 if (ib_port == 0 || 1970 user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO) 1971 return ERR_PTR(-EINVAL); 1972 ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags, 1973 &vport_idx, &vport, 1974 &ft_mdev, ib_port); 1975 if (ret) 1976 return ERR_PTR(ret); 1977 1978 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) 1979 max_table_size = 1980 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX( 1981 ft_mdev, log_max_ft_size)); 1982 else 1983 max_table_size = 1984 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX( 1985 ft_mdev, log_max_ft_size)); 1986 priority = user_priority; 1987 break; 1988 default: 1989 break; 1990 } 1991 1992 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1993 1994 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX || 1995 ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) 1996 ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx); 1997 else 1998 ns = mlx5_get_flow_namespace(ft_mdev, ns_type); 1999 2000 if (!ns) 2001 return ERR_PTR(-EOPNOTSUPP); 2002 2003 switch (ns_type) { 2004 case MLX5_FLOW_NAMESPACE_BYPASS: 2005 prio = &dev->flow_db->prios[priority]; 2006 break; 2007 case MLX5_FLOW_NAMESPACE_EGRESS: 2008 prio = &dev->flow_db->egress_prios[priority]; 2009 break; 2010 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 2011 prio = &dev->flow_db->fdb[priority]; 2012 break; 2013 case MLX5_FLOW_NAMESPACE_RDMA_RX: 2014 prio = &dev->flow_db->rdma_rx[priority]; 2015 break; 2016 case MLX5_FLOW_NAMESPACE_RDMA_TX: 2017 prio = &dev->flow_db->rdma_tx[priority]; 2018 break; 2019 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 2020 prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1]; 2021 break; 2022 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 2023 prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1]; 2024 break; 2025 default: return ERR_PTR(-EINVAL); 2026 } 2027 2028 if (!prio) 2029 return ERR_PTR(-EINVAL); 2030 2031 if (prio->flow_table) 2032 return prio; 2033 2034 return _get_prio(dev, ns, prio, priority, max_table_size, 2035 MLX5_FS_MAX_TYPES, flags, vport); 2036 } 2037 2038 static struct mlx5_ib_flow_handler * 2039 _create_raw_flow_rule(struct mlx5_ib_dev *dev, 2040 struct mlx5_ib_flow_prio *ft_prio, 2041 struct mlx5_flow_destination *dst, 2042 struct mlx5_ib_flow_matcher *fs_matcher, 2043 struct mlx5_flow_context *flow_context, 2044 struct mlx5_flow_act *flow_act, 2045 void *cmd_in, int inlen, 2046 int dst_num) 2047 { 2048 struct mlx5_ib_flow_handler *handler; 2049 struct mlx5_flow_spec *spec; 2050 struct mlx5_flow_table *ft = ft_prio->flow_table; 2051 int err = 0; 2052 2053 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2054 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 2055 if (!handler || !spec) { 2056 err = -ENOMEM; 2057 goto free; 2058 } 2059 2060 INIT_LIST_HEAD(&handler->list); 2061 2062 memcpy(spec->match_value, cmd_in, inlen); 2063 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 2064 fs_matcher->mask_len); 2065 spec->match_criteria_enable = fs_matcher->match_criteria_enable; 2066 spec->flow_context = *flow_context; 2067 2068 handler->rule = mlx5_add_flow_rules(ft, spec, 2069 flow_act, dst, dst_num); 2070 2071 if (IS_ERR(handler->rule)) { 2072 err = PTR_ERR(handler->rule); 2073 goto free; 2074 } 2075 2076 ft_prio->refcount++; 2077 handler->prio = ft_prio; 2078 handler->dev = dev; 2079 ft_prio->flow_table = ft; 2080 2081 free: 2082 if (err) 2083 kfree(handler); 2084 kvfree(spec); 2085 return err ? ERR_PTR(err) : handler; 2086 } 2087 2088 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 2089 void *match_v) 2090 { 2091 void *match_c; 2092 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 2093 void *dmac, *dmac_mask; 2094 void *ipv4, *ipv4_mask; 2095 2096 if (!(fs_matcher->match_criteria_enable & 2097 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 2098 return false; 2099 2100 match_c = fs_matcher->matcher_mask.match_params; 2101 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 2102 outer_headers); 2103 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 2104 outer_headers); 2105 2106 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2107 dmac_47_16); 2108 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2109 dmac_47_16); 2110 2111 if (is_multicast_ether_addr(dmac) && 2112 is_multicast_ether_addr(dmac_mask)) 2113 return true; 2114 2115 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2116 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2117 2118 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2119 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2120 2121 if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 2122 ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 2123 return true; 2124 2125 return false; 2126 } 2127 2128 static struct mlx5_ib_flow_handler *raw_fs_rule_add( 2129 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 2130 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 2131 struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type) 2132 { 2133 struct mlx5_flow_destination *dst; 2134 struct mlx5_ib_flow_prio *ft_prio; 2135 struct mlx5_ib_flow_handler *handler; 2136 int dst_num = 0; 2137 bool mcast; 2138 int err; 2139 2140 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 2141 return ERR_PTR(-EOPNOTSUPP); 2142 2143 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 2144 return ERR_PTR(-ENOMEM); 2145 2146 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 2147 if (!dst) 2148 return ERR_PTR(-ENOMEM); 2149 2150 mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 2151 mutex_lock(&dev->flow_db->lock); 2152 2153 ft_prio = _get_flow_table(dev, fs_matcher->priority, 2154 fs_matcher->ns_type, mcast, 2155 fs_matcher->ib_port); 2156 if (IS_ERR(ft_prio)) { 2157 err = PTR_ERR(ft_prio); 2158 goto unlock; 2159 } 2160 2161 switch (dest_type) { 2162 case MLX5_FLOW_DESTINATION_TYPE_TIR: 2163 dst[dst_num].type = dest_type; 2164 dst[dst_num++].tir_num = dest_id; 2165 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2166 break; 2167 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: 2168 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 2169 dst[dst_num++].ft_num = dest_id; 2170 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2171 break; 2172 case MLX5_FLOW_DESTINATION_TYPE_PORT: 2173 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2174 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 2175 break; 2176 default: 2177 break; 2178 } 2179 2180 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 2181 if (WARN_ON(!counter)) { 2182 err = -EINVAL; 2183 goto unlock; 2184 } 2185 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2186 dst[dst_num].counter = counter; 2187 dst_num++; 2188 } 2189 2190 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, 2191 fs_matcher, flow_context, flow_act, 2192 cmd_in, inlen, dst_num); 2193 2194 if (IS_ERR(handler)) { 2195 err = PTR_ERR(handler); 2196 goto destroy_ft; 2197 } 2198 2199 mutex_unlock(&dev->flow_db->lock); 2200 atomic_inc(&fs_matcher->usecnt); 2201 handler->flow_matcher = fs_matcher; 2202 2203 kfree(dst); 2204 2205 return handler; 2206 2207 destroy_ft: 2208 put_flow_table(dev, ft_prio, false); 2209 unlock: 2210 mutex_unlock(&dev->flow_db->lock); 2211 kfree(dst); 2212 2213 return ERR_PTR(err); 2214 } 2215 2216 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 2217 { 2218 switch (maction->flow_action_raw.sub_type) { 2219 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 2220 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 2221 maction->flow_action_raw.modify_hdr); 2222 break; 2223 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 2224 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 2225 maction->flow_action_raw.pkt_reformat); 2226 break; 2227 case MLX5_IB_FLOW_ACTION_DECAP: 2228 break; 2229 default: 2230 break; 2231 } 2232 } 2233 2234 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 2235 { 2236 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 2237 2238 switch (action->type) { 2239 case IB_FLOW_ACTION_UNSPECIFIED: 2240 destroy_flow_action_raw(maction); 2241 break; 2242 default: 2243 WARN_ON(true); 2244 break; 2245 } 2246 2247 kfree(maction); 2248 return 0; 2249 } 2250 2251 static int 2252 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 2253 enum mlx5_flow_namespace_type *namespace) 2254 { 2255 switch (table_type) { 2256 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 2257 *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 2258 break; 2259 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 2260 *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 2261 break; 2262 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 2263 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS; 2264 break; 2265 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 2266 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 2267 break; 2268 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 2269 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 2270 break; 2271 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX: 2272 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX; 2273 break; 2274 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX: 2275 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX; 2276 break; 2277 default: 2278 return -EINVAL; 2279 } 2280 2281 return 0; 2282 } 2283 2284 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 2285 [MLX5_IB_FLOW_TYPE_NORMAL] = { 2286 .type = UVERBS_ATTR_TYPE_PTR_IN, 2287 .u.ptr = { 2288 .len = sizeof(u16), /* data is priority */ 2289 .min_len = sizeof(u16), 2290 } 2291 }, 2292 [MLX5_IB_FLOW_TYPE_SNIFFER] = { 2293 .type = UVERBS_ATTR_TYPE_PTR_IN, 2294 UVERBS_ATTR_NO_DATA(), 2295 }, 2296 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 2297 .type = UVERBS_ATTR_TYPE_PTR_IN, 2298 UVERBS_ATTR_NO_DATA(), 2299 }, 2300 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 2301 .type = UVERBS_ATTR_TYPE_PTR_IN, 2302 UVERBS_ATTR_NO_DATA(), 2303 }, 2304 }; 2305 2306 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 2307 { 2308 struct devx_obj *devx_obj = obj; 2309 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2310 2311 switch (opcode) { 2312 case MLX5_CMD_OP_DESTROY_TIR: 2313 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2314 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 2315 obj_id); 2316 return true; 2317 2318 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 2319 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2320 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 2321 table_id); 2322 return true; 2323 default: 2324 return false; 2325 } 2326 } 2327 2328 static int get_dests(struct uverbs_attr_bundle *attrs, 2329 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 2330 int *dest_type, struct ib_qp **qp, u32 *flags) 2331 { 2332 bool dest_devx, dest_qp; 2333 void *devx_obj; 2334 int err; 2335 2336 dest_devx = uverbs_attr_is_valid(attrs, 2337 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2338 dest_qp = uverbs_attr_is_valid(attrs, 2339 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2340 2341 *flags = 0; 2342 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2343 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 2344 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 2345 if (err) 2346 return err; 2347 2348 /* Both flags are not allowed */ 2349 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 2350 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2351 return -EINVAL; 2352 2353 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 2354 if (dest_devx && (dest_qp || *flags)) 2355 return -EINVAL; 2356 else if (dest_qp && *flags) 2357 return -EINVAL; 2358 } 2359 2360 /* Allow only DEVX object, drop as dest for FDB */ 2361 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2362 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 2363 return -EINVAL; 2364 2365 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 2366 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2367 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2368 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 2369 return -EINVAL; 2370 2371 *qp = NULL; 2372 if (dest_devx) { 2373 devx_obj = 2374 uverbs_attr_get_obj(attrs, 2375 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2376 2377 /* Verify that the given DEVX object is a flow 2378 * steering destination. 2379 */ 2380 if (!is_flow_dest(devx_obj, dest_id, dest_type)) 2381 return -EINVAL; 2382 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 2383 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS || 2384 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2385 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2386 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 2387 return -EINVAL; 2388 } else if (dest_qp) { 2389 struct mlx5_ib_qp *mqp; 2390 2391 *qp = uverbs_attr_get_obj(attrs, 2392 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2393 if (IS_ERR(*qp)) 2394 return PTR_ERR(*qp); 2395 2396 if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 2397 return -EINVAL; 2398 2399 mqp = to_mqp(*qp); 2400 if (mqp->is_rss) 2401 *dest_id = mqp->rss_qp.tirn; 2402 else 2403 *dest_id = mqp->raw_packet_qp.rq.tirn; 2404 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2405 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2406 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2407 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) && 2408 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { 2409 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2410 } 2411 2412 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 2413 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2414 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2415 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)) 2416 return -EINVAL; 2417 2418 return 0; 2419 } 2420 2421 static bool 2422 is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size) 2423 { 2424 struct devx_obj *devx_obj = obj; 2425 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2426 2427 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 2428 2429 if (offset && offset >= devx_obj->flow_counter_bulk_size) 2430 return false; 2431 2432 *fc_bulk_size = devx_obj->flow_counter_bulk_size; 2433 *counter_id = MLX5_GET(dealloc_flow_counter_in, 2434 devx_obj->dinbox, 2435 flow_counter_id); 2436 *counter_id += offset; 2437 return true; 2438 } 2439 2440 return false; 2441 } 2442 2443 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 2444 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 2445 struct uverbs_attr_bundle *attrs) 2446 { 2447 struct mlx5_flow_context flow_context = {.flow_tag = 2448 MLX5_FS_DEFAULT_FLOW_TAG}; 2449 int dest_id, dest_type = -1, inlen, len, ret, i; 2450 struct mlx5_ib_flow_handler *flow_handler; 2451 struct mlx5_ib_flow_matcher *fs_matcher; 2452 struct ib_uobject **arr_flow_actions; 2453 struct ib_uflow_resources *uflow_res; 2454 struct mlx5_flow_act flow_act = {}; 2455 struct mlx5_fc *counter = NULL; 2456 struct ib_qp *qp = NULL; 2457 void *devx_obj, *cmd_in; 2458 struct ib_uobject *uobj; 2459 struct mlx5_ib_dev *dev; 2460 u32 flags; 2461 2462 if (!rdma_uattrs_has_raw_cap(attrs)) 2463 return -EPERM; 2464 2465 fs_matcher = uverbs_attr_get_obj(attrs, 2466 MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 2467 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 2468 dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2469 2470 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 2471 return -EINVAL; 2472 2473 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 2474 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 2475 2476 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2477 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 2478 2479 len = uverbs_attr_get_uobjs_arr(attrs, 2480 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 2481 if (len) { 2482 u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0; 2483 devx_obj = arr_flow_actions[0]->object; 2484 2485 if (uverbs_attr_is_valid(attrs, 2486 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 2487 2488 int num_offsets = uverbs_attr_ptr_get_array_size( 2489 attrs, 2490 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2491 sizeof(u32)); 2492 2493 if (num_offsets != 1) 2494 return -EINVAL; 2495 2496 offset_attr = uverbs_attr_get_alloced_ptr( 2497 attrs, 2498 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 2499 offset = *offset_attr; 2500 } 2501 2502 if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size)) 2503 return -EINVAL; 2504 counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size); 2505 if (IS_ERR(counter)) 2506 return PTR_ERR(counter); 2507 2508 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 2509 } 2510 2511 cmd_in = uverbs_attr_get_alloced_ptr( 2512 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2513 inlen = uverbs_attr_get_len(attrs, 2514 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2515 2516 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 2517 if (!uflow_res) { 2518 ret = -ENOMEM; 2519 goto destroy_counter; 2520 } 2521 2522 len = uverbs_attr_get_uobjs_arr(attrs, 2523 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 2524 for (i = 0; i < len; i++) { 2525 struct mlx5_ib_flow_action *maction = 2526 to_mflow_act(arr_flow_actions[i]->object); 2527 2528 ret = parse_flow_flow_action(maction, false, &flow_act); 2529 if (ret) 2530 goto err_out; 2531 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 2532 arr_flow_actions[i]->object); 2533 } 2534 2535 ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 2536 MLX5_IB_ATTR_CREATE_FLOW_TAG); 2537 if (!ret) { 2538 if (flow_context.flow_tag >= BIT(24)) { 2539 ret = -EINVAL; 2540 goto err_out; 2541 } 2542 flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 2543 } 2544 2545 flow_handler = 2546 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 2547 counter, cmd_in, inlen, dest_id, dest_type); 2548 if (IS_ERR(flow_handler)) { 2549 ret = PTR_ERR(flow_handler); 2550 goto err_out; 2551 } 2552 2553 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2554 2555 return 0; 2556 err_out: 2557 ib_uverbs_flow_resources_free(uflow_res); 2558 destroy_counter: 2559 if (counter) 2560 mlx5_fc_local_destroy(counter); 2561 return ret; 2562 } 2563 2564 static int flow_matcher_cleanup(struct ib_uobject *uobject, 2565 enum rdma_remove_reason why, 2566 struct uverbs_attr_bundle *attrs) 2567 { 2568 struct mlx5_ib_flow_matcher *obj = uobject->object; 2569 2570 if (atomic_read(&obj->usecnt)) 2571 return -EBUSY; 2572 2573 kfree(obj); 2574 return 0; 2575 } 2576 2577 static int steering_anchor_create_ft(struct mlx5_ib_dev *dev, 2578 struct mlx5_ib_flow_prio *ft_prio, 2579 enum mlx5_flow_namespace_type ns_type) 2580 { 2581 struct mlx5_flow_table_attr ft_attr = {}; 2582 struct mlx5_flow_namespace *ns; 2583 struct mlx5_flow_table *ft; 2584 2585 if (ft_prio->anchor.ft) 2586 return 0; 2587 2588 ns = mlx5_get_flow_namespace(dev->mdev, ns_type); 2589 if (!ns) 2590 return -EOPNOTSUPP; 2591 2592 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 2593 ft_attr.uid = MLX5_SHARED_RESOURCE_UID; 2594 ft_attr.prio = 0; 2595 ft_attr.max_fte = 2; 2596 ft_attr.level = 1; 2597 2598 ft = mlx5_create_flow_table(ns, &ft_attr); 2599 if (IS_ERR(ft)) 2600 return PTR_ERR(ft); 2601 2602 ft_prio->anchor.ft = ft; 2603 2604 return 0; 2605 } 2606 2607 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio) 2608 { 2609 if (ft_prio->anchor.ft) { 2610 mlx5_destroy_flow_table(ft_prio->anchor.ft); 2611 ft_prio->anchor.ft = NULL; 2612 } 2613 } 2614 2615 static int 2616 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2617 { 2618 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2619 struct mlx5_flow_group *fg; 2620 void *flow_group_in; 2621 int err = 0; 2622 2623 if (ft_prio->anchor.fg_drop) 2624 return 0; 2625 2626 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2627 if (!flow_group_in) 2628 return -ENOMEM; 2629 2630 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1); 2631 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); 2632 2633 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2634 if (IS_ERR(fg)) { 2635 err = PTR_ERR(fg); 2636 goto out; 2637 } 2638 2639 ft_prio->anchor.fg_drop = fg; 2640 2641 out: 2642 kvfree(flow_group_in); 2643 2644 return err; 2645 } 2646 2647 static void 2648 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2649 { 2650 if (ft_prio->anchor.fg_drop) { 2651 mlx5_destroy_flow_group(ft_prio->anchor.fg_drop); 2652 ft_prio->anchor.fg_drop = NULL; 2653 } 2654 } 2655 2656 static int 2657 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2658 { 2659 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2660 struct mlx5_flow_group *fg; 2661 void *flow_group_in; 2662 int err = 0; 2663 2664 if (ft_prio->anchor.fg_goto_table) 2665 return 0; 2666 2667 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2668 if (!flow_group_in) 2669 return -ENOMEM; 2670 2671 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2672 if (IS_ERR(fg)) { 2673 err = PTR_ERR(fg); 2674 goto out; 2675 } 2676 ft_prio->anchor.fg_goto_table = fg; 2677 2678 out: 2679 kvfree(flow_group_in); 2680 2681 return err; 2682 } 2683 2684 static void 2685 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2686 { 2687 if (ft_prio->anchor.fg_goto_table) { 2688 mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table); 2689 ft_prio->anchor.fg_goto_table = NULL; 2690 } 2691 } 2692 2693 static int 2694 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2695 { 2696 struct mlx5_flow_act flow_act = {}; 2697 struct mlx5_flow_handle *handle; 2698 2699 if (ft_prio->anchor.rule_drop) 2700 return 0; 2701 2702 flow_act.fg = ft_prio->anchor.fg_drop; 2703 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2704 2705 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2706 NULL, 0); 2707 if (IS_ERR(handle)) 2708 return PTR_ERR(handle); 2709 2710 ft_prio->anchor.rule_drop = handle; 2711 2712 return 0; 2713 } 2714 2715 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2716 { 2717 if (ft_prio->anchor.rule_drop) { 2718 mlx5_del_flow_rules(ft_prio->anchor.rule_drop); 2719 ft_prio->anchor.rule_drop = NULL; 2720 } 2721 } 2722 2723 static int 2724 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2725 { 2726 struct mlx5_flow_destination dest = {}; 2727 struct mlx5_flow_act flow_act = {}; 2728 struct mlx5_flow_handle *handle; 2729 2730 if (ft_prio->anchor.rule_goto_table) 2731 return 0; 2732 2733 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2734 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 2735 flow_act.fg = ft_prio->anchor.fg_goto_table; 2736 2737 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2738 dest.ft = ft_prio->flow_table; 2739 2740 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2741 &dest, 1); 2742 if (IS_ERR(handle)) 2743 return PTR_ERR(handle); 2744 2745 ft_prio->anchor.rule_goto_table = handle; 2746 2747 return 0; 2748 } 2749 2750 static void 2751 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2752 { 2753 if (ft_prio->anchor.rule_goto_table) { 2754 mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table); 2755 ft_prio->anchor.rule_goto_table = NULL; 2756 } 2757 } 2758 2759 static int steering_anchor_create_res(struct mlx5_ib_dev *dev, 2760 struct mlx5_ib_flow_prio *ft_prio, 2761 enum mlx5_flow_namespace_type ns_type) 2762 { 2763 int err; 2764 2765 err = steering_anchor_create_ft(dev, ft_prio, ns_type); 2766 if (err) 2767 return err; 2768 2769 err = steering_anchor_create_fg_drop(ft_prio); 2770 if (err) 2771 goto destroy_ft; 2772 2773 err = steering_anchor_create_fg_goto_table(ft_prio); 2774 if (err) 2775 goto destroy_fg_drop; 2776 2777 err = steering_anchor_create_rule_drop(ft_prio); 2778 if (err) 2779 goto destroy_fg_goto_table; 2780 2781 err = steering_anchor_create_rule_goto_table(ft_prio); 2782 if (err) 2783 goto destroy_rule_drop; 2784 2785 return 0; 2786 2787 destroy_rule_drop: 2788 steering_anchor_destroy_rule_drop(ft_prio); 2789 destroy_fg_goto_table: 2790 steering_anchor_destroy_fg_goto_table(ft_prio); 2791 destroy_fg_drop: 2792 steering_anchor_destroy_fg_drop(ft_prio); 2793 destroy_ft: 2794 steering_anchor_destroy_ft(ft_prio); 2795 2796 return err; 2797 } 2798 2799 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio) 2800 { 2801 steering_anchor_destroy_rule_goto_table(ft_prio); 2802 steering_anchor_destroy_rule_drop(ft_prio); 2803 steering_anchor_destroy_fg_goto_table(ft_prio); 2804 steering_anchor_destroy_fg_drop(ft_prio); 2805 steering_anchor_destroy_ft(ft_prio); 2806 } 2807 2808 static int steering_anchor_cleanup(struct ib_uobject *uobject, 2809 enum rdma_remove_reason why, 2810 struct uverbs_attr_bundle *attrs) 2811 { 2812 struct mlx5_ib_steering_anchor *obj = uobject->object; 2813 2814 if (atomic_read(&obj->usecnt)) 2815 return -EBUSY; 2816 2817 mutex_lock(&obj->dev->flow_db->lock); 2818 if (!--obj->ft_prio->anchor.rule_goto_table_ref) 2819 steering_anchor_destroy_rule_goto_table(obj->ft_prio); 2820 2821 put_flow_table(obj->dev, obj->ft_prio, true); 2822 mutex_unlock(&obj->dev->flow_db->lock); 2823 2824 kfree(obj); 2825 return 0; 2826 } 2827 2828 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio, 2829 int count) 2830 { 2831 while (count--) 2832 mlx5_steering_anchor_destroy_res(&prio[count]); 2833 } 2834 2835 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev) 2836 { 2837 fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT); 2838 fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT); 2839 fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS); 2840 fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS); 2841 fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS); 2842 fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT); 2843 fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT); 2844 } 2845 2846 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2847 struct mlx5_ib_flow_matcher *obj) 2848 { 2849 enum mlx5_ib_uapi_flow_table_type ft_type = 2850 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2851 u32 flags; 2852 int err; 2853 2854 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2855 * users should switch to it. We leave this to not break userspace 2856 */ 2857 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2858 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2859 return -EINVAL; 2860 2861 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2862 err = uverbs_get_const(&ft_type, attrs, 2863 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2864 if (err) 2865 return err; 2866 2867 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2868 if (err) 2869 return err; 2870 2871 return 0; 2872 } 2873 2874 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2875 err = uverbs_get_flags32(&flags, attrs, 2876 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2877 IB_FLOW_ATTR_FLAGS_EGRESS); 2878 if (err) 2879 return err; 2880 2881 if (flags) 2882 return mlx5_ib_ft_type_to_namespace( 2883 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2884 &obj->ns_type); 2885 } 2886 2887 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2888 2889 return 0; 2890 } 2891 2892 static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps) 2893 { 2894 if (is_mdev_switchdev_mode(dev->mdev)) 2895 return UCAP_ENABLED(enabled_caps, 2896 RDMA_UCAP_MLX5_CTRL_OTHER_VHCA); 2897 2898 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL); 2899 } 2900 2901 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2902 struct uverbs_attr_bundle *attrs) 2903 { 2904 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2905 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2906 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2907 struct mlx5_ib_flow_matcher *obj; 2908 int err; 2909 2910 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2911 if (!obj) 2912 return -ENOMEM; 2913 2914 obj->mask_len = uverbs_attr_get_len( 2915 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2916 err = uverbs_copy_from(&obj->matcher_mask, 2917 attrs, 2918 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2919 if (err) 2920 goto end; 2921 2922 obj->flow_type = uverbs_attr_get_enum_id( 2923 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2924 2925 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2926 err = uverbs_copy_from(&obj->priority, 2927 attrs, 2928 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2929 if (err) 2930 goto end; 2931 } 2932 2933 err = uverbs_copy_from(&obj->match_criteria_enable, 2934 attrs, 2935 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2936 if (err) 2937 goto end; 2938 2939 err = mlx5_ib_matcher_ns(attrs, obj); 2940 if (err) 2941 goto end; 2942 2943 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2944 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { 2945 err = -EINVAL; 2946 goto end; 2947 } 2948 2949 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) { 2950 err = uverbs_copy_from(&obj->ib_port, attrs, 2951 MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT); 2952 if (err) 2953 goto end; 2954 if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) { 2955 err = -EINVAL; 2956 goto end; 2957 } 2958 if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX && 2959 obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) { 2960 err = -EINVAL; 2961 goto end; 2962 } 2963 if (!verify_context_caps(dev, uobj->context->enabled_caps)) { 2964 err = -EOPNOTSUPP; 2965 goto end; 2966 } 2967 } 2968 2969 uobj->object = obj; 2970 obj->mdev = dev->mdev; 2971 atomic_set(&obj->usecnt, 0); 2972 return 0; 2973 2974 end: 2975 kfree(obj); 2976 return err; 2977 } 2978 2979 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( 2980 struct uverbs_attr_bundle *attrs) 2981 { 2982 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2983 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE); 2984 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2985 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type; 2986 enum mlx5_flow_namespace_type ns_type; 2987 struct mlx5_ib_steering_anchor *obj; 2988 struct mlx5_ib_flow_prio *ft_prio; 2989 u16 priority; 2990 u32 ft_id; 2991 int err; 2992 2993 if (!rdma_dev_has_raw_cap(&dev->ib_dev)) 2994 return -EPERM; 2995 2996 err = uverbs_get_const(&ib_uapi_ft_type, attrs, 2997 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE); 2998 if (err) 2999 return err; 3000 3001 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type); 3002 if (err) 3003 return err; 3004 3005 err = uverbs_copy_from(&priority, attrs, 3006 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY); 3007 if (err) 3008 return err; 3009 3010 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 3011 if (!obj) 3012 return -ENOMEM; 3013 3014 mutex_lock(&dev->flow_db->lock); 3015 3016 ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0); 3017 if (IS_ERR(ft_prio)) { 3018 err = PTR_ERR(ft_prio); 3019 goto free_obj; 3020 } 3021 3022 ft_prio->refcount++; 3023 3024 if (!ft_prio->anchor.rule_goto_table_ref) { 3025 err = steering_anchor_create_res(dev, ft_prio, ns_type); 3026 if (err) 3027 goto put_flow_table; 3028 } 3029 3030 ft_prio->anchor.rule_goto_table_ref++; 3031 3032 ft_id = mlx5_flow_table_id(ft_prio->anchor.ft); 3033 3034 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3035 &ft_id, sizeof(ft_id)); 3036 if (err) 3037 goto destroy_res; 3038 3039 mutex_unlock(&dev->flow_db->lock); 3040 3041 uobj->object = obj; 3042 obj->dev = dev; 3043 obj->ft_prio = ft_prio; 3044 atomic_set(&obj->usecnt, 0); 3045 3046 return 0; 3047 3048 destroy_res: 3049 --ft_prio->anchor.rule_goto_table_ref; 3050 mlx5_steering_anchor_destroy_res(ft_prio); 3051 put_flow_table: 3052 put_flow_table(dev, ft_prio, true); 3053 free_obj: 3054 mutex_unlock(&dev->flow_db->lock); 3055 kfree(obj); 3056 3057 return err; 3058 } 3059 3060 static struct ib_flow_action * 3061 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 3062 enum mlx5_ib_uapi_flow_table_type ft_type, 3063 u8 num_actions, void *in) 3064 { 3065 enum mlx5_flow_namespace_type namespace; 3066 struct mlx5_ib_flow_action *maction; 3067 int ret; 3068 3069 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3070 if (ret) 3071 return ERR_PTR(-EINVAL); 3072 3073 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3074 if (!maction) 3075 return ERR_PTR(-ENOMEM); 3076 3077 maction->flow_action_raw.modify_hdr = 3078 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 3079 3080 if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 3081 ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 3082 kfree(maction); 3083 return ERR_PTR(ret); 3084 } 3085 maction->flow_action_raw.sub_type = 3086 MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 3087 maction->flow_action_raw.dev = dev; 3088 3089 return &maction->ib_action; 3090 } 3091 3092 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 3093 { 3094 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3095 max_modify_header_actions) || 3096 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 3097 max_modify_header_actions) || 3098 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 3099 max_modify_header_actions); 3100 } 3101 3102 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 3103 struct uverbs_attr_bundle *attrs) 3104 { 3105 struct ib_uobject *uobj = uverbs_attr_get_uobject( 3106 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 3107 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3108 enum mlx5_ib_uapi_flow_table_type ft_type; 3109 struct ib_flow_action *action; 3110 int num_actions; 3111 void *in; 3112 int ret; 3113 3114 if (!mlx5_ib_modify_header_supported(mdev)) 3115 return -EOPNOTSUPP; 3116 3117 in = uverbs_attr_get_alloced_ptr(attrs, 3118 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 3119 3120 num_actions = uverbs_attr_ptr_get_array_size( 3121 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3122 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 3123 if (num_actions < 0) 3124 return num_actions; 3125 3126 ret = uverbs_get_const(&ft_type, attrs, 3127 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 3128 if (ret) 3129 return ret; 3130 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 3131 if (IS_ERR(action)) 3132 return PTR_ERR(action); 3133 3134 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 3135 IB_FLOW_ACTION_UNSPECIFIED); 3136 3137 return 0; 3138 } 3139 3140 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 3141 u8 packet_reformat_type, 3142 u8 ft_type) 3143 { 3144 switch (packet_reformat_type) { 3145 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3146 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3147 return MLX5_CAP_FLOWTABLE(ibdev->mdev, 3148 encap_general_header); 3149 break; 3150 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3151 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3152 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 3153 reformat_l2_to_l3_tunnel); 3154 break; 3155 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3156 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3157 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 3158 reformat_l3_tunnel_to_l2); 3159 break; 3160 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 3161 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3162 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 3163 break; 3164 default: 3165 break; 3166 } 3167 3168 return false; 3169 } 3170 3171 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 3172 { 3173 switch (dv_prt) { 3174 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3175 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 3176 break; 3177 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3178 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 3179 break; 3180 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3181 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 3182 break; 3183 default: 3184 return -EINVAL; 3185 } 3186 3187 return 0; 3188 } 3189 3190 static int mlx5_ib_flow_action_create_packet_reformat_ctx( 3191 struct mlx5_ib_dev *dev, 3192 struct mlx5_ib_flow_action *maction, 3193 u8 ft_type, u8 dv_prt, 3194 void *in, size_t len) 3195 { 3196 struct mlx5_pkt_reformat_params reformat_params; 3197 enum mlx5_flow_namespace_type namespace; 3198 u8 prm_prt; 3199 int ret; 3200 3201 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3202 if (ret) 3203 return ret; 3204 3205 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 3206 if (ret) 3207 return ret; 3208 3209 memset(&reformat_params, 0, sizeof(reformat_params)); 3210 reformat_params.type = prm_prt; 3211 reformat_params.size = len; 3212 reformat_params.data = in; 3213 maction->flow_action_raw.pkt_reformat = 3214 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params, 3215 namespace); 3216 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 3217 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 3218 return ret; 3219 } 3220 3221 maction->flow_action_raw.sub_type = 3222 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 3223 maction->flow_action_raw.dev = dev; 3224 3225 return 0; 3226 } 3227 3228 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 3229 struct uverbs_attr_bundle *attrs) 3230 { 3231 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 3232 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 3233 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3234 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 3235 enum mlx5_ib_uapi_flow_table_type ft_type; 3236 struct mlx5_ib_flow_action *maction; 3237 int ret; 3238 3239 ret = uverbs_get_const(&ft_type, attrs, 3240 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 3241 if (ret) 3242 return ret; 3243 3244 ret = uverbs_get_const(&dv_prt, attrs, 3245 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 3246 if (ret) 3247 return ret; 3248 3249 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 3250 return -EOPNOTSUPP; 3251 3252 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3253 if (!maction) 3254 return -ENOMEM; 3255 3256 if (dv_prt == 3257 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 3258 maction->flow_action_raw.sub_type = 3259 MLX5_IB_FLOW_ACTION_DECAP; 3260 maction->flow_action_raw.dev = mdev; 3261 } else { 3262 void *in; 3263 int len; 3264 3265 in = uverbs_attr_get_alloced_ptr(attrs, 3266 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3267 if (IS_ERR(in)) { 3268 ret = PTR_ERR(in); 3269 goto free_maction; 3270 } 3271 3272 len = uverbs_attr_get_len(attrs, 3273 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3274 3275 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 3276 maction, ft_type, dv_prt, in, len); 3277 if (ret) 3278 goto free_maction; 3279 } 3280 3281 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 3282 IB_FLOW_ACTION_UNSPECIFIED); 3283 return 0; 3284 3285 free_maction: 3286 kfree(maction); 3287 return ret; 3288 } 3289 3290 DECLARE_UVERBS_NAMED_METHOD( 3291 MLX5_IB_METHOD_CREATE_FLOW, 3292 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3293 UVERBS_OBJECT_FLOW, 3294 UVERBS_ACCESS_NEW, 3295 UA_MANDATORY), 3296 UVERBS_ATTR_PTR_IN( 3297 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 3298 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3299 UA_MANDATORY, 3300 UA_ALLOC_AND_COPY), 3301 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 3302 MLX5_IB_OBJECT_FLOW_MATCHER, 3303 UVERBS_ACCESS_READ, 3304 UA_MANDATORY), 3305 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 3306 UVERBS_OBJECT_QP, 3307 UVERBS_ACCESS_READ), 3308 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 3309 MLX5_IB_OBJECT_DEVX_OBJ, 3310 UVERBS_ACCESS_READ), 3311 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 3312 UVERBS_OBJECT_FLOW_ACTION, 3313 UVERBS_ACCESS_READ, 1, 3314 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 3315 UA_OPTIONAL), 3316 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 3317 UVERBS_ATTR_TYPE(u32), 3318 UA_OPTIONAL), 3319 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 3320 MLX5_IB_OBJECT_DEVX_OBJ, 3321 UVERBS_ACCESS_READ, 1, 1, 3322 UA_OPTIONAL), 3323 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 3324 UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 3325 UA_OPTIONAL, 3326 UA_ALLOC_AND_COPY), 3327 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 3328 enum mlx5_ib_create_flow_flags, 3329 UA_OPTIONAL)); 3330 3331 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3332 MLX5_IB_METHOD_DESTROY_FLOW, 3333 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3334 UVERBS_OBJECT_FLOW, 3335 UVERBS_ACCESS_DESTROY, 3336 UA_MANDATORY)); 3337 3338 ADD_UVERBS_METHODS(mlx5_ib_fs, 3339 UVERBS_OBJECT_FLOW, 3340 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 3341 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 3342 3343 DECLARE_UVERBS_NAMED_METHOD( 3344 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 3345 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 3346 UVERBS_OBJECT_FLOW_ACTION, 3347 UVERBS_ACCESS_NEW, 3348 UA_MANDATORY), 3349 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3350 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 3351 set_add_copy_action_in_auto)), 3352 UA_MANDATORY, 3353 UA_ALLOC_AND_COPY), 3354 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 3355 enum mlx5_ib_uapi_flow_table_type, 3356 UA_MANDATORY)); 3357 3358 DECLARE_UVERBS_NAMED_METHOD( 3359 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 3360 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 3361 UVERBS_OBJECT_FLOW_ACTION, 3362 UVERBS_ACCESS_NEW, 3363 UA_MANDATORY), 3364 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 3365 UVERBS_ATTR_MIN_SIZE(1), 3366 UA_ALLOC_AND_COPY, 3367 UA_OPTIONAL), 3368 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 3369 enum mlx5_ib_uapi_flow_action_packet_reformat_type, 3370 UA_MANDATORY), 3371 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 3372 enum mlx5_ib_uapi_flow_table_type, 3373 UA_MANDATORY)); 3374 3375 ADD_UVERBS_METHODS( 3376 mlx5_ib_flow_actions, 3377 UVERBS_OBJECT_FLOW_ACTION, 3378 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 3379 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 3380 3381 DECLARE_UVERBS_NAMED_METHOD( 3382 MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 3383 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 3384 MLX5_IB_OBJECT_FLOW_MATCHER, 3385 UVERBS_ACCESS_NEW, 3386 UA_MANDATORY), 3387 UVERBS_ATTR_PTR_IN( 3388 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 3389 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3390 UA_MANDATORY), 3391 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 3392 mlx5_ib_flow_type, 3393 UA_MANDATORY), 3394 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 3395 UVERBS_ATTR_TYPE(u8), 3396 UA_MANDATORY), 3397 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 3398 enum ib_flow_flags, 3399 UA_OPTIONAL), 3400 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 3401 enum mlx5_ib_uapi_flow_table_type, 3402 UA_OPTIONAL), 3403 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT, 3404 UVERBS_ATTR_TYPE(u32), 3405 UA_OPTIONAL)); 3406 3407 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3408 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 3409 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 3410 MLX5_IB_OBJECT_FLOW_MATCHER, 3411 UVERBS_ACCESS_DESTROY, 3412 UA_MANDATORY)); 3413 3414 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 3415 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 3416 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 3417 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 3418 3419 DECLARE_UVERBS_NAMED_METHOD( 3420 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE, 3421 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE, 3422 MLX5_IB_OBJECT_STEERING_ANCHOR, 3423 UVERBS_ACCESS_NEW, 3424 UA_MANDATORY), 3425 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE, 3426 enum mlx5_ib_uapi_flow_table_type, 3427 UA_MANDATORY), 3428 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY, 3429 UVERBS_ATTR_TYPE(u16), 3430 UA_MANDATORY), 3431 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3432 UVERBS_ATTR_TYPE(u32), 3433 UA_MANDATORY)); 3434 3435 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3436 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY, 3437 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE, 3438 MLX5_IB_OBJECT_STEERING_ANCHOR, 3439 UVERBS_ACCESS_DESTROY, 3440 UA_MANDATORY)); 3441 3442 DECLARE_UVERBS_NAMED_OBJECT( 3443 MLX5_IB_OBJECT_STEERING_ANCHOR, 3444 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup), 3445 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE), 3446 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY)); 3447 3448 const struct uapi_definition mlx5_ib_flow_defs[] = { 3449 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3450 MLX5_IB_OBJECT_FLOW_MATCHER), 3451 UAPI_DEF_CHAIN_OBJ_TREE( 3452 UVERBS_OBJECT_FLOW, 3453 &mlx5_ib_fs), 3454 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 3455 &mlx5_ib_flow_actions), 3456 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3457 MLX5_IB_OBJECT_STEERING_ANCHOR, 3458 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)), 3459 {}, 3460 }; 3461 3462 static const struct ib_device_ops flow_ops = { 3463 .create_flow = mlx5_ib_create_flow, 3464 .destroy_flow = mlx5_ib_destroy_flow, 3465 .destroy_flow_action = mlx5_ib_destroy_flow_action, 3466 }; 3467 3468 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 3469 { 3470 int i, j; 3471 3472 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 3473 3474 if (!dev->flow_db) 3475 return -ENOMEM; 3476 3477 for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) { 3478 dev->flow_db->rdma_transport_rx[i] = 3479 kcalloc(dev->num_ports, 3480 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL); 3481 if (!dev->flow_db->rdma_transport_rx[i]) 3482 goto free_rdma_transport_rx; 3483 } 3484 3485 for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) { 3486 dev->flow_db->rdma_transport_tx[j] = 3487 kcalloc(dev->num_ports, 3488 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL); 3489 if (!dev->flow_db->rdma_transport_tx[j]) 3490 goto free_rdma_transport_tx; 3491 } 3492 3493 mutex_init(&dev->flow_db->lock); 3494 3495 ib_set_device_ops(&dev->ib_dev, &flow_ops); 3496 return 0; 3497 3498 free_rdma_transport_tx: 3499 while (j--) 3500 kfree(dev->flow_db->rdma_transport_tx[j]); 3501 free_rdma_transport_rx: 3502 while (i--) 3503 kfree(dev->flow_db->rdma_transport_rx[i]); 3504 kfree(dev->flow_db); 3505 return -ENOMEM; 3506 } 3507