1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/uverbs_std_types.h> 11 #include <rdma/mlx5_user_ioctl_cmds.h> 12 #include <rdma/mlx5_user_ioctl_verbs.h> 13 #include <rdma/ib_hdrs.h> 14 #include <rdma/ib_umem.h> 15 #include <rdma/ib_ucaps.h> 16 #include <linux/mlx5/driver.h> 17 #include <linux/mlx5/fs.h> 18 #include <linux/mlx5/fs_helpers.h> 19 #include <linux/mlx5/eswitch.h> 20 #include <net/inet_ecn.h> 21 #include "mlx5_ib.h" 22 #include "counters.h" 23 #include "devx.h" 24 #include "fs.h" 25 26 #define UVERBS_MODULE_NAME mlx5_ib 27 #include <rdma/uverbs_named_ioctl.h> 28 29 enum { 30 MATCH_CRITERIA_ENABLE_OUTER_BIT, 31 MATCH_CRITERIA_ENABLE_MISC_BIT, 32 MATCH_CRITERIA_ENABLE_INNER_BIT, 33 MATCH_CRITERIA_ENABLE_MISC2_BIT 34 }; 35 36 37 struct mlx5_per_qp_opfc { 38 struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX]; 39 }; 40 41 #define HEADER_IS_ZERO(match_criteria, headers) \ 42 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 43 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 44 45 static u8 get_match_criteria_enable(u32 *match_criteria) 46 { 47 u8 match_criteria_enable; 48 49 match_criteria_enable = 50 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 51 MATCH_CRITERIA_ENABLE_OUTER_BIT; 52 match_criteria_enable |= 53 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 54 MATCH_CRITERIA_ENABLE_MISC_BIT; 55 match_criteria_enable |= 56 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 57 MATCH_CRITERIA_ENABLE_INNER_BIT; 58 match_criteria_enable |= 59 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 60 MATCH_CRITERIA_ENABLE_MISC2_BIT; 61 62 return match_criteria_enable; 63 } 64 65 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 66 { 67 u8 entry_mask; 68 u8 entry_val; 69 int err = 0; 70 71 if (!mask) 72 goto out; 73 74 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 75 ip_protocol); 76 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 77 ip_protocol); 78 if (!entry_mask) { 79 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 80 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 81 goto out; 82 } 83 /* Don't override existing ip protocol */ 84 if (mask != entry_mask || val != entry_val) 85 err = -EINVAL; 86 out: 87 return err; 88 } 89 90 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 91 bool inner) 92 { 93 if (inner) { 94 MLX5_SET(fte_match_set_misc, 95 misc_c, inner_ipv6_flow_label, mask); 96 MLX5_SET(fte_match_set_misc, 97 misc_v, inner_ipv6_flow_label, val); 98 } else { 99 MLX5_SET(fte_match_set_misc, 100 misc_c, outer_ipv6_flow_label, mask); 101 MLX5_SET(fte_match_set_misc, 102 misc_v, outer_ipv6_flow_label, val); 103 } 104 } 105 106 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 107 { 108 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 109 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 110 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 111 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 112 } 113 114 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 115 { 116 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 117 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 118 return -EOPNOTSUPP; 119 120 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 121 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 122 return -EOPNOTSUPP; 123 124 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 125 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 126 return -EOPNOTSUPP; 127 128 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 129 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 130 return -EOPNOTSUPP; 131 132 return 0; 133 } 134 135 #define LAST_ETH_FIELD vlan_tag 136 #define LAST_IPV4_FIELD tos 137 #define LAST_IPV6_FIELD traffic_class 138 #define LAST_TCP_UDP_FIELD src_port 139 #define LAST_TUNNEL_FIELD tunnel_id 140 #define LAST_FLOW_TAG_FIELD tag_id 141 #define LAST_DROP_FIELD size 142 #define LAST_COUNTERS_FIELD counters 143 144 /* Field is the last supported field */ 145 #define FIELDS_NOT_SUPPORTED(filter, field) \ 146 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ 147 sizeof(filter) - offsetofend(typeof(filter), field)) 148 149 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 150 bool is_egress, 151 struct mlx5_flow_act *action) 152 { 153 154 switch (maction->ib_action.type) { 155 case IB_FLOW_ACTION_UNSPECIFIED: 156 if (maction->flow_action_raw.sub_type == 157 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 158 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 159 return -EINVAL; 160 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 161 action->modify_hdr = 162 maction->flow_action_raw.modify_hdr; 163 return 0; 164 } 165 if (maction->flow_action_raw.sub_type == 166 MLX5_IB_FLOW_ACTION_DECAP) { 167 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 168 return -EINVAL; 169 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 170 return 0; 171 } 172 if (maction->flow_action_raw.sub_type == 173 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 174 if (action->action & 175 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 176 return -EINVAL; 177 action->action |= 178 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 179 action->pkt_reformat = 180 maction->flow_action_raw.pkt_reformat; 181 return 0; 182 } 183 fallthrough; 184 default: 185 return -EOPNOTSUPP; 186 } 187 } 188 189 static int parse_flow_attr(struct mlx5_core_dev *mdev, 190 struct mlx5_flow_spec *spec, 191 const union ib_flow_spec *ib_spec, 192 const struct ib_flow_attr *flow_attr, 193 struct mlx5_flow_act *action, u32 prev_type) 194 { 195 struct mlx5_flow_context *flow_context = &spec->flow_context; 196 u32 *match_c = spec->match_criteria; 197 u32 *match_v = spec->match_value; 198 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 199 misc_parameters); 200 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 201 misc_parameters); 202 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 203 misc_parameters_2); 204 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 205 misc_parameters_2); 206 void *headers_c; 207 void *headers_v; 208 int match_ipv; 209 int ret; 210 211 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 212 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 213 inner_headers); 214 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 215 inner_headers); 216 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 217 ft_field_support.inner_ip_version); 218 } else { 219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 220 outer_headers); 221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 222 outer_headers); 223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 224 ft_field_support.outer_ip_version); 225 } 226 227 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 228 case IB_FLOW_SPEC_ETH: 229 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 230 return -EOPNOTSUPP; 231 232 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 233 dmac_47_16), 234 ib_spec->eth.mask.dst_mac); 235 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 236 dmac_47_16), 237 ib_spec->eth.val.dst_mac); 238 239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 240 smac_47_16), 241 ib_spec->eth.mask.src_mac); 242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 243 smac_47_16), 244 ib_spec->eth.val.src_mac); 245 246 if (ib_spec->eth.mask.vlan_tag) { 247 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 248 cvlan_tag, 1); 249 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 250 cvlan_tag, 1); 251 252 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 253 first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 254 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 255 first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 256 257 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 258 first_cfi, 259 ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 260 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 261 first_cfi, 262 ntohs(ib_spec->eth.val.vlan_tag) >> 12); 263 264 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 265 first_prio, 266 ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 267 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 268 first_prio, 269 ntohs(ib_spec->eth.val.vlan_tag) >> 13); 270 } 271 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 272 ethertype, ntohs(ib_spec->eth.mask.ether_type)); 273 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 274 ethertype, ntohs(ib_spec->eth.val.ether_type)); 275 break; 276 case IB_FLOW_SPEC_IPV4: 277 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 278 return -EOPNOTSUPP; 279 280 if (match_ipv) { 281 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 282 ip_version, 0xf); 283 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 284 ip_version, MLX5_FS_IPV4_VERSION); 285 } else { 286 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 287 ethertype, 0xffff); 288 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 289 ethertype, ETH_P_IP); 290 } 291 292 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 293 src_ipv4_src_ipv6.ipv4_layout.ipv4), 294 &ib_spec->ipv4.mask.src_ip, 295 sizeof(ib_spec->ipv4.mask.src_ip)); 296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 297 src_ipv4_src_ipv6.ipv4_layout.ipv4), 298 &ib_spec->ipv4.val.src_ip, 299 sizeof(ib_spec->ipv4.val.src_ip)); 300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 301 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 302 &ib_spec->ipv4.mask.dst_ip, 303 sizeof(ib_spec->ipv4.mask.dst_ip)); 304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 306 &ib_spec->ipv4.val.dst_ip, 307 sizeof(ib_spec->ipv4.val.dst_ip)); 308 309 set_tos(headers_c, headers_v, 310 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 311 312 if (set_proto(headers_c, headers_v, 313 ib_spec->ipv4.mask.proto, 314 ib_spec->ipv4.val.proto)) 315 return -EINVAL; 316 break; 317 case IB_FLOW_SPEC_IPV6: 318 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 319 return -EOPNOTSUPP; 320 321 if (match_ipv) { 322 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 323 ip_version, 0xf); 324 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 325 ip_version, MLX5_FS_IPV6_VERSION); 326 } else { 327 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 328 ethertype, 0xffff); 329 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 330 ethertype, ETH_P_IPV6); 331 } 332 333 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 334 src_ipv4_src_ipv6.ipv6_layout.ipv6), 335 &ib_spec->ipv6.mask.src_ip, 336 sizeof(ib_spec->ipv6.mask.src_ip)); 337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 338 src_ipv4_src_ipv6.ipv6_layout.ipv6), 339 &ib_spec->ipv6.val.src_ip, 340 sizeof(ib_spec->ipv6.val.src_ip)); 341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 342 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 343 &ib_spec->ipv6.mask.dst_ip, 344 sizeof(ib_spec->ipv6.mask.dst_ip)); 345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 347 &ib_spec->ipv6.val.dst_ip, 348 sizeof(ib_spec->ipv6.val.dst_ip)); 349 350 set_tos(headers_c, headers_v, 351 ib_spec->ipv6.mask.traffic_class, 352 ib_spec->ipv6.val.traffic_class); 353 354 if (set_proto(headers_c, headers_v, 355 ib_spec->ipv6.mask.next_hdr, 356 ib_spec->ipv6.val.next_hdr)) 357 return -EINVAL; 358 359 set_flow_label(misc_params_c, misc_params_v, 360 ntohl(ib_spec->ipv6.mask.flow_label), 361 ntohl(ib_spec->ipv6.val.flow_label), 362 ib_spec->type & IB_FLOW_SPEC_INNER); 363 break; 364 case IB_FLOW_SPEC_ESP: 365 return -EOPNOTSUPP; 366 case IB_FLOW_SPEC_TCP: 367 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 368 LAST_TCP_UDP_FIELD)) 369 return -EOPNOTSUPP; 370 371 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 372 return -EINVAL; 373 374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 375 ntohs(ib_spec->tcp_udp.mask.src_port)); 376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 377 ntohs(ib_spec->tcp_udp.val.src_port)); 378 379 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 380 ntohs(ib_spec->tcp_udp.mask.dst_port)); 381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 382 ntohs(ib_spec->tcp_udp.val.dst_port)); 383 break; 384 case IB_FLOW_SPEC_UDP: 385 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 386 LAST_TCP_UDP_FIELD)) 387 return -EOPNOTSUPP; 388 389 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 390 return -EINVAL; 391 392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 393 ntohs(ib_spec->tcp_udp.mask.src_port)); 394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 395 ntohs(ib_spec->tcp_udp.val.src_port)); 396 397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 398 ntohs(ib_spec->tcp_udp.mask.dst_port)); 399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 400 ntohs(ib_spec->tcp_udp.val.dst_port)); 401 break; 402 case IB_FLOW_SPEC_GRE: 403 if (ib_spec->gre.mask.c_ks_res0_ver) 404 return -EOPNOTSUPP; 405 406 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 407 return -EINVAL; 408 409 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 410 0xff); 411 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 412 IPPROTO_GRE); 413 414 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 415 ntohs(ib_spec->gre.mask.protocol)); 416 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 417 ntohs(ib_spec->gre.val.protocol)); 418 419 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 420 gre_key.nvgre.hi), 421 &ib_spec->gre.mask.key, 422 sizeof(ib_spec->gre.mask.key)); 423 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 424 gre_key.nvgre.hi), 425 &ib_spec->gre.val.key, 426 sizeof(ib_spec->gre.val.key)); 427 break; 428 case IB_FLOW_SPEC_MPLS: 429 switch (prev_type) { 430 case IB_FLOW_SPEC_UDP: 431 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 432 ft_field_support.outer_first_mpls_over_udp), 433 &ib_spec->mpls.mask.tag)) 434 return -EOPNOTSUPP; 435 436 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 437 outer_first_mpls_over_udp), 438 &ib_spec->mpls.val.tag, 439 sizeof(ib_spec->mpls.val.tag)); 440 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 441 outer_first_mpls_over_udp), 442 &ib_spec->mpls.mask.tag, 443 sizeof(ib_spec->mpls.mask.tag)); 444 break; 445 case IB_FLOW_SPEC_GRE: 446 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 447 ft_field_support.outer_first_mpls_over_gre), 448 &ib_spec->mpls.mask.tag)) 449 return -EOPNOTSUPP; 450 451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 452 outer_first_mpls_over_gre), 453 &ib_spec->mpls.val.tag, 454 sizeof(ib_spec->mpls.val.tag)); 455 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 456 outer_first_mpls_over_gre), 457 &ib_spec->mpls.mask.tag, 458 sizeof(ib_spec->mpls.mask.tag)); 459 break; 460 default: 461 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 462 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 463 ft_field_support.inner_first_mpls), 464 &ib_spec->mpls.mask.tag)) 465 return -EOPNOTSUPP; 466 467 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 468 inner_first_mpls), 469 &ib_spec->mpls.val.tag, 470 sizeof(ib_spec->mpls.val.tag)); 471 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 472 inner_first_mpls), 473 &ib_spec->mpls.mask.tag, 474 sizeof(ib_spec->mpls.mask.tag)); 475 } else { 476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 477 ft_field_support.outer_first_mpls), 478 &ib_spec->mpls.mask.tag)) 479 return -EOPNOTSUPP; 480 481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 482 outer_first_mpls), 483 &ib_spec->mpls.val.tag, 484 sizeof(ib_spec->mpls.val.tag)); 485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 486 outer_first_mpls), 487 &ib_spec->mpls.mask.tag, 488 sizeof(ib_spec->mpls.mask.tag)); 489 } 490 } 491 break; 492 case IB_FLOW_SPEC_VXLAN_TUNNEL: 493 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 494 LAST_TUNNEL_FIELD)) 495 return -EOPNOTSUPP; 496 497 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 498 ntohl(ib_spec->tunnel.mask.tunnel_id)); 499 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 500 ntohl(ib_spec->tunnel.val.tunnel_id)); 501 break; 502 case IB_FLOW_SPEC_ACTION_TAG: 503 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 504 LAST_FLOW_TAG_FIELD)) 505 return -EOPNOTSUPP; 506 if (ib_spec->flow_tag.tag_id >= BIT(24)) 507 return -EINVAL; 508 509 flow_context->flow_tag = ib_spec->flow_tag.tag_id; 510 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 511 break; 512 case IB_FLOW_SPEC_ACTION_DROP: 513 if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 514 LAST_DROP_FIELD)) 515 return -EOPNOTSUPP; 516 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 517 break; 518 case IB_FLOW_SPEC_ACTION_HANDLE: 519 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 520 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 521 if (ret) 522 return ret; 523 break; 524 case IB_FLOW_SPEC_ACTION_COUNT: 525 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 526 LAST_COUNTERS_FIELD)) 527 return -EOPNOTSUPP; 528 529 /* for now support only one counters spec per flow */ 530 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 531 return -EINVAL; 532 533 action->counters = ib_spec->flow_count.counters; 534 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 535 break; 536 default: 537 return -EINVAL; 538 } 539 540 return 0; 541 } 542 543 /* If a flow could catch both multicast and unicast packets, 544 * it won't fall into the multicast flow steering table and this rule 545 * could steal other multicast packets. 546 */ 547 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 548 { 549 union ib_flow_spec *flow_spec; 550 551 if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 552 ib_attr->num_of_specs < 1) 553 return false; 554 555 flow_spec = (union ib_flow_spec *)(ib_attr + 1); 556 if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 557 struct ib_flow_spec_ipv4 *ipv4_spec; 558 559 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 560 if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 561 return true; 562 563 return false; 564 } 565 566 if (flow_spec->type == IB_FLOW_SPEC_ETH) { 567 struct ib_flow_spec_eth *eth_spec; 568 569 eth_spec = (struct ib_flow_spec_eth *)flow_spec; 570 return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 571 is_multicast_ether_addr(eth_spec->val.dst_mac); 572 } 573 574 return false; 575 } 576 577 static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 578 const struct ib_flow_attr *flow_attr, 579 bool check_inner) 580 { 581 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 582 int match_ipv = check_inner ? 583 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 584 ft_field_support.inner_ip_version) : 585 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 586 ft_field_support.outer_ip_version); 587 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 588 bool ipv4_spec_valid, ipv6_spec_valid; 589 unsigned int ip_spec_type = 0; 590 bool has_ethertype = false; 591 unsigned int spec_index; 592 bool mask_valid = true; 593 u16 eth_type = 0; 594 bool type_valid; 595 596 /* Validate that ethertype is correct */ 597 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 598 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 599 ib_spec->eth.mask.ether_type) { 600 mask_valid = (ib_spec->eth.mask.ether_type == 601 htons(0xffff)); 602 has_ethertype = true; 603 eth_type = ntohs(ib_spec->eth.val.ether_type); 604 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 605 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 606 ip_spec_type = ib_spec->type; 607 } 608 ib_spec = (void *)ib_spec + ib_spec->size; 609 } 610 611 type_valid = (!has_ethertype) || (!ip_spec_type); 612 if (!type_valid && mask_valid) { 613 ipv4_spec_valid = (eth_type == ETH_P_IP) && 614 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 615 ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 616 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 617 618 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 619 (((eth_type == ETH_P_MPLS_UC) || 620 (eth_type == ETH_P_MPLS_MC)) && match_ipv); 621 } 622 623 return type_valid; 624 } 625 626 static bool is_valid_attr(struct mlx5_core_dev *mdev, 627 const struct ib_flow_attr *flow_attr) 628 { 629 return is_valid_ethertype(mdev, flow_attr, false) && 630 is_valid_ethertype(mdev, flow_attr, true); 631 } 632 633 static void put_flow_table(struct mlx5_ib_dev *dev, 634 struct mlx5_ib_flow_prio *prio, bool ft_added) 635 { 636 prio->refcount -= !!ft_added; 637 if (!prio->refcount) { 638 mlx5_destroy_flow_table(prio->flow_table); 639 prio->flow_table = NULL; 640 } 641 } 642 643 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 644 { 645 struct mlx5_ib_flow_handler *handler = container_of(flow_id, 646 struct mlx5_ib_flow_handler, 647 ibflow); 648 struct mlx5_ib_flow_handler *iter, *tmp; 649 struct mlx5_ib_dev *dev = handler->dev; 650 651 mutex_lock(&dev->flow_db->lock); 652 653 list_for_each_entry_safe(iter, tmp, &handler->list, list) { 654 mlx5_del_flow_rules(iter->rule); 655 put_flow_table(dev, iter->prio, true); 656 list_del(&iter->list); 657 kfree(iter); 658 } 659 660 mlx5_del_flow_rules(handler->rule); 661 put_flow_table(dev, handler->prio, true); 662 mlx5_ib_counters_clear_description(handler->ibcounters); 663 mutex_unlock(&dev->flow_db->lock); 664 if (handler->flow_matcher) 665 atomic_dec(&handler->flow_matcher->usecnt); 666 kfree(handler); 667 668 return 0; 669 } 670 671 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 672 { 673 priority *= 2; 674 if (!dont_trap) 675 priority++; 676 return priority; 677 } 678 679 enum flow_table_type { 680 MLX5_IB_FT_RX, 681 MLX5_IB_FT_TX 682 }; 683 684 #define MLX5_FS_MAX_TYPES 6 685 #define MLX5_FS_MAX_ENTRIES BIT(16) 686 687 static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device) 688 { 689 struct mlx5_ib_dev *dev = to_mdev(device); 690 691 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed); 692 } 693 694 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_ib_dev *dev, 695 struct mlx5_flow_namespace *ns, 696 struct mlx5_ib_flow_prio *prio, 697 int priority, 698 int num_entries, int num_groups, 699 u32 flags, u16 vport) 700 { 701 struct mlx5_flow_table_attr ft_attr = {}; 702 struct mlx5_flow_table *ft; 703 704 ft_attr.prio = priority; 705 ft_attr.max_fte = num_entries; 706 ft_attr.flags = flags; 707 ft_attr.vport = vport; 708 ft_attr.autogroup.max_num_groups = num_groups; 709 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 710 if (IS_ERR(ft)) 711 return ERR_CAST(ft); 712 713 prio->flow_table = ft; 714 prio->refcount = 0; 715 return prio; 716 } 717 718 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 719 struct ib_flow_attr *flow_attr, 720 enum flow_table_type ft_type) 721 { 722 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 723 struct mlx5_flow_namespace *ns = NULL; 724 enum mlx5_flow_namespace_type fn_type; 725 struct mlx5_ib_flow_prio *prio; 726 struct mlx5_flow_table *ft; 727 int max_table_size; 728 int num_entries; 729 int num_groups; 730 bool esw_encap; 731 u32 flags = 0; 732 int priority; 733 734 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 735 log_max_ft_size)); 736 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 737 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 738 switch (flow_attr->type) { 739 case IB_FLOW_ATTR_NORMAL: 740 if (flow_is_multicast_only(flow_attr) && !dont_trap) 741 priority = MLX5_IB_FLOW_MCAST_PRIO; 742 else 743 priority = ib_prio_to_core_prio(flow_attr->priority, 744 dont_trap); 745 if (ft_type == MLX5_IB_FT_RX) { 746 fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 747 prio = &dev->flow_db->prios[priority]; 748 if (!dev->is_rep && !esw_encap && 749 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 750 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 751 if (!dev->is_rep && !esw_encap && 752 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 753 reformat_l3_tunnel_to_l2)) 754 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 755 } else { 756 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( 757 dev->mdev, log_max_ft_size)); 758 fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 759 prio = &dev->flow_db->egress_prios[priority]; 760 if (!dev->is_rep && !esw_encap && 761 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 762 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 763 } 764 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 765 num_entries = MLX5_FS_MAX_ENTRIES; 766 num_groups = MLX5_FS_MAX_TYPES; 767 break; 768 case IB_FLOW_ATTR_ALL_DEFAULT: 769 case IB_FLOW_ATTR_MC_DEFAULT: 770 ns = mlx5_get_flow_namespace(dev->mdev, 771 MLX5_FLOW_NAMESPACE_LEFTOVERS); 772 build_leftovers_ft_param(&priority, &num_entries, &num_groups); 773 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 774 break; 775 case IB_FLOW_ATTR_SNIFFER: 776 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 777 allow_sniffer_and_nic_rx_shared_tir)) 778 return ERR_PTR(-EOPNOTSUPP); 779 780 ns = mlx5_get_flow_namespace( 781 dev->mdev, ft_type == MLX5_IB_FT_RX ? 782 MLX5_FLOW_NAMESPACE_SNIFFER_RX : 783 MLX5_FLOW_NAMESPACE_SNIFFER_TX); 784 785 prio = &dev->flow_db->sniffer[ft_type]; 786 priority = 0; 787 num_entries = 1; 788 num_groups = 1; 789 break; 790 default: 791 break; 792 } 793 794 if (!ns) 795 return ERR_PTR(-EOPNOTSUPP); 796 797 max_table_size = min_t(int, num_entries, max_table_size); 798 799 ft = prio->flow_table; 800 if (!ft) 801 return _get_prio(dev, ns, prio, priority, max_table_size, 802 num_groups, flags, 0); 803 804 return prio; 805 } 806 807 enum { 808 RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO, 809 RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO, 810 RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 811 RDMA_RX_ECN_OPCOUNTER_PRIO, 812 RDMA_RX_CNP_OPCOUNTER_PRIO, 813 RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO, 814 }; 815 816 enum { 817 RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO, 818 RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 819 RDMA_TX_CNP_OPCOUNTER_PRIO, 820 RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO, 821 }; 822 823 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num, 824 struct mlx5_flow_spec *spec) 825 { 826 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 827 ft_field_support.source_vhca_port) || 828 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 829 ft_field_support.source_vhca_port)) 830 return -EOPNOTSUPP; 831 832 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria, 833 misc_parameters.source_vhca_port); 834 MLX5_SET(fte_match_param, &spec->match_value, 835 misc_parameters.source_vhca_port, port_num); 836 837 return 0; 838 } 839 840 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num, 841 struct mlx5_flow_spec *spec, int ipv) 842 { 843 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 844 ft_field_support.outer_ip_version)) 845 return -EOPNOTSUPP; 846 847 if (mlx5_core_mp_enabled(dev->mdev) && 848 set_vhca_port_spec(dev, port_num, spec)) 849 return -EOPNOTSUPP; 850 851 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 852 outer_headers.ip_ecn); 853 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn, 854 INET_ECN_CE); 855 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 856 outer_headers.ip_version); 857 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 858 ipv); 859 860 spec->match_criteria_enable = 861 get_match_criteria_enable(spec->match_criteria); 862 863 return 0; 864 } 865 866 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num, 867 struct mlx5_flow_spec *spec) 868 { 869 if (mlx5_core_mp_enabled(dev->mdev) && 870 set_vhca_port_spec(dev, port_num, spec)) 871 return -EOPNOTSUPP; 872 873 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 874 misc_parameters.bth_opcode); 875 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode, 876 IB_BTH_OPCODE_CNP); 877 878 spec->match_criteria_enable = 879 get_match_criteria_enable(spec->match_criteria); 880 881 return 0; 882 } 883 884 /* Returns the prio we should use for the given optional counter type, 885 * whereas for bytes type we use the packet type, since they share the same 886 * resources. 887 */ 888 static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev, 889 u32 type) 890 { 891 u32 prio_type; 892 893 switch (type) { 894 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 895 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS; 896 break; 897 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 898 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS; 899 break; 900 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 901 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 902 break; 903 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 904 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 905 break; 906 default: 907 prio_type = type; 908 } 909 910 return &dev->flow_db->opfcs[prio_type]; 911 } 912 913 static void put_per_qp_prio(struct mlx5_ib_dev *dev, 914 enum mlx5_ib_optional_counter_type type) 915 { 916 enum mlx5_ib_optional_counter_type per_qp_type; 917 struct mlx5_ib_flow_prio *prio; 918 919 switch (type) { 920 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 921 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 922 break; 923 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 924 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 925 break; 926 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 927 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 928 break; 929 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 930 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 931 break; 932 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 933 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 934 break; 935 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 936 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 937 break; 938 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 939 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 940 break; 941 default: 942 return; 943 } 944 945 prio = get_opfc_prio(dev, per_qp_type); 946 put_flow_table(dev, prio, true); 947 } 948 949 static int get_per_qp_prio(struct mlx5_ib_dev *dev, 950 enum mlx5_ib_optional_counter_type type) 951 { 952 enum mlx5_ib_optional_counter_type per_qp_type; 953 enum mlx5_flow_namespace_type fn_type; 954 struct mlx5_flow_namespace *ns; 955 struct mlx5_ib_flow_prio *prio; 956 int priority; 957 958 switch (type) { 959 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 960 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 961 priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO; 962 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 963 break; 964 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 965 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 966 priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO; 967 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 968 break; 969 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 970 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 971 priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO; 972 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 973 break; 974 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 975 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 976 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 977 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 978 break; 979 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 980 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 981 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 982 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 983 break; 984 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 985 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 986 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 987 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 988 break; 989 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 990 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 991 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 992 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 993 break; 994 default: 995 return -EINVAL; 996 } 997 998 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 999 if (!ns) 1000 return -EOPNOTSUPP; 1001 1002 prio = get_opfc_prio(dev, per_qp_type); 1003 if (prio->flow_table) 1004 return 0; 1005 1006 prio = _get_prio(dev, ns, prio, priority, MLX5_FS_MAX_POOL_SIZE, 1, 0, 0); 1007 if (IS_ERR(prio)) 1008 return PTR_ERR(prio); 1009 1010 prio->refcount = 1; 1011 1012 return 0; 1013 } 1014 1015 static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa, 1016 u32 qp_num, bool *new) 1017 { 1018 struct mlx5_per_qp_opfc *per_qp_opfc; 1019 1020 *new = false; 1021 1022 per_qp_opfc = xa_load(qpn_opfc_xa, qp_num); 1023 if (per_qp_opfc) 1024 return per_qp_opfc; 1025 per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL); 1026 1027 if (!per_qp_opfc) 1028 return NULL; 1029 1030 *new = true; 1031 return per_qp_opfc; 1032 } 1033 1034 static int add_op_fc_rules(struct mlx5_ib_dev *dev, 1035 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], 1036 struct xarray *qpn_opfc_xa, 1037 struct mlx5_per_qp_opfc *per_qp_opfc, 1038 struct mlx5_ib_flow_prio *prio, 1039 enum mlx5_ib_optional_counter_type type, 1040 u32 qp_num, u32 port_num) 1041 { 1042 struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc; 1043 struct mlx5_flow_act flow_act = {}; 1044 struct mlx5_flow_destination dst; 1045 struct mlx5_flow_spec *spec; 1046 int i, err, spec_num; 1047 bool is_tx; 1048 1049 if (opfc->fc) 1050 return -EEXIST; 1051 1052 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type, 1053 &in_use_opfc)) { 1054 opfc->fc = in_use_opfc->fc; 1055 opfc->rule[0] = in_use_opfc->rule[0]; 1056 return 0; 1057 } 1058 1059 opfc->fc = fc_arr[type]; 1060 1061 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1062 if (!spec) { 1063 err = -ENOMEM; 1064 goto null_fc; 1065 } 1066 1067 switch (type) { 1068 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP: 1069 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1070 MLX5_FS_IPV4_VERSION) || 1071 set_ecn_ce_spec(dev, port_num, &spec[1], 1072 MLX5_FS_IPV6_VERSION)) { 1073 err = -EOPNOTSUPP; 1074 goto free_spec; 1075 } 1076 spec_num = 2; 1077 is_tx = false; 1078 1079 MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria, 1080 misc_parameters.bth_dst_qp); 1081 MLX5_SET(fte_match_param, spec[1].match_value, 1082 misc_parameters.bth_dst_qp, qp_num); 1083 spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1084 break; 1085 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP: 1086 if (!MLX5_CAP_FLOWTABLE( 1087 dev->mdev, 1088 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1089 set_cnp_spec(dev, port_num, &spec[0])) { 1090 err = -EOPNOTSUPP; 1091 goto free_spec; 1092 } 1093 spec_num = 1; 1094 is_tx = false; 1095 break; 1096 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP: 1097 if (!MLX5_CAP_FLOWTABLE( 1098 dev->mdev, 1099 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1100 set_cnp_spec(dev, port_num, &spec[0])) { 1101 err = -EOPNOTSUPP; 1102 goto free_spec; 1103 } 1104 spec_num = 1; 1105 is_tx = true; 1106 break; 1107 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1108 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1109 spec_num = 1; 1110 is_tx = true; 1111 break; 1112 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1113 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1114 spec_num = 1; 1115 is_tx = false; 1116 break; 1117 default: 1118 err = -EINVAL; 1119 goto free_spec; 1120 } 1121 1122 if (is_tx) { 1123 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1124 misc_parameters.source_sqn); 1125 MLX5_SET(fte_match_param, spec->match_value, 1126 misc_parameters.source_sqn, qp_num); 1127 } else { 1128 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1129 misc_parameters.bth_dst_qp); 1130 MLX5_SET(fte_match_param, spec->match_value, 1131 misc_parameters.bth_dst_qp, qp_num); 1132 } 1133 1134 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1135 1136 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1137 dst.counter = opfc->fc; 1138 1139 flow_act.action = 1140 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1141 1142 for (i = 0; i < spec_num; i++) { 1143 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1144 &flow_act, &dst, 1); 1145 if (IS_ERR(opfc->rule[i])) { 1146 err = PTR_ERR(opfc->rule[i]); 1147 goto del_rules; 1148 } 1149 } 1150 prio->refcount += spec_num; 1151 1152 err = xa_err(xa_store(qpn_opfc_xa, qp_num, per_qp_opfc, GFP_KERNEL)); 1153 if (err) 1154 goto del_rules; 1155 1156 kfree(spec); 1157 1158 return 0; 1159 1160 del_rules: 1161 while (i--) 1162 mlx5_del_flow_rules(opfc->rule[i]); 1163 put_flow_table(dev, prio, false); 1164 free_spec: 1165 kfree(spec); 1166 null_fc: 1167 opfc->fc = NULL; 1168 return err; 1169 } 1170 1171 static bool 1172 is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type, 1173 struct mlx5_fc **fc) 1174 { 1175 u32 shared_fc_type; 1176 1177 switch (type) { 1178 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1179 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 1180 break; 1181 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1182 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 1183 break; 1184 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1185 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 1186 break; 1187 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1188 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 1189 break; 1190 default: 1191 return false; 1192 } 1193 1194 *fc = fc_arr[shared_fc_type]; 1195 if (!(*fc)) 1196 return false; 1197 1198 return true; 1199 } 1200 1201 void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev, 1202 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX]) 1203 { 1204 struct mlx5_fc *in_use_fc; 1205 int i; 1206 1207 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1208 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1209 if (!fc_arr[i]) 1210 continue; 1211 1212 if (is_fc_shared_and_in_use(fc_arr, i, &in_use_fc)) { 1213 fc_arr[i] = NULL; 1214 continue; 1215 } 1216 1217 mlx5_fc_destroy(dev->mdev, fc_arr[i]); 1218 fc_arr[i] = NULL; 1219 } 1220 } 1221 1222 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num, 1223 struct mlx5_ib_op_fc *opfc, 1224 enum mlx5_ib_optional_counter_type type) 1225 { 1226 enum mlx5_flow_namespace_type fn_type; 1227 int priority, i, err, spec_num; 1228 struct mlx5_flow_act flow_act = {}; 1229 struct mlx5_flow_destination dst; 1230 struct mlx5_flow_namespace *ns; 1231 struct mlx5_ib_flow_prio *prio; 1232 struct mlx5_flow_spec *spec; 1233 1234 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1235 if (!spec) 1236 return -ENOMEM; 1237 1238 switch (type) { 1239 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 1240 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1241 MLX5_FS_IPV4_VERSION) || 1242 set_ecn_ce_spec(dev, port_num, &spec[1], 1243 MLX5_FS_IPV6_VERSION)) { 1244 err = -EOPNOTSUPP; 1245 goto free; 1246 } 1247 spec_num = 2; 1248 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1249 priority = RDMA_RX_ECN_OPCOUNTER_PRIO; 1250 break; 1251 1252 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 1253 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1254 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1255 set_cnp_spec(dev, port_num, &spec[0])) { 1256 err = -EOPNOTSUPP; 1257 goto free; 1258 } 1259 spec_num = 1; 1260 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1261 priority = RDMA_RX_CNP_OPCOUNTER_PRIO; 1262 break; 1263 1264 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 1265 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1266 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1267 set_cnp_spec(dev, port_num, &spec[0])) { 1268 err = -EOPNOTSUPP; 1269 goto free; 1270 } 1271 spec_num = 1; 1272 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1273 priority = RDMA_TX_CNP_OPCOUNTER_PRIO; 1274 break; 1275 1276 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 1277 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 1278 spec_num = 1; 1279 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1280 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO; 1281 break; 1282 1283 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 1284 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 1285 spec_num = 1; 1286 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1287 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO; 1288 break; 1289 1290 default: 1291 err = -EOPNOTSUPP; 1292 goto free; 1293 } 1294 1295 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 1296 if (!ns) { 1297 err = -EOPNOTSUPP; 1298 goto free; 1299 } 1300 1301 prio = get_opfc_prio(dev, type); 1302 if (!prio->flow_table) { 1303 err = get_per_qp_prio(dev, type); 1304 if (err) 1305 goto free; 1306 1307 prio = _get_prio(dev, ns, prio, priority, 1308 dev->num_ports * MAX_OPFC_RULES, 1, 0, 0); 1309 if (IS_ERR(prio)) { 1310 err = PTR_ERR(prio); 1311 goto put_prio; 1312 } 1313 } 1314 1315 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1316 dst.counter = opfc->fc; 1317 1318 flow_act.action = 1319 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1320 1321 for (i = 0; i < spec_num; i++) { 1322 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1323 &flow_act, &dst, 1); 1324 if (IS_ERR(opfc->rule[i])) { 1325 err = PTR_ERR(opfc->rule[i]); 1326 goto del_rules; 1327 } 1328 } 1329 prio->refcount += spec_num; 1330 kfree(spec); 1331 1332 return 0; 1333 1334 del_rules: 1335 for (i -= 1; i >= 0; i--) 1336 mlx5_del_flow_rules(opfc->rule[i]); 1337 put_flow_table(dev, prio, false); 1338 put_prio: 1339 put_per_qp_prio(dev, type); 1340 free: 1341 kfree(spec); 1342 return err; 1343 } 1344 1345 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev, 1346 struct mlx5_ib_op_fc *opfc, 1347 enum mlx5_ib_optional_counter_type type) 1348 { 1349 struct mlx5_ib_flow_prio *prio; 1350 int i; 1351 1352 prio = get_opfc_prio(dev, type); 1353 1354 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) { 1355 mlx5_del_flow_rules(opfc->rule[i]); 1356 put_flow_table(dev, prio, true); 1357 } 1358 1359 put_per_qp_prio(dev, type); 1360 } 1361 1362 void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa) 1363 { 1364 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1365 struct mlx5_per_qp_opfc *per_qp_opfc; 1366 struct mlx5_ib_op_fc *in_use_opfc; 1367 struct mlx5_ib_flow_prio *prio; 1368 int i, j; 1369 1370 per_qp_opfc = xa_load(qpn_opfc_xa, qp->qp_num); 1371 if (!per_qp_opfc) 1372 return; 1373 1374 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1375 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1376 if (!per_qp_opfc->opfcs[i].fc) 1377 continue; 1378 1379 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i, 1380 &in_use_opfc)) { 1381 per_qp_opfc->opfcs[i].fc = NULL; 1382 continue; 1383 } 1384 1385 for (j = 0; j < MAX_OPFC_RULES; j++) { 1386 if (!per_qp_opfc->opfcs[i].rule[j]) 1387 continue; 1388 mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]); 1389 prio = get_opfc_prio(dev, i); 1390 put_flow_table(dev, prio, true); 1391 } 1392 per_qp_opfc->opfcs[i].fc = NULL; 1393 } 1394 1395 kfree(per_qp_opfc); 1396 xa_erase(qpn_opfc_xa, qp->qp_num); 1397 } 1398 1399 int mlx5r_fs_bind_op_fc(struct ib_qp *qp, 1400 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], 1401 struct xarray *qpn_opfc_xa, u32 port) 1402 { 1403 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1404 struct mlx5_per_qp_opfc *per_qp_opfc; 1405 struct mlx5_ib_flow_prio *prio; 1406 struct mlx5_ib_counters *cnts; 1407 struct mlx5_ib_op_fc *opfc; 1408 struct mlx5_fc *in_use_fc; 1409 int i, err, per_qp_type; 1410 bool new; 1411 1412 cnts = &dev->port[port - 1].cnts; 1413 1414 for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) { 1415 opfc = &cnts->opfcs[i]; 1416 if (!opfc->fc) 1417 continue; 1418 1419 per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1420 prio = get_opfc_prio(dev, per_qp_type); 1421 WARN_ON(!prio->flow_table); 1422 1423 if (is_fc_shared_and_in_use(fc_arr, per_qp_type, &in_use_fc)) 1424 fc_arr[per_qp_type] = in_use_fc; 1425 1426 if (!fc_arr[per_qp_type]) { 1427 fc_arr[per_qp_type] = mlx5_fc_create(dev->mdev, false); 1428 if (IS_ERR(fc_arr[per_qp_type])) 1429 return PTR_ERR(fc_arr[per_qp_type]); 1430 } 1431 1432 per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp->qp_num, &new); 1433 if (!per_qp_opfc) { 1434 err = -ENOMEM; 1435 goto free_fc; 1436 } 1437 err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc, 1438 prio, per_qp_type, qp->qp_num, port); 1439 if (err) 1440 goto del_rules; 1441 } 1442 1443 return 0; 1444 1445 del_rules: 1446 mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa); 1447 if (new) 1448 kfree(per_qp_opfc); 1449 free_fc: 1450 if (xa_empty(qpn_opfc_xa)) 1451 mlx5r_fs_destroy_fcs(dev, fc_arr); 1452 return err; 1453 } 1454 1455 static void set_underlay_qp(struct mlx5_ib_dev *dev, 1456 struct mlx5_flow_spec *spec, 1457 u32 underlay_qpn) 1458 { 1459 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 1460 spec->match_criteria, 1461 misc_parameters); 1462 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1463 misc_parameters); 1464 1465 if (underlay_qpn && 1466 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1467 ft_field_support.bth_dst_qp)) { 1468 MLX5_SET(fte_match_set_misc, 1469 misc_params_v, bth_dst_qp, underlay_qpn); 1470 MLX5_SET(fte_match_set_misc, 1471 misc_params_c, bth_dst_qp, 0xffffff); 1472 } 1473 } 1474 1475 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 1476 struct mlx5_flow_spec *spec, 1477 struct mlx5_eswitch_rep *rep) 1478 { 1479 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 1480 void *misc; 1481 1482 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1483 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1484 misc_parameters_2); 1485 1486 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1487 mlx5_eswitch_get_vport_metadata_for_match(rep->esw, 1488 rep->vport)); 1489 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1490 misc_parameters_2); 1491 1492 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1493 mlx5_eswitch_get_vport_metadata_mask()); 1494 } else { 1495 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1496 misc_parameters); 1497 1498 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 1499 1500 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1501 misc_parameters); 1502 1503 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1504 } 1505 } 1506 1507 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 1508 struct mlx5_ib_flow_prio *ft_prio, 1509 const struct ib_flow_attr *flow_attr, 1510 struct mlx5_flow_destination *dst, 1511 u32 underlay_qpn, 1512 struct mlx5_ib_create_flow *ucmd) 1513 { 1514 struct mlx5_flow_table *ft = ft_prio->flow_table; 1515 struct mlx5_ib_flow_handler *handler; 1516 struct mlx5_flow_act flow_act = {}; 1517 struct mlx5_flow_spec *spec; 1518 struct mlx5_flow_destination dest_arr[2] = {}; 1519 struct mlx5_flow_destination *rule_dst = dest_arr; 1520 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 1521 unsigned int spec_index; 1522 u32 prev_type = 0; 1523 int err = 0; 1524 int dest_num = 0; 1525 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1526 1527 if (!is_valid_attr(dev->mdev, flow_attr)) 1528 return ERR_PTR(-EINVAL); 1529 1530 if (dev->is_rep && is_egress) 1531 return ERR_PTR(-EINVAL); 1532 1533 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1534 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1535 if (!handler || !spec) { 1536 err = -ENOMEM; 1537 goto free; 1538 } 1539 1540 INIT_LIST_HEAD(&handler->list); 1541 1542 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 1543 err = parse_flow_attr(dev->mdev, spec, 1544 ib_flow, flow_attr, &flow_act, 1545 prev_type); 1546 if (err < 0) 1547 goto free; 1548 1549 prev_type = ((union ib_flow_spec *)ib_flow)->type; 1550 ib_flow += ((union ib_flow_spec *)ib_flow)->size; 1551 } 1552 1553 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 1554 memcpy(&dest_arr[0], dst, sizeof(*dst)); 1555 dest_num++; 1556 } 1557 1558 if (!flow_is_multicast_only(flow_attr)) 1559 set_underlay_qp(dev, spec, underlay_qpn); 1560 1561 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { 1562 struct mlx5_eswitch_rep *rep; 1563 1564 rep = dev->port[flow_attr->port - 1].rep; 1565 if (!rep) { 1566 err = -EINVAL; 1567 goto free; 1568 } 1569 1570 mlx5_ib_set_rule_source_port(dev, spec, rep); 1571 } 1572 1573 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 1574 1575 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1576 struct mlx5_ib_mcounters *mcounters; 1577 1578 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 1579 if (err) 1580 goto free; 1581 1582 mcounters = to_mcounters(flow_act.counters); 1583 handler->ibcounters = flow_act.counters; 1584 dest_arr[dest_num].type = 1585 MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1586 dest_arr[dest_num].counter = 1587 mcounters->hw_cntrs_hndl; 1588 dest_num++; 1589 } 1590 1591 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 1592 if (!dest_num) 1593 rule_dst = NULL; 1594 } else { 1595 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 1596 flow_act.action |= 1597 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1598 if (is_egress) 1599 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1600 else if (dest_num) 1601 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1602 } 1603 1604 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1605 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1606 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1607 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1608 spec->flow_context.flow_tag, flow_attr->type); 1609 err = -EINVAL; 1610 goto free; 1611 } 1612 handler->rule = mlx5_add_flow_rules(ft, spec, 1613 &flow_act, 1614 rule_dst, dest_num); 1615 1616 if (IS_ERR(handler->rule)) { 1617 err = PTR_ERR(handler->rule); 1618 goto free; 1619 } 1620 1621 ft_prio->refcount++; 1622 handler->prio = ft_prio; 1623 handler->dev = dev; 1624 1625 ft_prio->flow_table = ft; 1626 free: 1627 if (err && handler) { 1628 mlx5_ib_counters_clear_description(handler->ibcounters); 1629 kfree(handler); 1630 } 1631 kvfree(spec); 1632 return err ? ERR_PTR(err) : handler; 1633 } 1634 1635 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1636 struct mlx5_ib_flow_prio *ft_prio, 1637 const struct ib_flow_attr *flow_attr, 1638 struct mlx5_flow_destination *dst) 1639 { 1640 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1641 } 1642 1643 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1644 struct mlx5_ib_flow_prio *ft_prio, 1645 struct ib_flow_attr *flow_attr, 1646 struct mlx5_flow_destination *dst) 1647 { 1648 struct mlx5_ib_flow_handler *handler_ucast = NULL; 1649 struct mlx5_ib_flow_handler *handler = NULL; 1650 1651 static struct { 1652 struct ib_flow_spec_eth eth_flow; 1653 struct ib_flow_attr flow_attr; 1654 } leftovers_wc = { .flow_attr = { .num_of_specs = 1, 1655 .size = sizeof(leftovers_wc) }, 1656 .eth_flow = { 1657 .type = IB_FLOW_SPEC_ETH, 1658 .size = sizeof(struct ib_flow_spec_eth), 1659 .mask = { .dst_mac = { 0x1 } }, 1660 .val = { .dst_mac = { 0x1 } } } }; 1661 1662 static struct { 1663 struct ib_flow_spec_eth eth_flow; 1664 struct ib_flow_attr flow_attr; 1665 } leftovers_uc = { .flow_attr = { .num_of_specs = 1, 1666 .size = sizeof(leftovers_uc) }, 1667 .eth_flow = { 1668 .type = IB_FLOW_SPEC_ETH, 1669 .size = sizeof(struct ib_flow_spec_eth), 1670 .mask = { .dst_mac = { 0x1 } }, 1671 .val = { .dst_mac = {} } } }; 1672 1673 handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst); 1674 if (!IS_ERR(handler) && 1675 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1676 handler_ucast = create_flow_rule(dev, ft_prio, 1677 &leftovers_uc.flow_attr, dst); 1678 if (IS_ERR(handler_ucast)) { 1679 mlx5_del_flow_rules(handler->rule); 1680 ft_prio->refcount--; 1681 kfree(handler); 1682 handler = handler_ucast; 1683 } else { 1684 list_add(&handler_ucast->list, &handler->list); 1685 } 1686 } 1687 1688 return handler; 1689 } 1690 1691 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1692 struct mlx5_ib_flow_prio *ft_rx, 1693 struct mlx5_ib_flow_prio *ft_tx, 1694 struct mlx5_flow_destination *dst) 1695 { 1696 struct mlx5_ib_flow_handler *handler_rx; 1697 struct mlx5_ib_flow_handler *handler_tx; 1698 int err; 1699 static const struct ib_flow_attr flow_attr = { 1700 .num_of_specs = 0, 1701 .type = IB_FLOW_ATTR_SNIFFER, 1702 .size = sizeof(flow_attr) 1703 }; 1704 1705 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1706 if (IS_ERR(handler_rx)) { 1707 err = PTR_ERR(handler_rx); 1708 goto err; 1709 } 1710 1711 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1712 if (IS_ERR(handler_tx)) { 1713 err = PTR_ERR(handler_tx); 1714 goto err_tx; 1715 } 1716 1717 list_add(&handler_tx->list, &handler_rx->list); 1718 1719 return handler_rx; 1720 1721 err_tx: 1722 mlx5_del_flow_rules(handler_rx->rule); 1723 ft_rx->refcount--; 1724 kfree(handler_rx); 1725 err: 1726 return ERR_PTR(err); 1727 } 1728 1729 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1730 struct ib_flow_attr *flow_attr, 1731 struct ib_udata *udata) 1732 { 1733 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1734 struct mlx5_ib_qp *mqp = to_mqp(qp); 1735 struct mlx5_ib_flow_handler *handler = NULL; 1736 struct mlx5_flow_destination *dst = NULL; 1737 struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1738 struct mlx5_ib_flow_prio *ft_prio; 1739 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1740 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1741 size_t min_ucmd_sz, required_ucmd_sz; 1742 int err; 1743 int underlay_qpn; 1744 1745 if (udata && udata->inlen) { 1746 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); 1747 if (udata->inlen < min_ucmd_sz) 1748 return ERR_PTR(-EOPNOTSUPP); 1749 1750 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1751 if (err) 1752 return ERR_PTR(err); 1753 1754 /* currently supports only one counters data */ 1755 if (ucmd_hdr.ncounters_data > 1) 1756 return ERR_PTR(-EINVAL); 1757 1758 required_ucmd_sz = min_ucmd_sz + 1759 sizeof(struct mlx5_ib_flow_counters_data) * 1760 ucmd_hdr.ncounters_data; 1761 if (udata->inlen > required_ucmd_sz && 1762 !ib_is_udata_cleared(udata, required_ucmd_sz, 1763 udata->inlen - required_ucmd_sz)) 1764 return ERR_PTR(-EOPNOTSUPP); 1765 1766 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1767 if (!ucmd) 1768 return ERR_PTR(-ENOMEM); 1769 1770 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1771 if (err) 1772 goto free_ucmd; 1773 } 1774 1775 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1776 err = -ENOMEM; 1777 goto free_ucmd; 1778 } 1779 1780 if (flow_attr->flags & 1781 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) { 1782 err = -EINVAL; 1783 goto free_ucmd; 1784 } 1785 1786 if (is_egress && 1787 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1788 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1789 err = -EINVAL; 1790 goto free_ucmd; 1791 } 1792 1793 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1794 if (!dst) { 1795 err = -ENOMEM; 1796 goto free_ucmd; 1797 } 1798 1799 mutex_lock(&dev->flow_db->lock); 1800 1801 ft_prio = get_flow_table(dev, flow_attr, 1802 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1803 if (IS_ERR(ft_prio)) { 1804 err = PTR_ERR(ft_prio); 1805 goto unlock; 1806 } 1807 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1808 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1809 if (IS_ERR(ft_prio_tx)) { 1810 err = PTR_ERR(ft_prio_tx); 1811 ft_prio_tx = NULL; 1812 goto destroy_ft; 1813 } 1814 } 1815 1816 if (is_egress) { 1817 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1818 } else { 1819 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1820 if (mqp->is_rss) 1821 dst->tir_num = mqp->rss_qp.tirn; 1822 else 1823 dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1824 } 1825 1826 switch (flow_attr->type) { 1827 case IB_FLOW_ATTR_NORMAL: 1828 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1829 mqp->underlay_qpn : 1830 0; 1831 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1832 underlay_qpn, ucmd); 1833 break; 1834 case IB_FLOW_ATTR_ALL_DEFAULT: 1835 case IB_FLOW_ATTR_MC_DEFAULT: 1836 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); 1837 break; 1838 case IB_FLOW_ATTR_SNIFFER: 1839 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1840 break; 1841 default: 1842 err = -EINVAL; 1843 goto destroy_ft; 1844 } 1845 1846 if (IS_ERR(handler)) { 1847 err = PTR_ERR(handler); 1848 handler = NULL; 1849 goto destroy_ft; 1850 } 1851 1852 mutex_unlock(&dev->flow_db->lock); 1853 kfree(dst); 1854 kfree(ucmd); 1855 1856 return &handler->ibflow; 1857 1858 destroy_ft: 1859 put_flow_table(dev, ft_prio, false); 1860 if (ft_prio_tx) 1861 put_flow_table(dev, ft_prio_tx, false); 1862 unlock: 1863 mutex_unlock(&dev->flow_db->lock); 1864 kfree(dst); 1865 free_ucmd: 1866 kfree(ucmd); 1867 return ERR_PTR(err); 1868 } 1869 1870 static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev, 1871 enum mlx5_flow_namespace_type type, 1872 u32 *flags, u16 *vport_idx, 1873 u16 *vport, 1874 struct mlx5_core_dev **ft_mdev, 1875 u32 ib_port) 1876 { 1877 struct mlx5_core_dev *esw_mdev; 1878 1879 if (!is_mdev_switchdev_mode(dev->mdev)) 1880 return 0; 1881 1882 if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager)) 1883 return -EOPNOTSUPP; 1884 1885 if (!dev->port[ib_port - 1].rep) 1886 return -EINVAL; 1887 1888 esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw); 1889 if (esw_mdev != dev->mdev) 1890 return -EOPNOTSUPP; 1891 1892 *flags |= MLX5_FLOW_TABLE_OTHER_VPORT; 1893 *ft_mdev = esw_mdev; 1894 *vport = dev->port[ib_port - 1].rep->vport; 1895 *vport_idx = dev->port[ib_port - 1].rep->vport_index; 1896 1897 return 0; 1898 } 1899 1900 static struct mlx5_ib_flow_prio * 1901 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority, 1902 enum mlx5_flow_namespace_type ns_type, 1903 bool mcast, u32 ib_port) 1904 { 1905 struct mlx5_core_dev *ft_mdev = dev->mdev; 1906 struct mlx5_flow_namespace *ns = NULL; 1907 struct mlx5_ib_flow_prio *prio = NULL; 1908 int max_table_size = 0; 1909 u16 vport_idx = 0; 1910 bool esw_encap; 1911 u32 flags = 0; 1912 u16 vport = 0; 1913 int priority; 1914 int ret; 1915 1916 if (mcast) 1917 priority = MLX5_IB_FLOW_MCAST_PRIO; 1918 else 1919 priority = ib_prio_to_core_prio(user_priority, false); 1920 1921 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1922 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1923 switch (ns_type) { 1924 case MLX5_FLOW_NAMESPACE_BYPASS: 1925 max_table_size = BIT( 1926 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); 1927 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1928 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1929 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1930 reformat_l3_tunnel_to_l2) && 1931 !esw_encap) 1932 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1933 break; 1934 case MLX5_FLOW_NAMESPACE_EGRESS: 1935 max_table_size = BIT( 1936 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1937 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && 1938 !esw_encap) 1939 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1940 break; 1941 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1942 max_table_size = BIT( 1943 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1944 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1945 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1946 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, 1947 reformat_l3_tunnel_to_l2) && 1948 esw_encap) 1949 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1950 priority = user_priority; 1951 break; 1952 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1953 max_table_size = BIT( 1954 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); 1955 priority = user_priority; 1956 break; 1957 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1958 max_table_size = BIT( 1959 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); 1960 priority = user_priority; 1961 break; 1962 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 1963 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 1964 if (ib_port == 0 || 1965 user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO) 1966 return ERR_PTR(-EINVAL); 1967 ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags, 1968 &vport_idx, &vport, 1969 &ft_mdev, ib_port); 1970 if (ret) 1971 return ERR_PTR(ret); 1972 1973 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) 1974 max_table_size = 1975 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX( 1976 ft_mdev, log_max_ft_size)); 1977 else 1978 max_table_size = 1979 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX( 1980 ft_mdev, log_max_ft_size)); 1981 priority = user_priority; 1982 break; 1983 default: 1984 break; 1985 } 1986 1987 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1988 1989 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX || 1990 ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) 1991 ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx); 1992 else 1993 ns = mlx5_get_flow_namespace(ft_mdev, ns_type); 1994 1995 if (!ns) 1996 return ERR_PTR(-EOPNOTSUPP); 1997 1998 switch (ns_type) { 1999 case MLX5_FLOW_NAMESPACE_BYPASS: 2000 prio = &dev->flow_db->prios[priority]; 2001 break; 2002 case MLX5_FLOW_NAMESPACE_EGRESS: 2003 prio = &dev->flow_db->egress_prios[priority]; 2004 break; 2005 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 2006 prio = &dev->flow_db->fdb[priority]; 2007 break; 2008 case MLX5_FLOW_NAMESPACE_RDMA_RX: 2009 prio = &dev->flow_db->rdma_rx[priority]; 2010 break; 2011 case MLX5_FLOW_NAMESPACE_RDMA_TX: 2012 prio = &dev->flow_db->rdma_tx[priority]; 2013 break; 2014 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 2015 prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1]; 2016 break; 2017 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 2018 prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1]; 2019 break; 2020 default: return ERR_PTR(-EINVAL); 2021 } 2022 2023 if (!prio) 2024 return ERR_PTR(-EINVAL); 2025 2026 if (prio->flow_table) 2027 return prio; 2028 2029 return _get_prio(dev, ns, prio, priority, max_table_size, 2030 MLX5_FS_MAX_TYPES, flags, vport); 2031 } 2032 2033 static struct mlx5_ib_flow_handler * 2034 _create_raw_flow_rule(struct mlx5_ib_dev *dev, 2035 struct mlx5_ib_flow_prio *ft_prio, 2036 struct mlx5_flow_destination *dst, 2037 struct mlx5_ib_flow_matcher *fs_matcher, 2038 struct mlx5_flow_context *flow_context, 2039 struct mlx5_flow_act *flow_act, 2040 void *cmd_in, int inlen, 2041 int dst_num) 2042 { 2043 struct mlx5_ib_flow_handler *handler; 2044 struct mlx5_flow_spec *spec; 2045 struct mlx5_flow_table *ft = ft_prio->flow_table; 2046 int err = 0; 2047 2048 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2049 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 2050 if (!handler || !spec) { 2051 err = -ENOMEM; 2052 goto free; 2053 } 2054 2055 INIT_LIST_HEAD(&handler->list); 2056 2057 memcpy(spec->match_value, cmd_in, inlen); 2058 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 2059 fs_matcher->mask_len); 2060 spec->match_criteria_enable = fs_matcher->match_criteria_enable; 2061 spec->flow_context = *flow_context; 2062 2063 handler->rule = mlx5_add_flow_rules(ft, spec, 2064 flow_act, dst, dst_num); 2065 2066 if (IS_ERR(handler->rule)) { 2067 err = PTR_ERR(handler->rule); 2068 goto free; 2069 } 2070 2071 ft_prio->refcount++; 2072 handler->prio = ft_prio; 2073 handler->dev = dev; 2074 ft_prio->flow_table = ft; 2075 2076 free: 2077 if (err) 2078 kfree(handler); 2079 kvfree(spec); 2080 return err ? ERR_PTR(err) : handler; 2081 } 2082 2083 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 2084 void *match_v) 2085 { 2086 void *match_c; 2087 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 2088 void *dmac, *dmac_mask; 2089 void *ipv4, *ipv4_mask; 2090 2091 if (!(fs_matcher->match_criteria_enable & 2092 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 2093 return false; 2094 2095 match_c = fs_matcher->matcher_mask.match_params; 2096 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 2097 outer_headers); 2098 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 2099 outer_headers); 2100 2101 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2102 dmac_47_16); 2103 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2104 dmac_47_16); 2105 2106 if (is_multicast_ether_addr(dmac) && 2107 is_multicast_ether_addr(dmac_mask)) 2108 return true; 2109 2110 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2111 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2112 2113 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2114 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2115 2116 if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 2117 ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 2118 return true; 2119 2120 return false; 2121 } 2122 2123 static struct mlx5_ib_flow_handler *raw_fs_rule_add( 2124 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 2125 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 2126 struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type) 2127 { 2128 struct mlx5_flow_destination *dst; 2129 struct mlx5_ib_flow_prio *ft_prio; 2130 struct mlx5_ib_flow_handler *handler; 2131 int dst_num = 0; 2132 bool mcast; 2133 int err; 2134 2135 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 2136 return ERR_PTR(-EOPNOTSUPP); 2137 2138 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 2139 return ERR_PTR(-ENOMEM); 2140 2141 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 2142 if (!dst) 2143 return ERR_PTR(-ENOMEM); 2144 2145 mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 2146 mutex_lock(&dev->flow_db->lock); 2147 2148 ft_prio = _get_flow_table(dev, fs_matcher->priority, 2149 fs_matcher->ns_type, mcast, 2150 fs_matcher->ib_port); 2151 if (IS_ERR(ft_prio)) { 2152 err = PTR_ERR(ft_prio); 2153 goto unlock; 2154 } 2155 2156 switch (dest_type) { 2157 case MLX5_FLOW_DESTINATION_TYPE_TIR: 2158 dst[dst_num].type = dest_type; 2159 dst[dst_num++].tir_num = dest_id; 2160 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2161 break; 2162 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: 2163 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 2164 dst[dst_num++].ft_num = dest_id; 2165 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2166 break; 2167 case MLX5_FLOW_DESTINATION_TYPE_PORT: 2168 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2169 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 2170 break; 2171 default: 2172 break; 2173 } 2174 2175 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 2176 if (WARN_ON(!counter)) { 2177 err = -EINVAL; 2178 goto unlock; 2179 } 2180 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2181 dst[dst_num].counter = counter; 2182 dst_num++; 2183 } 2184 2185 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, 2186 fs_matcher, flow_context, flow_act, 2187 cmd_in, inlen, dst_num); 2188 2189 if (IS_ERR(handler)) { 2190 err = PTR_ERR(handler); 2191 goto destroy_ft; 2192 } 2193 2194 mutex_unlock(&dev->flow_db->lock); 2195 atomic_inc(&fs_matcher->usecnt); 2196 handler->flow_matcher = fs_matcher; 2197 2198 kfree(dst); 2199 2200 return handler; 2201 2202 destroy_ft: 2203 put_flow_table(dev, ft_prio, false); 2204 unlock: 2205 mutex_unlock(&dev->flow_db->lock); 2206 kfree(dst); 2207 2208 return ERR_PTR(err); 2209 } 2210 2211 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 2212 { 2213 switch (maction->flow_action_raw.sub_type) { 2214 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 2215 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 2216 maction->flow_action_raw.modify_hdr); 2217 break; 2218 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 2219 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 2220 maction->flow_action_raw.pkt_reformat); 2221 break; 2222 case MLX5_IB_FLOW_ACTION_DECAP: 2223 break; 2224 default: 2225 break; 2226 } 2227 } 2228 2229 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 2230 { 2231 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 2232 2233 switch (action->type) { 2234 case IB_FLOW_ACTION_UNSPECIFIED: 2235 destroy_flow_action_raw(maction); 2236 break; 2237 default: 2238 WARN_ON(true); 2239 break; 2240 } 2241 2242 kfree(maction); 2243 return 0; 2244 } 2245 2246 static int 2247 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 2248 enum mlx5_flow_namespace_type *namespace) 2249 { 2250 switch (table_type) { 2251 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 2252 *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 2253 break; 2254 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 2255 *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 2256 break; 2257 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 2258 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS; 2259 break; 2260 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 2261 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 2262 break; 2263 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 2264 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 2265 break; 2266 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX: 2267 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX; 2268 break; 2269 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX: 2270 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX; 2271 break; 2272 default: 2273 return -EINVAL; 2274 } 2275 2276 return 0; 2277 } 2278 2279 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 2280 [MLX5_IB_FLOW_TYPE_NORMAL] = { 2281 .type = UVERBS_ATTR_TYPE_PTR_IN, 2282 .u.ptr = { 2283 .len = sizeof(u16), /* data is priority */ 2284 .min_len = sizeof(u16), 2285 } 2286 }, 2287 [MLX5_IB_FLOW_TYPE_SNIFFER] = { 2288 .type = UVERBS_ATTR_TYPE_PTR_IN, 2289 UVERBS_ATTR_NO_DATA(), 2290 }, 2291 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 2292 .type = UVERBS_ATTR_TYPE_PTR_IN, 2293 UVERBS_ATTR_NO_DATA(), 2294 }, 2295 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 2296 .type = UVERBS_ATTR_TYPE_PTR_IN, 2297 UVERBS_ATTR_NO_DATA(), 2298 }, 2299 }; 2300 2301 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 2302 { 2303 struct devx_obj *devx_obj = obj; 2304 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2305 2306 switch (opcode) { 2307 case MLX5_CMD_OP_DESTROY_TIR: 2308 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2309 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 2310 obj_id); 2311 return true; 2312 2313 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 2314 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2315 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 2316 table_id); 2317 return true; 2318 default: 2319 return false; 2320 } 2321 } 2322 2323 static int get_dests(struct uverbs_attr_bundle *attrs, 2324 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 2325 int *dest_type, struct ib_qp **qp, u32 *flags) 2326 { 2327 bool dest_devx, dest_qp; 2328 void *devx_obj; 2329 int err; 2330 2331 dest_devx = uverbs_attr_is_valid(attrs, 2332 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2333 dest_qp = uverbs_attr_is_valid(attrs, 2334 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2335 2336 *flags = 0; 2337 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2338 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 2339 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 2340 if (err) 2341 return err; 2342 2343 /* Both flags are not allowed */ 2344 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 2345 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2346 return -EINVAL; 2347 2348 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 2349 if (dest_devx && (dest_qp || *flags)) 2350 return -EINVAL; 2351 else if (dest_qp && *flags) 2352 return -EINVAL; 2353 } 2354 2355 /* Allow only DEVX object, drop as dest for FDB */ 2356 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2357 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 2358 return -EINVAL; 2359 2360 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 2361 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2362 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2363 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 2364 return -EINVAL; 2365 2366 *qp = NULL; 2367 if (dest_devx) { 2368 devx_obj = 2369 uverbs_attr_get_obj(attrs, 2370 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2371 2372 /* Verify that the given DEVX object is a flow 2373 * steering destination. 2374 */ 2375 if (!is_flow_dest(devx_obj, dest_id, dest_type)) 2376 return -EINVAL; 2377 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 2378 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS || 2379 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2380 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2381 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 2382 return -EINVAL; 2383 } else if (dest_qp) { 2384 struct mlx5_ib_qp *mqp; 2385 2386 *qp = uverbs_attr_get_obj(attrs, 2387 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2388 if (IS_ERR(*qp)) 2389 return PTR_ERR(*qp); 2390 2391 if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 2392 return -EINVAL; 2393 2394 mqp = to_mqp(*qp); 2395 if (mqp->is_rss) 2396 *dest_id = mqp->rss_qp.tirn; 2397 else 2398 *dest_id = mqp->raw_packet_qp.rq.tirn; 2399 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2400 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2401 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2402 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) && 2403 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { 2404 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2405 } 2406 2407 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 2408 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2409 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2410 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)) 2411 return -EINVAL; 2412 2413 return 0; 2414 } 2415 2416 static bool 2417 is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size) 2418 { 2419 struct devx_obj *devx_obj = obj; 2420 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2421 2422 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 2423 2424 if (offset && offset >= devx_obj->flow_counter_bulk_size) 2425 return false; 2426 2427 *fc_bulk_size = devx_obj->flow_counter_bulk_size; 2428 *counter_id = MLX5_GET(dealloc_flow_counter_in, 2429 devx_obj->dinbox, 2430 flow_counter_id); 2431 *counter_id += offset; 2432 return true; 2433 } 2434 2435 return false; 2436 } 2437 2438 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 2439 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 2440 struct uverbs_attr_bundle *attrs) 2441 { 2442 struct mlx5_flow_context flow_context = {.flow_tag = 2443 MLX5_FS_DEFAULT_FLOW_TAG}; 2444 int dest_id, dest_type = -1, inlen, len, ret, i; 2445 struct mlx5_ib_flow_handler *flow_handler; 2446 struct mlx5_ib_flow_matcher *fs_matcher; 2447 struct ib_uobject **arr_flow_actions; 2448 struct ib_uflow_resources *uflow_res; 2449 struct mlx5_flow_act flow_act = {}; 2450 struct mlx5_fc *counter = NULL; 2451 struct ib_qp *qp = NULL; 2452 void *devx_obj, *cmd_in; 2453 struct ib_uobject *uobj; 2454 struct mlx5_ib_dev *dev; 2455 u32 flags; 2456 2457 if (!rdma_uattrs_has_raw_cap(attrs)) 2458 return -EPERM; 2459 2460 fs_matcher = uverbs_attr_get_obj(attrs, 2461 MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 2462 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 2463 dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2464 2465 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 2466 return -EINVAL; 2467 2468 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 2469 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 2470 2471 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2472 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 2473 2474 len = uverbs_attr_get_uobjs_arr(attrs, 2475 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 2476 if (len) { 2477 u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0; 2478 devx_obj = arr_flow_actions[0]->object; 2479 2480 if (uverbs_attr_is_valid(attrs, 2481 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 2482 2483 int num_offsets = uverbs_attr_ptr_get_array_size( 2484 attrs, 2485 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2486 sizeof(u32)); 2487 2488 if (num_offsets != 1) 2489 return -EINVAL; 2490 2491 offset_attr = uverbs_attr_get_alloced_ptr( 2492 attrs, 2493 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 2494 offset = *offset_attr; 2495 } 2496 2497 if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size)) 2498 return -EINVAL; 2499 counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size); 2500 if (IS_ERR(counter)) 2501 return PTR_ERR(counter); 2502 2503 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 2504 } 2505 2506 cmd_in = uverbs_attr_get_alloced_ptr( 2507 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2508 inlen = uverbs_attr_get_len(attrs, 2509 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2510 2511 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 2512 if (!uflow_res) { 2513 ret = -ENOMEM; 2514 goto destroy_counter; 2515 } 2516 2517 len = uverbs_attr_get_uobjs_arr(attrs, 2518 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 2519 for (i = 0; i < len; i++) { 2520 struct mlx5_ib_flow_action *maction = 2521 to_mflow_act(arr_flow_actions[i]->object); 2522 2523 ret = parse_flow_flow_action(maction, false, &flow_act); 2524 if (ret) 2525 goto err_out; 2526 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 2527 arr_flow_actions[i]->object); 2528 } 2529 2530 ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 2531 MLX5_IB_ATTR_CREATE_FLOW_TAG); 2532 if (!ret) { 2533 if (flow_context.flow_tag >= BIT(24)) { 2534 ret = -EINVAL; 2535 goto err_out; 2536 } 2537 flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 2538 } 2539 2540 flow_handler = 2541 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 2542 counter, cmd_in, inlen, dest_id, dest_type); 2543 if (IS_ERR(flow_handler)) { 2544 ret = PTR_ERR(flow_handler); 2545 goto err_out; 2546 } 2547 2548 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2549 2550 return 0; 2551 err_out: 2552 ib_uverbs_flow_resources_free(uflow_res); 2553 destroy_counter: 2554 if (counter) 2555 mlx5_fc_local_destroy(counter); 2556 return ret; 2557 } 2558 2559 static int flow_matcher_cleanup(struct ib_uobject *uobject, 2560 enum rdma_remove_reason why, 2561 struct uverbs_attr_bundle *attrs) 2562 { 2563 struct mlx5_ib_flow_matcher *obj = uobject->object; 2564 2565 if (atomic_read(&obj->usecnt)) 2566 return -EBUSY; 2567 2568 kfree(obj); 2569 return 0; 2570 } 2571 2572 static int steering_anchor_create_ft(struct mlx5_ib_dev *dev, 2573 struct mlx5_ib_flow_prio *ft_prio, 2574 enum mlx5_flow_namespace_type ns_type) 2575 { 2576 struct mlx5_flow_table_attr ft_attr = {}; 2577 struct mlx5_flow_namespace *ns; 2578 struct mlx5_flow_table *ft; 2579 2580 if (ft_prio->anchor.ft) 2581 return 0; 2582 2583 ns = mlx5_get_flow_namespace(dev->mdev, ns_type); 2584 if (!ns) 2585 return -EOPNOTSUPP; 2586 2587 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 2588 ft_attr.uid = MLX5_SHARED_RESOURCE_UID; 2589 ft_attr.prio = 0; 2590 ft_attr.max_fte = 2; 2591 ft_attr.level = 1; 2592 2593 ft = mlx5_create_flow_table(ns, &ft_attr); 2594 if (IS_ERR(ft)) 2595 return PTR_ERR(ft); 2596 2597 ft_prio->anchor.ft = ft; 2598 2599 return 0; 2600 } 2601 2602 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio) 2603 { 2604 if (ft_prio->anchor.ft) { 2605 mlx5_destroy_flow_table(ft_prio->anchor.ft); 2606 ft_prio->anchor.ft = NULL; 2607 } 2608 } 2609 2610 static int 2611 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2612 { 2613 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2614 struct mlx5_flow_group *fg; 2615 void *flow_group_in; 2616 int err = 0; 2617 2618 if (ft_prio->anchor.fg_drop) 2619 return 0; 2620 2621 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2622 if (!flow_group_in) 2623 return -ENOMEM; 2624 2625 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1); 2626 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); 2627 2628 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2629 if (IS_ERR(fg)) { 2630 err = PTR_ERR(fg); 2631 goto out; 2632 } 2633 2634 ft_prio->anchor.fg_drop = fg; 2635 2636 out: 2637 kvfree(flow_group_in); 2638 2639 return err; 2640 } 2641 2642 static void 2643 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2644 { 2645 if (ft_prio->anchor.fg_drop) { 2646 mlx5_destroy_flow_group(ft_prio->anchor.fg_drop); 2647 ft_prio->anchor.fg_drop = NULL; 2648 } 2649 } 2650 2651 static int 2652 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2653 { 2654 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2655 struct mlx5_flow_group *fg; 2656 void *flow_group_in; 2657 int err = 0; 2658 2659 if (ft_prio->anchor.fg_goto_table) 2660 return 0; 2661 2662 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2663 if (!flow_group_in) 2664 return -ENOMEM; 2665 2666 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2667 if (IS_ERR(fg)) { 2668 err = PTR_ERR(fg); 2669 goto out; 2670 } 2671 ft_prio->anchor.fg_goto_table = fg; 2672 2673 out: 2674 kvfree(flow_group_in); 2675 2676 return err; 2677 } 2678 2679 static void 2680 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2681 { 2682 if (ft_prio->anchor.fg_goto_table) { 2683 mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table); 2684 ft_prio->anchor.fg_goto_table = NULL; 2685 } 2686 } 2687 2688 static int 2689 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2690 { 2691 struct mlx5_flow_act flow_act = {}; 2692 struct mlx5_flow_handle *handle; 2693 2694 if (ft_prio->anchor.rule_drop) 2695 return 0; 2696 2697 flow_act.fg = ft_prio->anchor.fg_drop; 2698 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2699 2700 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2701 NULL, 0); 2702 if (IS_ERR(handle)) 2703 return PTR_ERR(handle); 2704 2705 ft_prio->anchor.rule_drop = handle; 2706 2707 return 0; 2708 } 2709 2710 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2711 { 2712 if (ft_prio->anchor.rule_drop) { 2713 mlx5_del_flow_rules(ft_prio->anchor.rule_drop); 2714 ft_prio->anchor.rule_drop = NULL; 2715 } 2716 } 2717 2718 static int 2719 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2720 { 2721 struct mlx5_flow_destination dest = {}; 2722 struct mlx5_flow_act flow_act = {}; 2723 struct mlx5_flow_handle *handle; 2724 2725 if (ft_prio->anchor.rule_goto_table) 2726 return 0; 2727 2728 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2729 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 2730 flow_act.fg = ft_prio->anchor.fg_goto_table; 2731 2732 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2733 dest.ft = ft_prio->flow_table; 2734 2735 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2736 &dest, 1); 2737 if (IS_ERR(handle)) 2738 return PTR_ERR(handle); 2739 2740 ft_prio->anchor.rule_goto_table = handle; 2741 2742 return 0; 2743 } 2744 2745 static void 2746 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2747 { 2748 if (ft_prio->anchor.rule_goto_table) { 2749 mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table); 2750 ft_prio->anchor.rule_goto_table = NULL; 2751 } 2752 } 2753 2754 static int steering_anchor_create_res(struct mlx5_ib_dev *dev, 2755 struct mlx5_ib_flow_prio *ft_prio, 2756 enum mlx5_flow_namespace_type ns_type) 2757 { 2758 int err; 2759 2760 err = steering_anchor_create_ft(dev, ft_prio, ns_type); 2761 if (err) 2762 return err; 2763 2764 err = steering_anchor_create_fg_drop(ft_prio); 2765 if (err) 2766 goto destroy_ft; 2767 2768 err = steering_anchor_create_fg_goto_table(ft_prio); 2769 if (err) 2770 goto destroy_fg_drop; 2771 2772 err = steering_anchor_create_rule_drop(ft_prio); 2773 if (err) 2774 goto destroy_fg_goto_table; 2775 2776 err = steering_anchor_create_rule_goto_table(ft_prio); 2777 if (err) 2778 goto destroy_rule_drop; 2779 2780 return 0; 2781 2782 destroy_rule_drop: 2783 steering_anchor_destroy_rule_drop(ft_prio); 2784 destroy_fg_goto_table: 2785 steering_anchor_destroy_fg_goto_table(ft_prio); 2786 destroy_fg_drop: 2787 steering_anchor_destroy_fg_drop(ft_prio); 2788 destroy_ft: 2789 steering_anchor_destroy_ft(ft_prio); 2790 2791 return err; 2792 } 2793 2794 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio) 2795 { 2796 steering_anchor_destroy_rule_goto_table(ft_prio); 2797 steering_anchor_destroy_rule_drop(ft_prio); 2798 steering_anchor_destroy_fg_goto_table(ft_prio); 2799 steering_anchor_destroy_fg_drop(ft_prio); 2800 steering_anchor_destroy_ft(ft_prio); 2801 } 2802 2803 static int steering_anchor_cleanup(struct ib_uobject *uobject, 2804 enum rdma_remove_reason why, 2805 struct uverbs_attr_bundle *attrs) 2806 { 2807 struct mlx5_ib_steering_anchor *obj = uobject->object; 2808 2809 if (atomic_read(&obj->usecnt)) 2810 return -EBUSY; 2811 2812 mutex_lock(&obj->dev->flow_db->lock); 2813 if (!--obj->ft_prio->anchor.rule_goto_table_ref) 2814 steering_anchor_destroy_rule_goto_table(obj->ft_prio); 2815 2816 put_flow_table(obj->dev, obj->ft_prio, true); 2817 mutex_unlock(&obj->dev->flow_db->lock); 2818 2819 kfree(obj); 2820 return 0; 2821 } 2822 2823 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio, 2824 int count) 2825 { 2826 while (count--) 2827 mlx5_steering_anchor_destroy_res(&prio[count]); 2828 } 2829 2830 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev) 2831 { 2832 fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT); 2833 fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT); 2834 fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS); 2835 fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS); 2836 fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS); 2837 fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT); 2838 fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT); 2839 } 2840 2841 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2842 struct mlx5_ib_flow_matcher *obj) 2843 { 2844 enum mlx5_ib_uapi_flow_table_type ft_type = 2845 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2846 u32 flags; 2847 int err; 2848 2849 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2850 * users should switch to it. We leave this to not break userspace 2851 */ 2852 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2853 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2854 return -EINVAL; 2855 2856 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2857 err = uverbs_get_const(&ft_type, attrs, 2858 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2859 if (err) 2860 return err; 2861 2862 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2863 if (err) 2864 return err; 2865 2866 return 0; 2867 } 2868 2869 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2870 err = uverbs_get_flags32(&flags, attrs, 2871 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2872 IB_FLOW_ATTR_FLAGS_EGRESS); 2873 if (err) 2874 return err; 2875 2876 if (flags) 2877 return mlx5_ib_ft_type_to_namespace( 2878 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2879 &obj->ns_type); 2880 } 2881 2882 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2883 2884 return 0; 2885 } 2886 2887 static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps) 2888 { 2889 if (is_mdev_switchdev_mode(dev->mdev)) 2890 return UCAP_ENABLED(enabled_caps, 2891 RDMA_UCAP_MLX5_CTRL_OTHER_VHCA); 2892 2893 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL); 2894 } 2895 2896 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2897 struct uverbs_attr_bundle *attrs) 2898 { 2899 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2900 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2901 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2902 struct mlx5_ib_flow_matcher *obj; 2903 int err; 2904 2905 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2906 if (!obj) 2907 return -ENOMEM; 2908 2909 obj->mask_len = uverbs_attr_get_len( 2910 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2911 err = uverbs_copy_from(&obj->matcher_mask, 2912 attrs, 2913 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2914 if (err) 2915 goto end; 2916 2917 obj->flow_type = uverbs_attr_get_enum_id( 2918 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2919 2920 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2921 err = uverbs_copy_from(&obj->priority, 2922 attrs, 2923 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2924 if (err) 2925 goto end; 2926 } 2927 2928 err = uverbs_copy_from(&obj->match_criteria_enable, 2929 attrs, 2930 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2931 if (err) 2932 goto end; 2933 2934 err = mlx5_ib_matcher_ns(attrs, obj); 2935 if (err) 2936 goto end; 2937 2938 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2939 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { 2940 err = -EINVAL; 2941 goto end; 2942 } 2943 2944 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) { 2945 err = uverbs_copy_from(&obj->ib_port, attrs, 2946 MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT); 2947 if (err) 2948 goto end; 2949 if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) { 2950 err = -EINVAL; 2951 goto end; 2952 } 2953 if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX && 2954 obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) { 2955 err = -EINVAL; 2956 goto end; 2957 } 2958 if (!verify_context_caps(dev, uobj->context->enabled_caps)) { 2959 err = -EOPNOTSUPP; 2960 goto end; 2961 } 2962 } 2963 2964 uobj->object = obj; 2965 obj->mdev = dev->mdev; 2966 atomic_set(&obj->usecnt, 0); 2967 return 0; 2968 2969 end: 2970 kfree(obj); 2971 return err; 2972 } 2973 2974 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( 2975 struct uverbs_attr_bundle *attrs) 2976 { 2977 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2978 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE); 2979 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2980 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type; 2981 enum mlx5_flow_namespace_type ns_type; 2982 struct mlx5_ib_steering_anchor *obj; 2983 struct mlx5_ib_flow_prio *ft_prio; 2984 u16 priority; 2985 u32 ft_id; 2986 int err; 2987 2988 if (!rdma_dev_has_raw_cap(&dev->ib_dev)) 2989 return -EPERM; 2990 2991 err = uverbs_get_const(&ib_uapi_ft_type, attrs, 2992 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE); 2993 if (err) 2994 return err; 2995 2996 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type); 2997 if (err) 2998 return err; 2999 3000 err = uverbs_copy_from(&priority, attrs, 3001 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY); 3002 if (err) 3003 return err; 3004 3005 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 3006 if (!obj) 3007 return -ENOMEM; 3008 3009 mutex_lock(&dev->flow_db->lock); 3010 3011 ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0); 3012 if (IS_ERR(ft_prio)) { 3013 err = PTR_ERR(ft_prio); 3014 goto free_obj; 3015 } 3016 3017 ft_prio->refcount++; 3018 3019 if (!ft_prio->anchor.rule_goto_table_ref) { 3020 err = steering_anchor_create_res(dev, ft_prio, ns_type); 3021 if (err) 3022 goto put_flow_table; 3023 } 3024 3025 ft_prio->anchor.rule_goto_table_ref++; 3026 3027 ft_id = mlx5_flow_table_id(ft_prio->anchor.ft); 3028 3029 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3030 &ft_id, sizeof(ft_id)); 3031 if (err) 3032 goto destroy_res; 3033 3034 mutex_unlock(&dev->flow_db->lock); 3035 3036 uobj->object = obj; 3037 obj->dev = dev; 3038 obj->ft_prio = ft_prio; 3039 atomic_set(&obj->usecnt, 0); 3040 3041 return 0; 3042 3043 destroy_res: 3044 --ft_prio->anchor.rule_goto_table_ref; 3045 mlx5_steering_anchor_destroy_res(ft_prio); 3046 put_flow_table: 3047 put_flow_table(dev, ft_prio, true); 3048 free_obj: 3049 mutex_unlock(&dev->flow_db->lock); 3050 kfree(obj); 3051 3052 return err; 3053 } 3054 3055 static struct ib_flow_action * 3056 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 3057 enum mlx5_ib_uapi_flow_table_type ft_type, 3058 u8 num_actions, void *in) 3059 { 3060 enum mlx5_flow_namespace_type namespace; 3061 struct mlx5_ib_flow_action *maction; 3062 int ret; 3063 3064 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3065 if (ret) 3066 return ERR_PTR(-EINVAL); 3067 3068 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3069 if (!maction) 3070 return ERR_PTR(-ENOMEM); 3071 3072 maction->flow_action_raw.modify_hdr = 3073 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 3074 3075 if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 3076 ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 3077 kfree(maction); 3078 return ERR_PTR(ret); 3079 } 3080 maction->flow_action_raw.sub_type = 3081 MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 3082 maction->flow_action_raw.dev = dev; 3083 3084 return &maction->ib_action; 3085 } 3086 3087 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 3088 { 3089 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3090 max_modify_header_actions) || 3091 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 3092 max_modify_header_actions) || 3093 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 3094 max_modify_header_actions); 3095 } 3096 3097 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 3098 struct uverbs_attr_bundle *attrs) 3099 { 3100 struct ib_uobject *uobj = uverbs_attr_get_uobject( 3101 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 3102 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3103 enum mlx5_ib_uapi_flow_table_type ft_type; 3104 struct ib_flow_action *action; 3105 int num_actions; 3106 void *in; 3107 int ret; 3108 3109 if (!mlx5_ib_modify_header_supported(mdev)) 3110 return -EOPNOTSUPP; 3111 3112 in = uverbs_attr_get_alloced_ptr(attrs, 3113 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 3114 3115 num_actions = uverbs_attr_ptr_get_array_size( 3116 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3117 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 3118 if (num_actions < 0) 3119 return num_actions; 3120 3121 ret = uverbs_get_const(&ft_type, attrs, 3122 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 3123 if (ret) 3124 return ret; 3125 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 3126 if (IS_ERR(action)) 3127 return PTR_ERR(action); 3128 3129 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 3130 IB_FLOW_ACTION_UNSPECIFIED); 3131 3132 return 0; 3133 } 3134 3135 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 3136 u8 packet_reformat_type, 3137 u8 ft_type) 3138 { 3139 switch (packet_reformat_type) { 3140 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3141 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3142 return MLX5_CAP_FLOWTABLE(ibdev->mdev, 3143 encap_general_header); 3144 break; 3145 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3146 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3147 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 3148 reformat_l2_to_l3_tunnel); 3149 break; 3150 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3151 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3152 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 3153 reformat_l3_tunnel_to_l2); 3154 break; 3155 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 3156 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3157 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 3158 break; 3159 default: 3160 break; 3161 } 3162 3163 return false; 3164 } 3165 3166 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 3167 { 3168 switch (dv_prt) { 3169 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3170 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 3171 break; 3172 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3173 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 3174 break; 3175 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3176 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 3177 break; 3178 default: 3179 return -EINVAL; 3180 } 3181 3182 return 0; 3183 } 3184 3185 static int mlx5_ib_flow_action_create_packet_reformat_ctx( 3186 struct mlx5_ib_dev *dev, 3187 struct mlx5_ib_flow_action *maction, 3188 u8 ft_type, u8 dv_prt, 3189 void *in, size_t len) 3190 { 3191 struct mlx5_pkt_reformat_params reformat_params; 3192 enum mlx5_flow_namespace_type namespace; 3193 u8 prm_prt; 3194 int ret; 3195 3196 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3197 if (ret) 3198 return ret; 3199 3200 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 3201 if (ret) 3202 return ret; 3203 3204 memset(&reformat_params, 0, sizeof(reformat_params)); 3205 reformat_params.type = prm_prt; 3206 reformat_params.size = len; 3207 reformat_params.data = in; 3208 maction->flow_action_raw.pkt_reformat = 3209 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params, 3210 namespace); 3211 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 3212 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 3213 return ret; 3214 } 3215 3216 maction->flow_action_raw.sub_type = 3217 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 3218 maction->flow_action_raw.dev = dev; 3219 3220 return 0; 3221 } 3222 3223 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 3224 struct uverbs_attr_bundle *attrs) 3225 { 3226 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 3227 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 3228 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3229 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 3230 enum mlx5_ib_uapi_flow_table_type ft_type; 3231 struct mlx5_ib_flow_action *maction; 3232 int ret; 3233 3234 ret = uverbs_get_const(&ft_type, attrs, 3235 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 3236 if (ret) 3237 return ret; 3238 3239 ret = uverbs_get_const(&dv_prt, attrs, 3240 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 3241 if (ret) 3242 return ret; 3243 3244 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 3245 return -EOPNOTSUPP; 3246 3247 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3248 if (!maction) 3249 return -ENOMEM; 3250 3251 if (dv_prt == 3252 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 3253 maction->flow_action_raw.sub_type = 3254 MLX5_IB_FLOW_ACTION_DECAP; 3255 maction->flow_action_raw.dev = mdev; 3256 } else { 3257 void *in; 3258 int len; 3259 3260 in = uverbs_attr_get_alloced_ptr(attrs, 3261 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3262 if (IS_ERR(in)) { 3263 ret = PTR_ERR(in); 3264 goto free_maction; 3265 } 3266 3267 len = uverbs_attr_get_len(attrs, 3268 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3269 3270 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 3271 maction, ft_type, dv_prt, in, len); 3272 if (ret) 3273 goto free_maction; 3274 } 3275 3276 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 3277 IB_FLOW_ACTION_UNSPECIFIED); 3278 return 0; 3279 3280 free_maction: 3281 kfree(maction); 3282 return ret; 3283 } 3284 3285 DECLARE_UVERBS_NAMED_METHOD( 3286 MLX5_IB_METHOD_CREATE_FLOW, 3287 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3288 UVERBS_OBJECT_FLOW, 3289 UVERBS_ACCESS_NEW, 3290 UA_MANDATORY), 3291 UVERBS_ATTR_PTR_IN( 3292 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 3293 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3294 UA_MANDATORY, 3295 UA_ALLOC_AND_COPY), 3296 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 3297 MLX5_IB_OBJECT_FLOW_MATCHER, 3298 UVERBS_ACCESS_READ, 3299 UA_MANDATORY), 3300 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 3301 UVERBS_OBJECT_QP, 3302 UVERBS_ACCESS_READ), 3303 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 3304 MLX5_IB_OBJECT_DEVX_OBJ, 3305 UVERBS_ACCESS_READ), 3306 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 3307 UVERBS_OBJECT_FLOW_ACTION, 3308 UVERBS_ACCESS_READ, 1, 3309 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 3310 UA_OPTIONAL), 3311 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 3312 UVERBS_ATTR_TYPE(u32), 3313 UA_OPTIONAL), 3314 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 3315 MLX5_IB_OBJECT_DEVX_OBJ, 3316 UVERBS_ACCESS_READ, 1, 1, 3317 UA_OPTIONAL), 3318 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 3319 UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 3320 UA_OPTIONAL, 3321 UA_ALLOC_AND_COPY), 3322 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 3323 enum mlx5_ib_create_flow_flags, 3324 UA_OPTIONAL)); 3325 3326 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3327 MLX5_IB_METHOD_DESTROY_FLOW, 3328 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3329 UVERBS_OBJECT_FLOW, 3330 UVERBS_ACCESS_DESTROY, 3331 UA_MANDATORY)); 3332 3333 ADD_UVERBS_METHODS(mlx5_ib_fs, 3334 UVERBS_OBJECT_FLOW, 3335 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 3336 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 3337 3338 DECLARE_UVERBS_NAMED_METHOD( 3339 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 3340 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 3341 UVERBS_OBJECT_FLOW_ACTION, 3342 UVERBS_ACCESS_NEW, 3343 UA_MANDATORY), 3344 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3345 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 3346 set_add_copy_action_in_auto)), 3347 UA_MANDATORY, 3348 UA_ALLOC_AND_COPY), 3349 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 3350 enum mlx5_ib_uapi_flow_table_type, 3351 UA_MANDATORY)); 3352 3353 DECLARE_UVERBS_NAMED_METHOD( 3354 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 3355 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 3356 UVERBS_OBJECT_FLOW_ACTION, 3357 UVERBS_ACCESS_NEW, 3358 UA_MANDATORY), 3359 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 3360 UVERBS_ATTR_MIN_SIZE(1), 3361 UA_ALLOC_AND_COPY, 3362 UA_OPTIONAL), 3363 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 3364 enum mlx5_ib_uapi_flow_action_packet_reformat_type, 3365 UA_MANDATORY), 3366 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 3367 enum mlx5_ib_uapi_flow_table_type, 3368 UA_MANDATORY)); 3369 3370 ADD_UVERBS_METHODS( 3371 mlx5_ib_flow_actions, 3372 UVERBS_OBJECT_FLOW_ACTION, 3373 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 3374 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 3375 3376 DECLARE_UVERBS_NAMED_METHOD( 3377 MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 3378 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 3379 MLX5_IB_OBJECT_FLOW_MATCHER, 3380 UVERBS_ACCESS_NEW, 3381 UA_MANDATORY), 3382 UVERBS_ATTR_PTR_IN( 3383 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 3384 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3385 UA_MANDATORY), 3386 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 3387 mlx5_ib_flow_type, 3388 UA_MANDATORY), 3389 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 3390 UVERBS_ATTR_TYPE(u8), 3391 UA_MANDATORY), 3392 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 3393 enum ib_flow_flags, 3394 UA_OPTIONAL), 3395 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 3396 enum mlx5_ib_uapi_flow_table_type, 3397 UA_OPTIONAL), 3398 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT, 3399 UVERBS_ATTR_TYPE(u32), 3400 UA_OPTIONAL)); 3401 3402 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3403 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 3404 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 3405 MLX5_IB_OBJECT_FLOW_MATCHER, 3406 UVERBS_ACCESS_DESTROY, 3407 UA_MANDATORY)); 3408 3409 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 3410 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 3411 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 3412 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 3413 3414 DECLARE_UVERBS_NAMED_METHOD( 3415 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE, 3416 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE, 3417 MLX5_IB_OBJECT_STEERING_ANCHOR, 3418 UVERBS_ACCESS_NEW, 3419 UA_MANDATORY), 3420 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE, 3421 enum mlx5_ib_uapi_flow_table_type, 3422 UA_MANDATORY), 3423 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY, 3424 UVERBS_ATTR_TYPE(u16), 3425 UA_MANDATORY), 3426 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3427 UVERBS_ATTR_TYPE(u32), 3428 UA_MANDATORY)); 3429 3430 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3431 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY, 3432 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE, 3433 MLX5_IB_OBJECT_STEERING_ANCHOR, 3434 UVERBS_ACCESS_DESTROY, 3435 UA_MANDATORY)); 3436 3437 DECLARE_UVERBS_NAMED_OBJECT( 3438 MLX5_IB_OBJECT_STEERING_ANCHOR, 3439 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup), 3440 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE), 3441 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY)); 3442 3443 const struct uapi_definition mlx5_ib_flow_defs[] = { 3444 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3445 MLX5_IB_OBJECT_FLOW_MATCHER), 3446 UAPI_DEF_CHAIN_OBJ_TREE( 3447 UVERBS_OBJECT_FLOW, 3448 &mlx5_ib_fs), 3449 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 3450 &mlx5_ib_flow_actions), 3451 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3452 MLX5_IB_OBJECT_STEERING_ANCHOR, 3453 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)), 3454 {}, 3455 }; 3456 3457 static const struct ib_device_ops flow_ops = { 3458 .create_flow = mlx5_ib_create_flow, 3459 .destroy_flow = mlx5_ib_destroy_flow, 3460 .destroy_flow_action = mlx5_ib_destroy_flow_action, 3461 }; 3462 3463 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 3464 { 3465 int i, j; 3466 3467 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 3468 3469 if (!dev->flow_db) 3470 return -ENOMEM; 3471 3472 for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) { 3473 dev->flow_db->rdma_transport_rx[i] = 3474 kcalloc(dev->num_ports, 3475 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL); 3476 if (!dev->flow_db->rdma_transport_rx[i]) 3477 goto free_rdma_transport_rx; 3478 } 3479 3480 for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) { 3481 dev->flow_db->rdma_transport_tx[j] = 3482 kcalloc(dev->num_ports, 3483 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL); 3484 if (!dev->flow_db->rdma_transport_tx[j]) 3485 goto free_rdma_transport_tx; 3486 } 3487 3488 mutex_init(&dev->flow_db->lock); 3489 3490 ib_set_device_ops(&dev->ib_dev, &flow_ops); 3491 return 0; 3492 3493 free_rdma_transport_tx: 3494 while (j--) 3495 kfree(dev->flow_db->rdma_transport_tx[j]); 3496 free_rdma_transport_rx: 3497 while (i--) 3498 kfree(dev->flow_db->rdma_transport_rx[i]); 3499 kfree(dev->flow_db); 3500 return -ENOMEM; 3501 } 3502