1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/uverbs_std_types.h> 11 #include <rdma/mlx5_user_ioctl_cmds.h> 12 #include <rdma/mlx5_user_ioctl_verbs.h> 13 #include <rdma/ib_hdrs.h> 14 #include <rdma/ib_umem.h> 15 #include <rdma/ib_ucaps.h> 16 #include <linux/mlx5/driver.h> 17 #include <linux/mlx5/fs.h> 18 #include <linux/mlx5/fs_helpers.h> 19 #include <linux/mlx5/eswitch.h> 20 #include <net/inet_ecn.h> 21 #include "mlx5_ib.h" 22 #include "counters.h" 23 #include "devx.h" 24 #include "fs.h" 25 26 #define UVERBS_MODULE_NAME mlx5_ib 27 #include <rdma/uverbs_named_ioctl.h> 28 29 enum { 30 MATCH_CRITERIA_ENABLE_OUTER_BIT, 31 MATCH_CRITERIA_ENABLE_MISC_BIT, 32 MATCH_CRITERIA_ENABLE_INNER_BIT, 33 MATCH_CRITERIA_ENABLE_MISC2_BIT 34 }; 35 36 37 struct mlx5_per_qp_opfc { 38 struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX]; 39 }; 40 41 #define HEADER_IS_ZERO(match_criteria, headers) \ 42 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 43 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 44 45 static u8 get_match_criteria_enable(u32 *match_criteria) 46 { 47 u8 match_criteria_enable; 48 49 match_criteria_enable = 50 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 51 MATCH_CRITERIA_ENABLE_OUTER_BIT; 52 match_criteria_enable |= 53 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 54 MATCH_CRITERIA_ENABLE_MISC_BIT; 55 match_criteria_enable |= 56 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 57 MATCH_CRITERIA_ENABLE_INNER_BIT; 58 match_criteria_enable |= 59 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 60 MATCH_CRITERIA_ENABLE_MISC2_BIT; 61 62 return match_criteria_enable; 63 } 64 65 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 66 { 67 u8 entry_mask; 68 u8 entry_val; 69 int err = 0; 70 71 if (!mask) 72 goto out; 73 74 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 75 ip_protocol); 76 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 77 ip_protocol); 78 if (!entry_mask) { 79 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 80 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 81 goto out; 82 } 83 /* Don't override existing ip protocol */ 84 if (mask != entry_mask || val != entry_val) 85 err = -EINVAL; 86 out: 87 return err; 88 } 89 90 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 91 bool inner) 92 { 93 if (inner) { 94 MLX5_SET(fte_match_set_misc, 95 misc_c, inner_ipv6_flow_label, mask); 96 MLX5_SET(fte_match_set_misc, 97 misc_v, inner_ipv6_flow_label, val); 98 } else { 99 MLX5_SET(fte_match_set_misc, 100 misc_c, outer_ipv6_flow_label, mask); 101 MLX5_SET(fte_match_set_misc, 102 misc_v, outer_ipv6_flow_label, val); 103 } 104 } 105 106 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 107 { 108 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 109 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 110 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 111 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 112 } 113 114 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 115 { 116 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 117 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 118 return -EOPNOTSUPP; 119 120 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 121 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 122 return -EOPNOTSUPP; 123 124 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 125 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 126 return -EOPNOTSUPP; 127 128 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 129 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 130 return -EOPNOTSUPP; 131 132 return 0; 133 } 134 135 #define LAST_ETH_FIELD vlan_tag 136 #define LAST_IPV4_FIELD tos 137 #define LAST_IPV6_FIELD traffic_class 138 #define LAST_TCP_UDP_FIELD src_port 139 #define LAST_TUNNEL_FIELD tunnel_id 140 #define LAST_FLOW_TAG_FIELD tag_id 141 #define LAST_DROP_FIELD size 142 #define LAST_COUNTERS_FIELD counters 143 144 /* Field is the last supported field */ 145 #define FIELDS_NOT_SUPPORTED(filter, field) \ 146 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ 147 sizeof(filter) - offsetofend(typeof(filter), field)) 148 149 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 150 bool is_egress, 151 struct mlx5_flow_act *action) 152 { 153 154 switch (maction->ib_action.type) { 155 case IB_FLOW_ACTION_UNSPECIFIED: 156 if (maction->flow_action_raw.sub_type == 157 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 158 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 159 return -EINVAL; 160 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 161 action->modify_hdr = 162 maction->flow_action_raw.modify_hdr; 163 return 0; 164 } 165 if (maction->flow_action_raw.sub_type == 166 MLX5_IB_FLOW_ACTION_DECAP) { 167 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 168 return -EINVAL; 169 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 170 return 0; 171 } 172 if (maction->flow_action_raw.sub_type == 173 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 174 if (action->action & 175 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 176 return -EINVAL; 177 action->action |= 178 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 179 action->pkt_reformat = 180 maction->flow_action_raw.pkt_reformat; 181 return 0; 182 } 183 fallthrough; 184 default: 185 return -EOPNOTSUPP; 186 } 187 } 188 189 static int parse_flow_attr(struct mlx5_core_dev *mdev, 190 struct mlx5_flow_spec *spec, 191 const union ib_flow_spec *ib_spec, 192 const struct ib_flow_attr *flow_attr, 193 struct mlx5_flow_act *action, u32 prev_type) 194 { 195 struct mlx5_flow_context *flow_context = &spec->flow_context; 196 u32 *match_c = spec->match_criteria; 197 u32 *match_v = spec->match_value; 198 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 199 misc_parameters); 200 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 201 misc_parameters); 202 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 203 misc_parameters_2); 204 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 205 misc_parameters_2); 206 void *headers_c; 207 void *headers_v; 208 int match_ipv; 209 int ret; 210 211 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 212 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 213 inner_headers); 214 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 215 inner_headers); 216 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 217 ft_field_support.inner_ip_version); 218 } else { 219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 220 outer_headers); 221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 222 outer_headers); 223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 224 ft_field_support.outer_ip_version); 225 } 226 227 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 228 case IB_FLOW_SPEC_ETH: 229 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 230 return -EOPNOTSUPP; 231 232 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 233 dmac_47_16), 234 ib_spec->eth.mask.dst_mac); 235 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 236 dmac_47_16), 237 ib_spec->eth.val.dst_mac); 238 239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 240 smac_47_16), 241 ib_spec->eth.mask.src_mac); 242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 243 smac_47_16), 244 ib_spec->eth.val.src_mac); 245 246 if (ib_spec->eth.mask.vlan_tag) { 247 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 248 cvlan_tag, 1); 249 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 250 cvlan_tag, 1); 251 252 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 253 first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 254 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 255 first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 256 257 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 258 first_cfi, 259 ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 260 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 261 first_cfi, 262 ntohs(ib_spec->eth.val.vlan_tag) >> 12); 263 264 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 265 first_prio, 266 ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 267 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 268 first_prio, 269 ntohs(ib_spec->eth.val.vlan_tag) >> 13); 270 } 271 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 272 ethertype, ntohs(ib_spec->eth.mask.ether_type)); 273 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 274 ethertype, ntohs(ib_spec->eth.val.ether_type)); 275 break; 276 case IB_FLOW_SPEC_IPV4: 277 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 278 return -EOPNOTSUPP; 279 280 if (match_ipv) { 281 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 282 ip_version, 0xf); 283 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 284 ip_version, MLX5_FS_IPV4_VERSION); 285 } else { 286 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 287 ethertype, 0xffff); 288 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 289 ethertype, ETH_P_IP); 290 } 291 292 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 293 src_ipv4_src_ipv6.ipv4_layout.ipv4), 294 &ib_spec->ipv4.mask.src_ip, 295 sizeof(ib_spec->ipv4.mask.src_ip)); 296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 297 src_ipv4_src_ipv6.ipv4_layout.ipv4), 298 &ib_spec->ipv4.val.src_ip, 299 sizeof(ib_spec->ipv4.val.src_ip)); 300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 301 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 302 &ib_spec->ipv4.mask.dst_ip, 303 sizeof(ib_spec->ipv4.mask.dst_ip)); 304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 306 &ib_spec->ipv4.val.dst_ip, 307 sizeof(ib_spec->ipv4.val.dst_ip)); 308 309 set_tos(headers_c, headers_v, 310 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 311 312 if (set_proto(headers_c, headers_v, 313 ib_spec->ipv4.mask.proto, 314 ib_spec->ipv4.val.proto)) 315 return -EINVAL; 316 break; 317 case IB_FLOW_SPEC_IPV6: 318 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 319 return -EOPNOTSUPP; 320 321 if (match_ipv) { 322 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 323 ip_version, 0xf); 324 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 325 ip_version, MLX5_FS_IPV6_VERSION); 326 } else { 327 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 328 ethertype, 0xffff); 329 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 330 ethertype, ETH_P_IPV6); 331 } 332 333 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 334 src_ipv4_src_ipv6.ipv6_layout.ipv6), 335 &ib_spec->ipv6.mask.src_ip, 336 sizeof(ib_spec->ipv6.mask.src_ip)); 337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 338 src_ipv4_src_ipv6.ipv6_layout.ipv6), 339 &ib_spec->ipv6.val.src_ip, 340 sizeof(ib_spec->ipv6.val.src_ip)); 341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 342 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 343 &ib_spec->ipv6.mask.dst_ip, 344 sizeof(ib_spec->ipv6.mask.dst_ip)); 345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 347 &ib_spec->ipv6.val.dst_ip, 348 sizeof(ib_spec->ipv6.val.dst_ip)); 349 350 set_tos(headers_c, headers_v, 351 ib_spec->ipv6.mask.traffic_class, 352 ib_spec->ipv6.val.traffic_class); 353 354 if (set_proto(headers_c, headers_v, 355 ib_spec->ipv6.mask.next_hdr, 356 ib_spec->ipv6.val.next_hdr)) 357 return -EINVAL; 358 359 set_flow_label(misc_params_c, misc_params_v, 360 ntohl(ib_spec->ipv6.mask.flow_label), 361 ntohl(ib_spec->ipv6.val.flow_label), 362 ib_spec->type & IB_FLOW_SPEC_INNER); 363 break; 364 case IB_FLOW_SPEC_ESP: 365 return -EOPNOTSUPP; 366 case IB_FLOW_SPEC_TCP: 367 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 368 LAST_TCP_UDP_FIELD)) 369 return -EOPNOTSUPP; 370 371 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 372 return -EINVAL; 373 374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 375 ntohs(ib_spec->tcp_udp.mask.src_port)); 376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 377 ntohs(ib_spec->tcp_udp.val.src_port)); 378 379 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 380 ntohs(ib_spec->tcp_udp.mask.dst_port)); 381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 382 ntohs(ib_spec->tcp_udp.val.dst_port)); 383 break; 384 case IB_FLOW_SPEC_UDP: 385 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 386 LAST_TCP_UDP_FIELD)) 387 return -EOPNOTSUPP; 388 389 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 390 return -EINVAL; 391 392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 393 ntohs(ib_spec->tcp_udp.mask.src_port)); 394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 395 ntohs(ib_spec->tcp_udp.val.src_port)); 396 397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 398 ntohs(ib_spec->tcp_udp.mask.dst_port)); 399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 400 ntohs(ib_spec->tcp_udp.val.dst_port)); 401 break; 402 case IB_FLOW_SPEC_GRE: 403 if (ib_spec->gre.mask.c_ks_res0_ver) 404 return -EOPNOTSUPP; 405 406 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 407 return -EINVAL; 408 409 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 410 0xff); 411 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 412 IPPROTO_GRE); 413 414 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 415 ntohs(ib_spec->gre.mask.protocol)); 416 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 417 ntohs(ib_spec->gre.val.protocol)); 418 419 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 420 gre_key.nvgre.hi), 421 &ib_spec->gre.mask.key, 422 sizeof(ib_spec->gre.mask.key)); 423 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 424 gre_key.nvgre.hi), 425 &ib_spec->gre.val.key, 426 sizeof(ib_spec->gre.val.key)); 427 break; 428 case IB_FLOW_SPEC_MPLS: 429 switch (prev_type) { 430 case IB_FLOW_SPEC_UDP: 431 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 432 ft_field_support.outer_first_mpls_over_udp), 433 &ib_spec->mpls.mask.tag)) 434 return -EOPNOTSUPP; 435 436 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 437 outer_first_mpls_over_udp), 438 &ib_spec->mpls.val.tag, 439 sizeof(ib_spec->mpls.val.tag)); 440 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 441 outer_first_mpls_over_udp), 442 &ib_spec->mpls.mask.tag, 443 sizeof(ib_spec->mpls.mask.tag)); 444 break; 445 case IB_FLOW_SPEC_GRE: 446 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 447 ft_field_support.outer_first_mpls_over_gre), 448 &ib_spec->mpls.mask.tag)) 449 return -EOPNOTSUPP; 450 451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 452 outer_first_mpls_over_gre), 453 &ib_spec->mpls.val.tag, 454 sizeof(ib_spec->mpls.val.tag)); 455 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 456 outer_first_mpls_over_gre), 457 &ib_spec->mpls.mask.tag, 458 sizeof(ib_spec->mpls.mask.tag)); 459 break; 460 default: 461 if (ib_spec->type & IB_FLOW_SPEC_INNER) { 462 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 463 ft_field_support.inner_first_mpls), 464 &ib_spec->mpls.mask.tag)) 465 return -EOPNOTSUPP; 466 467 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 468 inner_first_mpls), 469 &ib_spec->mpls.val.tag, 470 sizeof(ib_spec->mpls.val.tag)); 471 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 472 inner_first_mpls), 473 &ib_spec->mpls.mask.tag, 474 sizeof(ib_spec->mpls.mask.tag)); 475 } else { 476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 477 ft_field_support.outer_first_mpls), 478 &ib_spec->mpls.mask.tag)) 479 return -EOPNOTSUPP; 480 481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 482 outer_first_mpls), 483 &ib_spec->mpls.val.tag, 484 sizeof(ib_spec->mpls.val.tag)); 485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 486 outer_first_mpls), 487 &ib_spec->mpls.mask.tag, 488 sizeof(ib_spec->mpls.mask.tag)); 489 } 490 } 491 break; 492 case IB_FLOW_SPEC_VXLAN_TUNNEL: 493 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 494 LAST_TUNNEL_FIELD)) 495 return -EOPNOTSUPP; 496 497 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 498 ntohl(ib_spec->tunnel.mask.tunnel_id)); 499 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 500 ntohl(ib_spec->tunnel.val.tunnel_id)); 501 break; 502 case IB_FLOW_SPEC_ACTION_TAG: 503 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 504 LAST_FLOW_TAG_FIELD)) 505 return -EOPNOTSUPP; 506 if (ib_spec->flow_tag.tag_id >= BIT(24)) 507 return -EINVAL; 508 509 flow_context->flow_tag = ib_spec->flow_tag.tag_id; 510 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 511 break; 512 case IB_FLOW_SPEC_ACTION_DROP: 513 if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 514 LAST_DROP_FIELD)) 515 return -EOPNOTSUPP; 516 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 517 break; 518 case IB_FLOW_SPEC_ACTION_HANDLE: 519 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 520 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 521 if (ret) 522 return ret; 523 break; 524 case IB_FLOW_SPEC_ACTION_COUNT: 525 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 526 LAST_COUNTERS_FIELD)) 527 return -EOPNOTSUPP; 528 529 /* for now support only one counters spec per flow */ 530 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 531 return -EINVAL; 532 533 action->counters = ib_spec->flow_count.counters; 534 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 535 break; 536 default: 537 return -EINVAL; 538 } 539 540 return 0; 541 } 542 543 /* If a flow could catch both multicast and unicast packets, 544 * it won't fall into the multicast flow steering table and this rule 545 * could steal other multicast packets. 546 */ 547 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 548 { 549 union ib_flow_spec *flow_spec; 550 551 if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 552 ib_attr->num_of_specs < 1) 553 return false; 554 555 flow_spec = (union ib_flow_spec *)(ib_attr + 1); 556 if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 557 struct ib_flow_spec_ipv4 *ipv4_spec; 558 559 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 560 if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 561 return true; 562 563 return false; 564 } 565 566 if (flow_spec->type == IB_FLOW_SPEC_ETH) { 567 struct ib_flow_spec_eth *eth_spec; 568 569 eth_spec = (struct ib_flow_spec_eth *)flow_spec; 570 return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 571 is_multicast_ether_addr(eth_spec->val.dst_mac); 572 } 573 574 return false; 575 } 576 577 static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 578 const struct ib_flow_attr *flow_attr, 579 bool check_inner) 580 { 581 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 582 int match_ipv = check_inner ? 583 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 584 ft_field_support.inner_ip_version) : 585 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 586 ft_field_support.outer_ip_version); 587 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 588 bool ipv4_spec_valid, ipv6_spec_valid; 589 unsigned int ip_spec_type = 0; 590 bool has_ethertype = false; 591 unsigned int spec_index; 592 bool mask_valid = true; 593 u16 eth_type = 0; 594 bool type_valid; 595 596 /* Validate that ethertype is correct */ 597 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 598 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 599 ib_spec->eth.mask.ether_type) { 600 mask_valid = (ib_spec->eth.mask.ether_type == 601 htons(0xffff)); 602 has_ethertype = true; 603 eth_type = ntohs(ib_spec->eth.val.ether_type); 604 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 605 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 606 ip_spec_type = ib_spec->type; 607 } 608 ib_spec = (void *)ib_spec + ib_spec->size; 609 } 610 611 type_valid = (!has_ethertype) || (!ip_spec_type); 612 if (!type_valid && mask_valid) { 613 ipv4_spec_valid = (eth_type == ETH_P_IP) && 614 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 615 ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 616 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 617 618 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 619 (((eth_type == ETH_P_MPLS_UC) || 620 (eth_type == ETH_P_MPLS_MC)) && match_ipv); 621 } 622 623 return type_valid; 624 } 625 626 static bool is_valid_attr(struct mlx5_core_dev *mdev, 627 const struct ib_flow_attr *flow_attr) 628 { 629 return is_valid_ethertype(mdev, flow_attr, false) && 630 is_valid_ethertype(mdev, flow_attr, true); 631 } 632 633 static void put_flow_table(struct mlx5_ib_dev *dev, 634 struct mlx5_ib_flow_prio *prio, bool ft_added) 635 { 636 prio->refcount -= !!ft_added; 637 if (!prio->refcount) { 638 mlx5_destroy_flow_table(prio->flow_table); 639 prio->flow_table = NULL; 640 } 641 } 642 643 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 644 { 645 struct mlx5_ib_flow_handler *handler = container_of(flow_id, 646 struct mlx5_ib_flow_handler, 647 ibflow); 648 struct mlx5_ib_flow_handler *iter, *tmp; 649 struct mlx5_ib_dev *dev = handler->dev; 650 651 mutex_lock(&dev->flow_db->lock); 652 653 list_for_each_entry_safe(iter, tmp, &handler->list, list) { 654 mlx5_del_flow_rules(iter->rule); 655 put_flow_table(dev, iter->prio, true); 656 list_del(&iter->list); 657 kfree(iter); 658 } 659 660 mlx5_del_flow_rules(handler->rule); 661 put_flow_table(dev, handler->prio, true); 662 mlx5_ib_counters_clear_description(handler->ibcounters); 663 mutex_unlock(&dev->flow_db->lock); 664 if (handler->flow_matcher) 665 atomic_dec(&handler->flow_matcher->usecnt); 666 kfree(handler); 667 668 return 0; 669 } 670 671 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 672 { 673 priority *= 2; 674 if (!dont_trap) 675 priority++; 676 return priority; 677 } 678 679 enum flow_table_type { 680 MLX5_IB_FT_RX, 681 MLX5_IB_FT_TX 682 }; 683 684 #define MLX5_FS_MAX_TYPES 6 685 #define MLX5_FS_MAX_ENTRIES BIT(16) 686 687 static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device) 688 { 689 struct mlx5_ib_dev *dev = to_mdev(device); 690 691 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed); 692 } 693 694 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, 695 struct mlx5_ib_flow_prio *prio, 696 struct mlx5_flow_table_attr *ft_attr) 697 { 698 struct mlx5_flow_table *ft; 699 700 ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr); 701 if (IS_ERR(ft)) 702 return ERR_CAST(ft); 703 704 prio->flow_table = ft; 705 prio->refcount = 0; 706 return prio; 707 } 708 709 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 710 struct ib_flow_attr *flow_attr, 711 enum flow_table_type ft_type) 712 { 713 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 714 struct mlx5_flow_table_attr ft_attr = {}; 715 struct mlx5_flow_namespace *ns = NULL; 716 enum mlx5_flow_namespace_type fn_type; 717 struct mlx5_ib_flow_prio *prio; 718 struct mlx5_flow_table *ft; 719 int max_table_size; 720 int num_entries; 721 int num_groups; 722 bool esw_encap; 723 u32 flags = 0; 724 int priority; 725 726 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 727 log_max_ft_size)); 728 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 729 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 730 switch (flow_attr->type) { 731 case IB_FLOW_ATTR_NORMAL: 732 if (flow_is_multicast_only(flow_attr) && !dont_trap) 733 priority = MLX5_IB_FLOW_MCAST_PRIO; 734 else 735 priority = ib_prio_to_core_prio(flow_attr->priority, 736 dont_trap); 737 if (ft_type == MLX5_IB_FT_RX) { 738 fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 739 prio = &dev->flow_db->prios[priority]; 740 if (!dev->is_rep && !esw_encap && 741 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 742 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 743 if (!dev->is_rep && !esw_encap && 744 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 745 reformat_l3_tunnel_to_l2)) 746 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 747 } else { 748 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( 749 dev->mdev, log_max_ft_size)); 750 fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 751 prio = &dev->flow_db->egress_prios[priority]; 752 if (!dev->is_rep && !esw_encap && 753 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 754 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 755 } 756 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 757 num_entries = MLX5_FS_MAX_ENTRIES; 758 num_groups = MLX5_FS_MAX_TYPES; 759 break; 760 case IB_FLOW_ATTR_ALL_DEFAULT: 761 case IB_FLOW_ATTR_MC_DEFAULT: 762 ns = mlx5_get_flow_namespace(dev->mdev, 763 MLX5_FLOW_NAMESPACE_LEFTOVERS); 764 build_leftovers_ft_param(&priority, &num_entries, &num_groups); 765 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 766 break; 767 case IB_FLOW_ATTR_SNIFFER: 768 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 769 allow_sniffer_and_nic_rx_shared_tir)) 770 return ERR_PTR(-EOPNOTSUPP); 771 772 ns = mlx5_get_flow_namespace( 773 dev->mdev, ft_type == MLX5_IB_FT_RX ? 774 MLX5_FLOW_NAMESPACE_SNIFFER_RX : 775 MLX5_FLOW_NAMESPACE_SNIFFER_TX); 776 777 prio = &dev->flow_db->sniffer[ft_type]; 778 priority = 0; 779 num_entries = 1; 780 num_groups = 1; 781 break; 782 default: 783 break; 784 } 785 786 if (!ns) 787 return ERR_PTR(-EOPNOTSUPP); 788 789 max_table_size = min_t(int, num_entries, max_table_size); 790 791 ft = prio->flow_table; 792 if (ft) 793 return prio; 794 795 ft_attr.prio = priority; 796 ft_attr.max_fte = max_table_size; 797 ft_attr.flags = flags; 798 ft_attr.autogroup.max_num_groups = num_groups; 799 return _get_prio(ns, prio, &ft_attr); 800 } 801 802 enum { 803 RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO, 804 RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO, 805 RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 806 RDMA_RX_ECN_OPCOUNTER_PRIO, 807 RDMA_RX_CNP_OPCOUNTER_PRIO, 808 RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO, 809 }; 810 811 enum { 812 RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO, 813 RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO, 814 RDMA_TX_CNP_OPCOUNTER_PRIO, 815 RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO, 816 }; 817 818 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num, 819 struct mlx5_flow_spec *spec) 820 { 821 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 822 ft_field_support.source_vhca_port) || 823 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 824 ft_field_support.source_vhca_port)) 825 return -EOPNOTSUPP; 826 827 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria, 828 misc_parameters.source_vhca_port); 829 MLX5_SET(fte_match_param, &spec->match_value, 830 misc_parameters.source_vhca_port, port_num); 831 832 return 0; 833 } 834 835 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num, 836 struct mlx5_flow_spec *spec, int ipv) 837 { 838 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 839 ft_field_support.outer_ip_version)) 840 return -EOPNOTSUPP; 841 842 if (mlx5_core_mp_enabled(dev->mdev) && 843 set_vhca_port_spec(dev, port_num, spec)) 844 return -EOPNOTSUPP; 845 846 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 847 outer_headers.ip_ecn); 848 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn, 849 INET_ECN_CE); 850 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 851 outer_headers.ip_version); 852 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 853 ipv); 854 855 spec->match_criteria_enable = 856 get_match_criteria_enable(spec->match_criteria); 857 858 return 0; 859 } 860 861 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num, 862 struct mlx5_flow_spec *spec) 863 { 864 if (mlx5_core_mp_enabled(dev->mdev) && 865 set_vhca_port_spec(dev, port_num, spec)) 866 return -EOPNOTSUPP; 867 868 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 869 misc_parameters.bth_opcode); 870 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode, 871 IB_BTH_OPCODE_CNP); 872 873 spec->match_criteria_enable = 874 get_match_criteria_enable(spec->match_criteria); 875 876 return 0; 877 } 878 879 /* Returns the prio we should use for the given optional counter type, 880 * whereas for bytes type we use the packet type, since they share the same 881 * resources. 882 */ 883 static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev, 884 u32 type) 885 { 886 u32 prio_type; 887 888 switch (type) { 889 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 890 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS; 891 break; 892 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 893 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS; 894 break; 895 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 896 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 897 break; 898 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 899 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 900 break; 901 default: 902 prio_type = type; 903 } 904 905 return &dev->flow_db->opfcs[prio_type]; 906 } 907 908 static void put_per_qp_prio(struct mlx5_ib_dev *dev, 909 enum mlx5_ib_optional_counter_type type) 910 { 911 enum mlx5_ib_optional_counter_type per_qp_type; 912 struct mlx5_ib_flow_prio *prio; 913 914 switch (type) { 915 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 916 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 917 break; 918 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 919 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 920 break; 921 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 922 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 923 break; 924 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 925 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 926 break; 927 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 928 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 929 break; 930 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 931 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 932 break; 933 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 934 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 935 break; 936 default: 937 return; 938 } 939 940 prio = get_opfc_prio(dev, per_qp_type); 941 put_flow_table(dev, prio, true); 942 } 943 944 static int get_per_qp_prio(struct mlx5_ib_dev *dev, 945 enum mlx5_ib_optional_counter_type type) 946 { 947 enum mlx5_ib_optional_counter_type per_qp_type; 948 struct mlx5_flow_table_attr ft_attr = {}; 949 enum mlx5_flow_namespace_type fn_type; 950 struct mlx5_flow_namespace *ns; 951 struct mlx5_ib_flow_prio *prio; 952 int priority; 953 954 switch (type) { 955 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 956 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 957 priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO; 958 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 959 break; 960 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 961 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 962 priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO; 963 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP; 964 break; 965 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 966 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 967 priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO; 968 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP; 969 break; 970 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 971 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 972 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 973 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 974 break; 975 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 976 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 977 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 978 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 979 break; 980 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 981 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 982 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 983 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 984 break; 985 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 986 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 987 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO; 988 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 989 break; 990 default: 991 return -EINVAL; 992 } 993 994 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 995 if (!ns) 996 return -EOPNOTSUPP; 997 998 prio = get_opfc_prio(dev, per_qp_type); 999 if (prio->flow_table) 1000 return 0; 1001 1002 ft_attr.prio = priority; 1003 ft_attr.max_fte = MLX5_FS_MAX_POOL_SIZE; 1004 ft_attr.autogroup.max_num_groups = 1; 1005 prio = _get_prio(ns, prio, &ft_attr); 1006 if (IS_ERR(prio)) 1007 return PTR_ERR(prio); 1008 1009 prio->refcount = 1; 1010 1011 return 0; 1012 } 1013 1014 static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa, 1015 u32 qp_num, bool *new) 1016 { 1017 struct mlx5_per_qp_opfc *per_qp_opfc; 1018 1019 *new = false; 1020 1021 per_qp_opfc = xa_load(qpn_opfc_xa, qp_num); 1022 if (per_qp_opfc) 1023 return per_qp_opfc; 1024 per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL); 1025 1026 if (!per_qp_opfc) 1027 return NULL; 1028 1029 *new = true; 1030 return per_qp_opfc; 1031 } 1032 1033 static int add_op_fc_rules(struct mlx5_ib_dev *dev, 1034 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], 1035 struct xarray *qpn_opfc_xa, 1036 struct mlx5_per_qp_opfc *per_qp_opfc, 1037 struct mlx5_ib_flow_prio *prio, 1038 enum mlx5_ib_optional_counter_type type, 1039 u32 qp_num, u32 port_num) 1040 { 1041 struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc; 1042 struct mlx5_flow_act flow_act = {}; 1043 struct mlx5_flow_destination dst; 1044 struct mlx5_flow_spec *spec; 1045 int i, err, spec_num; 1046 bool is_tx; 1047 1048 if (opfc->fc) 1049 return -EEXIST; 1050 1051 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type, 1052 &in_use_opfc)) { 1053 opfc->fc = in_use_opfc->fc; 1054 opfc->rule[0] = in_use_opfc->rule[0]; 1055 return 0; 1056 } 1057 1058 opfc->fc = fc_arr[type]; 1059 1060 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1061 if (!spec) { 1062 err = -ENOMEM; 1063 goto null_fc; 1064 } 1065 1066 switch (type) { 1067 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP: 1068 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1069 MLX5_FS_IPV4_VERSION) || 1070 set_ecn_ce_spec(dev, port_num, &spec[1], 1071 MLX5_FS_IPV6_VERSION)) { 1072 err = -EOPNOTSUPP; 1073 goto free_spec; 1074 } 1075 spec_num = 2; 1076 is_tx = false; 1077 1078 MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria, 1079 misc_parameters.bth_dst_qp); 1080 MLX5_SET(fte_match_param, spec[1].match_value, 1081 misc_parameters.bth_dst_qp, qp_num); 1082 spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1083 break; 1084 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP: 1085 if (!MLX5_CAP_FLOWTABLE( 1086 dev->mdev, 1087 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1088 set_cnp_spec(dev, port_num, &spec[0])) { 1089 err = -EOPNOTSUPP; 1090 goto free_spec; 1091 } 1092 spec_num = 1; 1093 is_tx = false; 1094 break; 1095 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP: 1096 if (!MLX5_CAP_FLOWTABLE( 1097 dev->mdev, 1098 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1099 set_cnp_spec(dev, port_num, &spec[0])) { 1100 err = -EOPNOTSUPP; 1101 goto free_spec; 1102 } 1103 spec_num = 1; 1104 is_tx = true; 1105 break; 1106 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1107 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1108 spec_num = 1; 1109 is_tx = true; 1110 break; 1111 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1112 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1113 spec_num = 1; 1114 is_tx = false; 1115 break; 1116 default: 1117 err = -EINVAL; 1118 goto free_spec; 1119 } 1120 1121 if (is_tx) { 1122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1123 misc_parameters.source_sqn); 1124 MLX5_SET(fte_match_param, spec->match_value, 1125 misc_parameters.source_sqn, qp_num); 1126 } else { 1127 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1128 misc_parameters.bth_dst_qp); 1129 MLX5_SET(fte_match_param, spec->match_value, 1130 misc_parameters.bth_dst_qp, qp_num); 1131 } 1132 1133 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1134 1135 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1136 dst.counter = opfc->fc; 1137 1138 flow_act.action = 1139 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1140 1141 for (i = 0; i < spec_num; i++) { 1142 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1143 &flow_act, &dst, 1); 1144 if (IS_ERR(opfc->rule[i])) { 1145 err = PTR_ERR(opfc->rule[i]); 1146 goto del_rules; 1147 } 1148 } 1149 prio->refcount += spec_num; 1150 1151 err = xa_err(xa_store(qpn_opfc_xa, qp_num, per_qp_opfc, GFP_KERNEL)); 1152 if (err) 1153 goto del_rules; 1154 1155 kfree(spec); 1156 1157 return 0; 1158 1159 del_rules: 1160 while (i--) 1161 mlx5_del_flow_rules(opfc->rule[i]); 1162 put_flow_table(dev, prio, false); 1163 free_spec: 1164 kfree(spec); 1165 null_fc: 1166 opfc->fc = NULL; 1167 return err; 1168 } 1169 1170 static bool 1171 is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type, 1172 struct mlx5_fc **fc) 1173 { 1174 u32 shared_fc_type; 1175 1176 switch (type) { 1177 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 1178 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 1179 break; 1180 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 1181 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 1182 break; 1183 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 1184 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 1185 break; 1186 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 1187 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 1188 break; 1189 default: 1190 return false; 1191 } 1192 1193 *fc = fc_arr[shared_fc_type]; 1194 if (!(*fc)) 1195 return false; 1196 1197 return true; 1198 } 1199 1200 void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev, 1201 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX]) 1202 { 1203 struct mlx5_fc *in_use_fc; 1204 int i; 1205 1206 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1207 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1208 if (!fc_arr[i]) 1209 continue; 1210 1211 if (is_fc_shared_and_in_use(fc_arr, i, &in_use_fc)) { 1212 fc_arr[i] = NULL; 1213 continue; 1214 } 1215 1216 mlx5_fc_destroy(dev->mdev, fc_arr[i]); 1217 fc_arr[i] = NULL; 1218 } 1219 } 1220 1221 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num, 1222 struct mlx5_ib_op_fc *opfc, 1223 enum mlx5_ib_optional_counter_type type) 1224 { 1225 struct mlx5_flow_table_attr ft_attr = {}; 1226 enum mlx5_flow_namespace_type fn_type; 1227 int priority, i, err, spec_num; 1228 struct mlx5_flow_act flow_act = {}; 1229 struct mlx5_flow_destination dst; 1230 struct mlx5_flow_namespace *ns; 1231 struct mlx5_ib_flow_prio *prio; 1232 struct mlx5_flow_spec *spec; 1233 1234 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL); 1235 if (!spec) 1236 return -ENOMEM; 1237 1238 switch (type) { 1239 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS: 1240 if (set_ecn_ce_spec(dev, port_num, &spec[0], 1241 MLX5_FS_IPV4_VERSION) || 1242 set_ecn_ce_spec(dev, port_num, &spec[1], 1243 MLX5_FS_IPV6_VERSION)) { 1244 err = -EOPNOTSUPP; 1245 goto free; 1246 } 1247 spec_num = 2; 1248 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1249 priority = RDMA_RX_ECN_OPCOUNTER_PRIO; 1250 break; 1251 1252 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS: 1253 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1254 ft_field_support_2_nic_receive_rdma.bth_opcode) || 1255 set_cnp_spec(dev, port_num, &spec[0])) { 1256 err = -EOPNOTSUPP; 1257 goto free; 1258 } 1259 spec_num = 1; 1260 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1261 priority = RDMA_RX_CNP_OPCOUNTER_PRIO; 1262 break; 1263 1264 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS: 1265 if (!MLX5_CAP_FLOWTABLE(dev->mdev, 1266 ft_field_support_2_nic_transmit_rdma.bth_opcode) || 1267 set_cnp_spec(dev, port_num, &spec[0])) { 1268 err = -EOPNOTSUPP; 1269 goto free; 1270 } 1271 spec_num = 1; 1272 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1273 priority = RDMA_TX_CNP_OPCOUNTER_PRIO; 1274 break; 1275 1276 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 1277 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 1278 spec_num = 1; 1279 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS; 1280 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO; 1281 break; 1282 1283 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 1284 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 1285 spec_num = 1; 1286 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS; 1287 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO; 1288 break; 1289 1290 default: 1291 err = -EOPNOTSUPP; 1292 goto free; 1293 } 1294 1295 ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 1296 if (!ns) { 1297 err = -EOPNOTSUPP; 1298 goto free; 1299 } 1300 1301 prio = get_opfc_prio(dev, type); 1302 if (!prio->flow_table) { 1303 err = get_per_qp_prio(dev, type); 1304 if (err) 1305 goto free; 1306 1307 ft_attr.prio = priority; 1308 ft_attr.max_fte = dev->num_ports * MAX_OPFC_RULES; 1309 ft_attr.autogroup.max_num_groups = 1; 1310 prio = _get_prio(ns, prio, &ft_attr); 1311 if (IS_ERR(prio)) { 1312 err = PTR_ERR(prio); 1313 goto put_prio; 1314 } 1315 } 1316 1317 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1318 dst.counter = opfc->fc; 1319 1320 flow_act.action = 1321 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1322 1323 for (i = 0; i < spec_num; i++) { 1324 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i], 1325 &flow_act, &dst, 1); 1326 if (IS_ERR(opfc->rule[i])) { 1327 err = PTR_ERR(opfc->rule[i]); 1328 goto del_rules; 1329 } 1330 } 1331 prio->refcount += spec_num; 1332 kfree(spec); 1333 1334 return 0; 1335 1336 del_rules: 1337 for (i -= 1; i >= 0; i--) 1338 mlx5_del_flow_rules(opfc->rule[i]); 1339 put_flow_table(dev, prio, false); 1340 put_prio: 1341 put_per_qp_prio(dev, type); 1342 free: 1343 kfree(spec); 1344 return err; 1345 } 1346 1347 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev, 1348 struct mlx5_ib_op_fc *opfc, 1349 enum mlx5_ib_optional_counter_type type) 1350 { 1351 struct mlx5_ib_flow_prio *prio; 1352 int i; 1353 1354 prio = get_opfc_prio(dev, type); 1355 1356 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) { 1357 mlx5_del_flow_rules(opfc->rule[i]); 1358 put_flow_table(dev, prio, true); 1359 } 1360 1361 put_per_qp_prio(dev, type); 1362 } 1363 1364 void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa) 1365 { 1366 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1367 struct mlx5_per_qp_opfc *per_qp_opfc; 1368 struct mlx5_ib_op_fc *in_use_opfc; 1369 struct mlx5_ib_flow_prio *prio; 1370 int i, j; 1371 1372 per_qp_opfc = xa_load(qpn_opfc_xa, qp->qp_num); 1373 if (!per_qp_opfc) 1374 return; 1375 1376 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1377 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 1378 if (!per_qp_opfc->opfcs[i].fc) 1379 continue; 1380 1381 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i, 1382 &in_use_opfc)) { 1383 per_qp_opfc->opfcs[i].fc = NULL; 1384 continue; 1385 } 1386 1387 for (j = 0; j < MAX_OPFC_RULES; j++) { 1388 if (!per_qp_opfc->opfcs[i].rule[j]) 1389 continue; 1390 mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]); 1391 prio = get_opfc_prio(dev, i); 1392 put_flow_table(dev, prio, true); 1393 } 1394 per_qp_opfc->opfcs[i].fc = NULL; 1395 } 1396 1397 kfree(per_qp_opfc); 1398 xa_erase(qpn_opfc_xa, qp->qp_num); 1399 } 1400 1401 int mlx5r_fs_bind_op_fc(struct ib_qp *qp, 1402 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], 1403 struct xarray *qpn_opfc_xa, u32 port) 1404 { 1405 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1406 struct mlx5_per_qp_opfc *per_qp_opfc; 1407 struct mlx5_ib_flow_prio *prio; 1408 struct mlx5_ib_counters *cnts; 1409 struct mlx5_ib_op_fc *opfc; 1410 struct mlx5_fc *in_use_fc; 1411 int i, err, per_qp_type; 1412 bool new; 1413 1414 cnts = &dev->port[port - 1].cnts; 1415 1416 for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) { 1417 opfc = &cnts->opfcs[i]; 1418 if (!opfc->fc) 1419 continue; 1420 1421 per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 1422 prio = get_opfc_prio(dev, per_qp_type); 1423 WARN_ON(!prio->flow_table); 1424 1425 if (is_fc_shared_and_in_use(fc_arr, per_qp_type, &in_use_fc)) 1426 fc_arr[per_qp_type] = in_use_fc; 1427 1428 if (!fc_arr[per_qp_type]) { 1429 fc_arr[per_qp_type] = mlx5_fc_create(dev->mdev, false); 1430 if (IS_ERR(fc_arr[per_qp_type])) 1431 return PTR_ERR(fc_arr[per_qp_type]); 1432 } 1433 1434 per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp->qp_num, &new); 1435 if (!per_qp_opfc) { 1436 err = -ENOMEM; 1437 goto free_fc; 1438 } 1439 err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc, 1440 prio, per_qp_type, qp->qp_num, port); 1441 if (err) 1442 goto del_rules; 1443 } 1444 1445 return 0; 1446 1447 del_rules: 1448 mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa); 1449 if (new) 1450 kfree(per_qp_opfc); 1451 free_fc: 1452 if (xa_empty(qpn_opfc_xa)) 1453 mlx5r_fs_destroy_fcs(dev, fc_arr); 1454 return err; 1455 } 1456 1457 static void set_underlay_qp(struct mlx5_ib_dev *dev, 1458 struct mlx5_flow_spec *spec, 1459 u32 underlay_qpn) 1460 { 1461 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 1462 spec->match_criteria, 1463 misc_parameters); 1464 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1465 misc_parameters); 1466 1467 if (underlay_qpn && 1468 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1469 ft_field_support.bth_dst_qp)) { 1470 MLX5_SET(fte_match_set_misc, 1471 misc_params_v, bth_dst_qp, underlay_qpn); 1472 MLX5_SET(fte_match_set_misc, 1473 misc_params_c, bth_dst_qp, 0xffffff); 1474 } 1475 } 1476 1477 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 1478 struct mlx5_flow_spec *spec, 1479 struct mlx5_eswitch_rep *rep) 1480 { 1481 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 1482 void *misc; 1483 1484 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1485 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1486 misc_parameters_2); 1487 1488 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1489 mlx5_eswitch_get_vport_metadata_for_match(rep->esw, 1490 rep->vport)); 1491 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1492 misc_parameters_2); 1493 1494 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1495 mlx5_eswitch_get_vport_metadata_mask()); 1496 } else { 1497 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1498 misc_parameters); 1499 1500 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 1501 1502 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1503 misc_parameters); 1504 1505 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1506 } 1507 } 1508 1509 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 1510 struct mlx5_ib_flow_prio *ft_prio, 1511 const struct ib_flow_attr *flow_attr, 1512 struct mlx5_flow_destination *dst, 1513 u32 underlay_qpn, 1514 struct mlx5_ib_create_flow *ucmd) 1515 { 1516 struct mlx5_flow_table *ft = ft_prio->flow_table; 1517 struct mlx5_ib_flow_handler *handler; 1518 struct mlx5_flow_act flow_act = {}; 1519 struct mlx5_flow_spec *spec; 1520 struct mlx5_flow_destination dest_arr[2] = {}; 1521 struct mlx5_flow_destination *rule_dst = dest_arr; 1522 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 1523 unsigned int spec_index; 1524 u32 prev_type = 0; 1525 int err = 0; 1526 int dest_num = 0; 1527 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1528 1529 if (!is_valid_attr(dev->mdev, flow_attr)) 1530 return ERR_PTR(-EINVAL); 1531 1532 if (dev->is_rep && is_egress) 1533 return ERR_PTR(-EINVAL); 1534 1535 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1536 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1537 if (!handler || !spec) { 1538 err = -ENOMEM; 1539 goto free; 1540 } 1541 1542 INIT_LIST_HEAD(&handler->list); 1543 1544 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 1545 err = parse_flow_attr(dev->mdev, spec, 1546 ib_flow, flow_attr, &flow_act, 1547 prev_type); 1548 if (err < 0) 1549 goto free; 1550 1551 prev_type = ((union ib_flow_spec *)ib_flow)->type; 1552 ib_flow += ((union ib_flow_spec *)ib_flow)->size; 1553 } 1554 1555 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 1556 memcpy(&dest_arr[0], dst, sizeof(*dst)); 1557 dest_num++; 1558 } 1559 1560 if (!flow_is_multicast_only(flow_attr)) 1561 set_underlay_qp(dev, spec, underlay_qpn); 1562 1563 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { 1564 struct mlx5_eswitch_rep *rep; 1565 1566 rep = dev->port[flow_attr->port - 1].rep; 1567 if (!rep) { 1568 err = -EINVAL; 1569 goto free; 1570 } 1571 1572 mlx5_ib_set_rule_source_port(dev, spec, rep); 1573 } 1574 1575 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 1576 1577 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1578 struct mlx5_ib_mcounters *mcounters; 1579 1580 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 1581 if (err) 1582 goto free; 1583 1584 mcounters = to_mcounters(flow_act.counters); 1585 handler->ibcounters = flow_act.counters; 1586 dest_arr[dest_num].type = 1587 MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1588 dest_arr[dest_num].counter = 1589 mcounters->hw_cntrs_hndl; 1590 dest_num++; 1591 } 1592 1593 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 1594 if (!dest_num) 1595 rule_dst = NULL; 1596 } else { 1597 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 1598 flow_act.action |= 1599 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1600 if (is_egress) 1601 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1602 else if (dest_num) 1603 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1604 } 1605 1606 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1607 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1608 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1609 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1610 spec->flow_context.flow_tag, flow_attr->type); 1611 err = -EINVAL; 1612 goto free; 1613 } 1614 handler->rule = mlx5_add_flow_rules(ft, spec, 1615 &flow_act, 1616 rule_dst, dest_num); 1617 1618 if (IS_ERR(handler->rule)) { 1619 err = PTR_ERR(handler->rule); 1620 goto free; 1621 } 1622 1623 ft_prio->refcount++; 1624 handler->prio = ft_prio; 1625 handler->dev = dev; 1626 1627 ft_prio->flow_table = ft; 1628 free: 1629 if (err && handler) { 1630 mlx5_ib_counters_clear_description(handler->ibcounters); 1631 kfree(handler); 1632 } 1633 kvfree(spec); 1634 return err ? ERR_PTR(err) : handler; 1635 } 1636 1637 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1638 struct mlx5_ib_flow_prio *ft_prio, 1639 const struct ib_flow_attr *flow_attr, 1640 struct mlx5_flow_destination *dst) 1641 { 1642 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1643 } 1644 1645 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1646 struct mlx5_ib_flow_prio *ft_prio, 1647 struct ib_flow_attr *flow_attr, 1648 struct mlx5_flow_destination *dst) 1649 { 1650 struct mlx5_ib_flow_handler *handler_ucast = NULL; 1651 struct mlx5_ib_flow_handler *handler = NULL; 1652 1653 static struct { 1654 struct ib_flow_spec_eth eth_flow; 1655 struct ib_flow_attr flow_attr; 1656 } leftovers_wc = { .flow_attr = { .num_of_specs = 1, 1657 .size = sizeof(leftovers_wc) }, 1658 .eth_flow = { 1659 .type = IB_FLOW_SPEC_ETH, 1660 .size = sizeof(struct ib_flow_spec_eth), 1661 .mask = { .dst_mac = { 0x1 } }, 1662 .val = { .dst_mac = { 0x1 } } } }; 1663 1664 static struct { 1665 struct ib_flow_spec_eth eth_flow; 1666 struct ib_flow_attr flow_attr; 1667 } leftovers_uc = { .flow_attr = { .num_of_specs = 1, 1668 .size = sizeof(leftovers_uc) }, 1669 .eth_flow = { 1670 .type = IB_FLOW_SPEC_ETH, 1671 .size = sizeof(struct ib_flow_spec_eth), 1672 .mask = { .dst_mac = { 0x1 } }, 1673 .val = { .dst_mac = {} } } }; 1674 1675 handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst); 1676 if (!IS_ERR(handler) && 1677 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1678 handler_ucast = create_flow_rule(dev, ft_prio, 1679 &leftovers_uc.flow_attr, dst); 1680 if (IS_ERR(handler_ucast)) { 1681 mlx5_del_flow_rules(handler->rule); 1682 ft_prio->refcount--; 1683 kfree(handler); 1684 handler = handler_ucast; 1685 } else { 1686 list_add(&handler_ucast->list, &handler->list); 1687 } 1688 } 1689 1690 return handler; 1691 } 1692 1693 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1694 struct mlx5_ib_flow_prio *ft_rx, 1695 struct mlx5_ib_flow_prio *ft_tx, 1696 struct mlx5_flow_destination *dst) 1697 { 1698 struct mlx5_ib_flow_handler *handler_rx; 1699 struct mlx5_ib_flow_handler *handler_tx; 1700 int err; 1701 static const struct ib_flow_attr flow_attr = { 1702 .num_of_specs = 0, 1703 .type = IB_FLOW_ATTR_SNIFFER, 1704 .size = sizeof(flow_attr) 1705 }; 1706 1707 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1708 if (IS_ERR(handler_rx)) { 1709 err = PTR_ERR(handler_rx); 1710 goto err; 1711 } 1712 1713 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1714 if (IS_ERR(handler_tx)) { 1715 err = PTR_ERR(handler_tx); 1716 goto err_tx; 1717 } 1718 1719 list_add(&handler_tx->list, &handler_rx->list); 1720 1721 return handler_rx; 1722 1723 err_tx: 1724 mlx5_del_flow_rules(handler_rx->rule); 1725 ft_rx->refcount--; 1726 kfree(handler_rx); 1727 err: 1728 return ERR_PTR(err); 1729 } 1730 1731 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1732 struct ib_flow_attr *flow_attr, 1733 struct ib_udata *udata) 1734 { 1735 struct mlx5_ib_dev *dev = to_mdev(qp->device); 1736 struct mlx5_ib_qp *mqp = to_mqp(qp); 1737 struct mlx5_ib_flow_handler *handler = NULL; 1738 struct mlx5_flow_destination *dst = NULL; 1739 struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1740 struct mlx5_ib_flow_prio *ft_prio; 1741 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1742 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1743 size_t min_ucmd_sz, required_ucmd_sz; 1744 int err; 1745 int underlay_qpn; 1746 1747 if (udata && udata->inlen) { 1748 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); 1749 if (udata->inlen < min_ucmd_sz) 1750 return ERR_PTR(-EOPNOTSUPP); 1751 1752 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1753 if (err) 1754 return ERR_PTR(err); 1755 1756 /* currently supports only one counters data */ 1757 if (ucmd_hdr.ncounters_data > 1) 1758 return ERR_PTR(-EINVAL); 1759 1760 required_ucmd_sz = min_ucmd_sz + 1761 sizeof(struct mlx5_ib_flow_counters_data) * 1762 ucmd_hdr.ncounters_data; 1763 if (udata->inlen > required_ucmd_sz && 1764 !ib_is_udata_cleared(udata, required_ucmd_sz, 1765 udata->inlen - required_ucmd_sz)) 1766 return ERR_PTR(-EOPNOTSUPP); 1767 1768 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1769 if (!ucmd) 1770 return ERR_PTR(-ENOMEM); 1771 1772 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1773 if (err) 1774 goto free_ucmd; 1775 } 1776 1777 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1778 err = -ENOMEM; 1779 goto free_ucmd; 1780 } 1781 1782 if (flow_attr->flags & 1783 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) { 1784 err = -EINVAL; 1785 goto free_ucmd; 1786 } 1787 1788 if (is_egress && 1789 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1790 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1791 err = -EINVAL; 1792 goto free_ucmd; 1793 } 1794 1795 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1796 if (!dst) { 1797 err = -ENOMEM; 1798 goto free_ucmd; 1799 } 1800 1801 mutex_lock(&dev->flow_db->lock); 1802 1803 ft_prio = get_flow_table(dev, flow_attr, 1804 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1805 if (IS_ERR(ft_prio)) { 1806 err = PTR_ERR(ft_prio); 1807 goto unlock; 1808 } 1809 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1810 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1811 if (IS_ERR(ft_prio_tx)) { 1812 err = PTR_ERR(ft_prio_tx); 1813 ft_prio_tx = NULL; 1814 goto destroy_ft; 1815 } 1816 } 1817 1818 if (is_egress) { 1819 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1820 } else { 1821 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1822 if (mqp->is_rss) 1823 dst->tir_num = mqp->rss_qp.tirn; 1824 else 1825 dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1826 } 1827 1828 switch (flow_attr->type) { 1829 case IB_FLOW_ATTR_NORMAL: 1830 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1831 mqp->underlay_qpn : 1832 0; 1833 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1834 underlay_qpn, ucmd); 1835 break; 1836 case IB_FLOW_ATTR_ALL_DEFAULT: 1837 case IB_FLOW_ATTR_MC_DEFAULT: 1838 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); 1839 break; 1840 case IB_FLOW_ATTR_SNIFFER: 1841 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1842 break; 1843 default: 1844 err = -EINVAL; 1845 goto destroy_ft; 1846 } 1847 1848 if (IS_ERR(handler)) { 1849 err = PTR_ERR(handler); 1850 handler = NULL; 1851 goto destroy_ft; 1852 } 1853 1854 mutex_unlock(&dev->flow_db->lock); 1855 kfree(dst); 1856 kfree(ucmd); 1857 1858 return &handler->ibflow; 1859 1860 destroy_ft: 1861 put_flow_table(dev, ft_prio, false); 1862 if (ft_prio_tx) 1863 put_flow_table(dev, ft_prio_tx, false); 1864 unlock: 1865 mutex_unlock(&dev->flow_db->lock); 1866 kfree(dst); 1867 free_ucmd: 1868 kfree(ucmd); 1869 return ERR_PTR(err); 1870 } 1871 1872 static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev, 1873 enum mlx5_flow_namespace_type type, 1874 u32 *flags, u16 *vport_idx, 1875 u16 *vport, 1876 struct mlx5_core_dev **ft_mdev, 1877 u32 ib_port, u16 *esw_owner_vhca_id) 1878 { 1879 struct mlx5_core_dev *esw_mdev; 1880 1881 if (!is_mdev_switchdev_mode(dev->mdev)) 1882 return 0; 1883 1884 if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager)) 1885 return -EOPNOTSUPP; 1886 1887 if (!dev->port[ib_port - 1].rep) 1888 return -EINVAL; 1889 1890 esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw); 1891 if (esw_mdev != dev->mdev) { 1892 if (!MLX5_CAP_ADV_RDMA(dev->mdev, 1893 rdma_transport_manager_other_eswitch)) 1894 return -EOPNOTSUPP; 1895 *flags |= MLX5_FLOW_TABLE_OTHER_ESWITCH; 1896 *esw_owner_vhca_id = MLX5_CAP_GEN(esw_mdev, vhca_id); 1897 } 1898 1899 *flags |= MLX5_FLOW_TABLE_OTHER_VPORT; 1900 *ft_mdev = esw_mdev; 1901 *vport = dev->port[ib_port - 1].rep->vport; 1902 *vport_idx = dev->port[ib_port - 1].rep->vport_index; 1903 1904 return 0; 1905 } 1906 1907 static struct mlx5_ib_flow_prio * 1908 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority, 1909 enum mlx5_flow_namespace_type ns_type, 1910 bool mcast, u32 ib_port) 1911 { 1912 struct mlx5_core_dev *ft_mdev = dev->mdev; 1913 struct mlx5_flow_table_attr ft_attr = {}; 1914 struct mlx5_flow_namespace *ns = NULL; 1915 struct mlx5_ib_flow_prio *prio = NULL; 1916 u16 esw_owner_vhca_id = 0; 1917 int max_table_size = 0; 1918 u16 vport_idx = 0; 1919 bool esw_encap; 1920 u32 flags = 0; 1921 u16 vport = 0; 1922 int priority; 1923 int ret; 1924 1925 if (mcast) 1926 priority = MLX5_IB_FLOW_MCAST_PRIO; 1927 else 1928 priority = ib_prio_to_core_prio(user_priority, false); 1929 1930 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1931 DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1932 switch (ns_type) { 1933 case MLX5_FLOW_NAMESPACE_BYPASS: 1934 max_table_size = BIT( 1935 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); 1936 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1937 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1938 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1939 reformat_l3_tunnel_to_l2) && 1940 !esw_encap) 1941 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1942 break; 1943 case MLX5_FLOW_NAMESPACE_EGRESS: 1944 max_table_size = BIT( 1945 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1946 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && 1947 !esw_encap) 1948 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1949 break; 1950 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 1951 max_table_size = BIT( 1952 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1953 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1954 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1955 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, 1956 reformat_l3_tunnel_to_l2) && 1957 esw_encap) 1958 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1959 priority = user_priority; 1960 break; 1961 case MLX5_FLOW_NAMESPACE_RDMA_RX: 1962 max_table_size = BIT( 1963 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); 1964 priority = user_priority; 1965 break; 1966 case MLX5_FLOW_NAMESPACE_RDMA_TX: 1967 max_table_size = BIT( 1968 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); 1969 priority = user_priority; 1970 break; 1971 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 1972 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 1973 if (ib_port == 0 || 1974 user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO) 1975 return ERR_PTR(-EINVAL); 1976 ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags, 1977 &vport_idx, &vport, 1978 &ft_mdev, ib_port, 1979 &esw_owner_vhca_id); 1980 if (ret) 1981 return ERR_PTR(ret); 1982 1983 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) 1984 max_table_size = 1985 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX( 1986 ft_mdev, log_max_ft_size)); 1987 else 1988 max_table_size = 1989 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX( 1990 ft_mdev, log_max_ft_size)); 1991 priority = user_priority; 1992 break; 1993 default: 1994 break; 1995 } 1996 1997 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1998 1999 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX || 2000 ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) 2001 ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx); 2002 else 2003 ns = mlx5_get_flow_namespace(ft_mdev, ns_type); 2004 2005 if (!ns) 2006 return ERR_PTR(-EOPNOTSUPP); 2007 2008 switch (ns_type) { 2009 case MLX5_FLOW_NAMESPACE_BYPASS: 2010 prio = &dev->flow_db->prios[priority]; 2011 break; 2012 case MLX5_FLOW_NAMESPACE_EGRESS: 2013 prio = &dev->flow_db->egress_prios[priority]; 2014 break; 2015 case MLX5_FLOW_NAMESPACE_FDB_BYPASS: 2016 prio = &dev->flow_db->fdb[priority]; 2017 break; 2018 case MLX5_FLOW_NAMESPACE_RDMA_RX: 2019 prio = &dev->flow_db->rdma_rx[priority]; 2020 break; 2021 case MLX5_FLOW_NAMESPACE_RDMA_TX: 2022 prio = &dev->flow_db->rdma_tx[priority]; 2023 break; 2024 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX: 2025 prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1]; 2026 break; 2027 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX: 2028 prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1]; 2029 break; 2030 default: return ERR_PTR(-EINVAL); 2031 } 2032 2033 if (!prio) 2034 return ERR_PTR(-EINVAL); 2035 2036 if (prio->flow_table) 2037 return prio; 2038 2039 ft_attr.prio = priority; 2040 ft_attr.max_fte = max_table_size; 2041 ft_attr.flags = flags; 2042 ft_attr.vport = vport; 2043 ft_attr.esw_owner_vhca_id = esw_owner_vhca_id; 2044 ft_attr.autogroup.max_num_groups = MLX5_FS_MAX_TYPES; 2045 return _get_prio(ns, prio, &ft_attr); 2046 } 2047 2048 static struct mlx5_ib_flow_handler * 2049 _create_raw_flow_rule(struct mlx5_ib_dev *dev, 2050 struct mlx5_ib_flow_prio *ft_prio, 2051 struct mlx5_flow_destination *dst, 2052 struct mlx5_ib_flow_matcher *fs_matcher, 2053 struct mlx5_flow_context *flow_context, 2054 struct mlx5_flow_act *flow_act, 2055 void *cmd_in, int inlen, 2056 int dst_num) 2057 { 2058 struct mlx5_ib_flow_handler *handler; 2059 struct mlx5_flow_spec *spec; 2060 struct mlx5_flow_table *ft = ft_prio->flow_table; 2061 int err = 0; 2062 2063 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2064 handler = kzalloc(sizeof(*handler), GFP_KERNEL); 2065 if (!handler || !spec) { 2066 err = -ENOMEM; 2067 goto free; 2068 } 2069 2070 INIT_LIST_HEAD(&handler->list); 2071 2072 memcpy(spec->match_value, cmd_in, inlen); 2073 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 2074 fs_matcher->mask_len); 2075 spec->match_criteria_enable = fs_matcher->match_criteria_enable; 2076 spec->flow_context = *flow_context; 2077 2078 handler->rule = mlx5_add_flow_rules(ft, spec, 2079 flow_act, dst, dst_num); 2080 2081 if (IS_ERR(handler->rule)) { 2082 err = PTR_ERR(handler->rule); 2083 goto free; 2084 } 2085 2086 ft_prio->refcount++; 2087 handler->prio = ft_prio; 2088 handler->dev = dev; 2089 ft_prio->flow_table = ft; 2090 2091 free: 2092 if (err) 2093 kfree(handler); 2094 kvfree(spec); 2095 return err ? ERR_PTR(err) : handler; 2096 } 2097 2098 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 2099 void *match_v) 2100 { 2101 void *match_c; 2102 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 2103 void *dmac, *dmac_mask; 2104 void *ipv4, *ipv4_mask; 2105 2106 if (!(fs_matcher->match_criteria_enable & 2107 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 2108 return false; 2109 2110 match_c = fs_matcher->matcher_mask.match_params; 2111 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 2112 outer_headers); 2113 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 2114 outer_headers); 2115 2116 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2117 dmac_47_16); 2118 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2119 dmac_47_16); 2120 2121 if (is_multicast_ether_addr(dmac) && 2122 is_multicast_ether_addr(dmac_mask)) 2123 return true; 2124 2125 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 2126 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2127 2128 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 2129 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 2130 2131 if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 2132 ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 2133 return true; 2134 2135 return false; 2136 } 2137 2138 static struct mlx5_ib_flow_handler *raw_fs_rule_add( 2139 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 2140 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 2141 struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type) 2142 { 2143 struct mlx5_flow_destination *dst; 2144 struct mlx5_ib_flow_prio *ft_prio; 2145 struct mlx5_ib_flow_handler *handler; 2146 int dst_num = 0; 2147 bool mcast; 2148 int err; 2149 2150 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 2151 return ERR_PTR(-EOPNOTSUPP); 2152 2153 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 2154 return ERR_PTR(-ENOMEM); 2155 2156 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 2157 if (!dst) 2158 return ERR_PTR(-ENOMEM); 2159 2160 mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 2161 mutex_lock(&dev->flow_db->lock); 2162 2163 ft_prio = _get_flow_table(dev, fs_matcher->priority, 2164 fs_matcher->ns_type, mcast, 2165 fs_matcher->ib_port); 2166 if (IS_ERR(ft_prio)) { 2167 err = PTR_ERR(ft_prio); 2168 goto unlock; 2169 } 2170 2171 switch (dest_type) { 2172 case MLX5_FLOW_DESTINATION_TYPE_TIR: 2173 dst[dst_num].type = dest_type; 2174 dst[dst_num++].tir_num = dest_id; 2175 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2176 break; 2177 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: 2178 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 2179 dst[dst_num++].ft_num = dest_id; 2180 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2181 break; 2182 case MLX5_FLOW_DESTINATION_TYPE_PORT: 2183 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2184 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 2185 break; 2186 default: 2187 break; 2188 } 2189 2190 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 2191 if (WARN_ON(!counter)) { 2192 err = -EINVAL; 2193 goto unlock; 2194 } 2195 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2196 dst[dst_num].counter = counter; 2197 dst_num++; 2198 } 2199 2200 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, 2201 fs_matcher, flow_context, flow_act, 2202 cmd_in, inlen, dst_num); 2203 2204 if (IS_ERR(handler)) { 2205 err = PTR_ERR(handler); 2206 goto destroy_ft; 2207 } 2208 2209 mutex_unlock(&dev->flow_db->lock); 2210 atomic_inc(&fs_matcher->usecnt); 2211 handler->flow_matcher = fs_matcher; 2212 2213 kfree(dst); 2214 2215 return handler; 2216 2217 destroy_ft: 2218 put_flow_table(dev, ft_prio, false); 2219 unlock: 2220 mutex_unlock(&dev->flow_db->lock); 2221 kfree(dst); 2222 2223 return ERR_PTR(err); 2224 } 2225 2226 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 2227 { 2228 switch (maction->flow_action_raw.sub_type) { 2229 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 2230 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 2231 maction->flow_action_raw.modify_hdr); 2232 break; 2233 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 2234 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 2235 maction->flow_action_raw.pkt_reformat); 2236 break; 2237 case MLX5_IB_FLOW_ACTION_DECAP: 2238 break; 2239 default: 2240 break; 2241 } 2242 } 2243 2244 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 2245 { 2246 struct mlx5_ib_flow_action *maction = to_mflow_act(action); 2247 2248 switch (action->type) { 2249 case IB_FLOW_ACTION_UNSPECIFIED: 2250 destroy_flow_action_raw(maction); 2251 break; 2252 default: 2253 WARN_ON(true); 2254 break; 2255 } 2256 2257 kfree(maction); 2258 return 0; 2259 } 2260 2261 static int 2262 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 2263 enum mlx5_flow_namespace_type *namespace) 2264 { 2265 switch (table_type) { 2266 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 2267 *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 2268 break; 2269 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 2270 *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 2271 break; 2272 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 2273 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS; 2274 break; 2275 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 2276 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 2277 break; 2278 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 2279 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 2280 break; 2281 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX: 2282 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX; 2283 break; 2284 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX: 2285 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX; 2286 break; 2287 default: 2288 return -EINVAL; 2289 } 2290 2291 return 0; 2292 } 2293 2294 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 2295 [MLX5_IB_FLOW_TYPE_NORMAL] = { 2296 .type = UVERBS_ATTR_TYPE_PTR_IN, 2297 .u.ptr = { 2298 .len = sizeof(u16), /* data is priority */ 2299 .min_len = sizeof(u16), 2300 } 2301 }, 2302 [MLX5_IB_FLOW_TYPE_SNIFFER] = { 2303 .type = UVERBS_ATTR_TYPE_PTR_IN, 2304 UVERBS_ATTR_NO_DATA(), 2305 }, 2306 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 2307 .type = UVERBS_ATTR_TYPE_PTR_IN, 2308 UVERBS_ATTR_NO_DATA(), 2309 }, 2310 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 2311 .type = UVERBS_ATTR_TYPE_PTR_IN, 2312 UVERBS_ATTR_NO_DATA(), 2313 }, 2314 }; 2315 2316 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 2317 { 2318 struct devx_obj *devx_obj = obj; 2319 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2320 2321 switch (opcode) { 2322 case MLX5_CMD_OP_DESTROY_TIR: 2323 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2324 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 2325 obj_id); 2326 return true; 2327 2328 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 2329 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2330 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 2331 table_id); 2332 return true; 2333 default: 2334 return false; 2335 } 2336 } 2337 2338 static int get_dests(struct uverbs_attr_bundle *attrs, 2339 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 2340 int *dest_type, struct ib_qp **qp, u32 *flags) 2341 { 2342 bool dest_devx, dest_qp; 2343 void *devx_obj; 2344 int err; 2345 2346 dest_devx = uverbs_attr_is_valid(attrs, 2347 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2348 dest_qp = uverbs_attr_is_valid(attrs, 2349 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2350 2351 *flags = 0; 2352 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2353 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 2354 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 2355 if (err) 2356 return err; 2357 2358 /* Both flags are not allowed */ 2359 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 2360 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2361 return -EINVAL; 2362 2363 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 2364 if (dest_devx && (dest_qp || *flags)) 2365 return -EINVAL; 2366 else if (dest_qp && *flags) 2367 return -EINVAL; 2368 } 2369 2370 /* Allow only DEVX object, drop as dest for FDB */ 2371 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2372 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 2373 return -EINVAL; 2374 2375 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 2376 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2377 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2378 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 2379 return -EINVAL; 2380 2381 *qp = NULL; 2382 if (dest_devx) { 2383 devx_obj = 2384 uverbs_attr_get_obj(attrs, 2385 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 2386 2387 /* Verify that the given DEVX object is a flow 2388 * steering destination. 2389 */ 2390 if (!is_flow_dest(devx_obj, dest_id, dest_type)) 2391 return -EINVAL; 2392 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 2393 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS || 2394 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX || 2395 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) && 2396 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 2397 return -EINVAL; 2398 } else if (dest_qp) { 2399 struct mlx5_ib_qp *mqp; 2400 2401 *qp = uverbs_attr_get_obj(attrs, 2402 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 2403 if (IS_ERR(*qp)) 2404 return PTR_ERR(*qp); 2405 2406 if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 2407 return -EINVAL; 2408 2409 mqp = to_mqp(*qp); 2410 if (mqp->is_rss) 2411 *dest_id = mqp->rss_qp.tirn; 2412 else 2413 *dest_id = mqp->raw_packet_qp.rq.tirn; 2414 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 2415 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2416 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2417 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) && 2418 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { 2419 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 2420 } 2421 2422 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 2423 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || 2424 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX || 2425 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)) 2426 return -EINVAL; 2427 2428 return 0; 2429 } 2430 2431 static bool 2432 is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size) 2433 { 2434 struct devx_obj *devx_obj = obj; 2435 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 2436 2437 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 2438 2439 if (offset && offset >= devx_obj->flow_counter_bulk_size) 2440 return false; 2441 2442 *fc_bulk_size = devx_obj->flow_counter_bulk_size; 2443 *counter_id = MLX5_GET(dealloc_flow_counter_in, 2444 devx_obj->dinbox, 2445 flow_counter_id); 2446 *counter_id += offset; 2447 return true; 2448 } 2449 2450 return false; 2451 } 2452 2453 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 2454 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 2455 struct uverbs_attr_bundle *attrs) 2456 { 2457 struct mlx5_flow_context flow_context = {.flow_tag = 2458 MLX5_FS_DEFAULT_FLOW_TAG}; 2459 int dest_id, dest_type = -1, inlen, len, ret, i; 2460 struct mlx5_ib_flow_handler *flow_handler; 2461 struct mlx5_ib_flow_matcher *fs_matcher; 2462 struct ib_uobject **arr_flow_actions; 2463 struct ib_uflow_resources *uflow_res; 2464 struct mlx5_flow_act flow_act = {}; 2465 struct mlx5_fc *counter = NULL; 2466 struct ib_qp *qp = NULL; 2467 void *devx_obj, *cmd_in; 2468 struct ib_uobject *uobj; 2469 struct mlx5_ib_dev *dev; 2470 u32 flags; 2471 2472 if (!rdma_uattrs_has_raw_cap(attrs)) 2473 return -EPERM; 2474 2475 fs_matcher = uverbs_attr_get_obj(attrs, 2476 MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 2477 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 2478 dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2479 2480 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 2481 return -EINVAL; 2482 2483 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 2484 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 2485 2486 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 2487 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 2488 2489 len = uverbs_attr_get_uobjs_arr(attrs, 2490 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 2491 if (len) { 2492 u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0; 2493 devx_obj = arr_flow_actions[0]->object; 2494 2495 if (uverbs_attr_is_valid(attrs, 2496 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 2497 2498 int num_offsets = uverbs_attr_ptr_get_array_size( 2499 attrs, 2500 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2501 sizeof(u32)); 2502 2503 if (num_offsets != 1) 2504 return -EINVAL; 2505 2506 offset_attr = uverbs_attr_get_alloced_ptr( 2507 attrs, 2508 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 2509 offset = *offset_attr; 2510 } 2511 2512 if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size)) 2513 return -EINVAL; 2514 counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size); 2515 if (IS_ERR(counter)) 2516 return PTR_ERR(counter); 2517 2518 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 2519 } 2520 2521 cmd_in = uverbs_attr_get_alloced_ptr( 2522 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2523 inlen = uverbs_attr_get_len(attrs, 2524 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 2525 2526 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 2527 if (!uflow_res) { 2528 ret = -ENOMEM; 2529 goto destroy_counter; 2530 } 2531 2532 len = uverbs_attr_get_uobjs_arr(attrs, 2533 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 2534 for (i = 0; i < len; i++) { 2535 struct mlx5_ib_flow_action *maction = 2536 to_mflow_act(arr_flow_actions[i]->object); 2537 2538 ret = parse_flow_flow_action(maction, false, &flow_act); 2539 if (ret) 2540 goto err_out; 2541 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 2542 arr_flow_actions[i]->object); 2543 } 2544 2545 ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 2546 MLX5_IB_ATTR_CREATE_FLOW_TAG); 2547 if (!ret) { 2548 if (flow_context.flow_tag >= BIT(24)) { 2549 ret = -EINVAL; 2550 goto err_out; 2551 } 2552 flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 2553 } 2554 2555 flow_handler = 2556 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 2557 counter, cmd_in, inlen, dest_id, dest_type); 2558 if (IS_ERR(flow_handler)) { 2559 ret = PTR_ERR(flow_handler); 2560 goto err_out; 2561 } 2562 2563 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2564 2565 return 0; 2566 err_out: 2567 ib_uverbs_flow_resources_free(uflow_res); 2568 destroy_counter: 2569 if (counter) 2570 mlx5_fc_local_destroy(counter); 2571 return ret; 2572 } 2573 2574 static int flow_matcher_cleanup(struct ib_uobject *uobject, 2575 enum rdma_remove_reason why, 2576 struct uverbs_attr_bundle *attrs) 2577 { 2578 struct mlx5_ib_flow_matcher *obj = uobject->object; 2579 2580 if (atomic_read(&obj->usecnt)) 2581 return -EBUSY; 2582 2583 kfree(obj); 2584 return 0; 2585 } 2586 2587 static int steering_anchor_create_ft(struct mlx5_ib_dev *dev, 2588 struct mlx5_ib_flow_prio *ft_prio, 2589 enum mlx5_flow_namespace_type ns_type) 2590 { 2591 struct mlx5_flow_table_attr ft_attr = {}; 2592 struct mlx5_flow_namespace *ns; 2593 struct mlx5_flow_table *ft; 2594 2595 if (ft_prio->anchor.ft) 2596 return 0; 2597 2598 ns = mlx5_get_flow_namespace(dev->mdev, ns_type); 2599 if (!ns) 2600 return -EOPNOTSUPP; 2601 2602 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 2603 ft_attr.uid = MLX5_SHARED_RESOURCE_UID; 2604 ft_attr.prio = 0; 2605 ft_attr.max_fte = 2; 2606 ft_attr.level = 1; 2607 2608 ft = mlx5_create_flow_table(ns, &ft_attr); 2609 if (IS_ERR(ft)) 2610 return PTR_ERR(ft); 2611 2612 ft_prio->anchor.ft = ft; 2613 2614 return 0; 2615 } 2616 2617 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio) 2618 { 2619 if (ft_prio->anchor.ft) { 2620 mlx5_destroy_flow_table(ft_prio->anchor.ft); 2621 ft_prio->anchor.ft = NULL; 2622 } 2623 } 2624 2625 static int 2626 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2627 { 2628 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2629 struct mlx5_flow_group *fg; 2630 void *flow_group_in; 2631 int err = 0; 2632 2633 if (ft_prio->anchor.fg_drop) 2634 return 0; 2635 2636 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2637 if (!flow_group_in) 2638 return -ENOMEM; 2639 2640 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1); 2641 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); 2642 2643 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2644 if (IS_ERR(fg)) { 2645 err = PTR_ERR(fg); 2646 goto out; 2647 } 2648 2649 ft_prio->anchor.fg_drop = fg; 2650 2651 out: 2652 kvfree(flow_group_in); 2653 2654 return err; 2655 } 2656 2657 static void 2658 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio) 2659 { 2660 if (ft_prio->anchor.fg_drop) { 2661 mlx5_destroy_flow_group(ft_prio->anchor.fg_drop); 2662 ft_prio->anchor.fg_drop = NULL; 2663 } 2664 } 2665 2666 static int 2667 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2668 { 2669 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2670 struct mlx5_flow_group *fg; 2671 void *flow_group_in; 2672 int err = 0; 2673 2674 if (ft_prio->anchor.fg_goto_table) 2675 return 0; 2676 2677 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2678 if (!flow_group_in) 2679 return -ENOMEM; 2680 2681 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in); 2682 if (IS_ERR(fg)) { 2683 err = PTR_ERR(fg); 2684 goto out; 2685 } 2686 ft_prio->anchor.fg_goto_table = fg; 2687 2688 out: 2689 kvfree(flow_group_in); 2690 2691 return err; 2692 } 2693 2694 static void 2695 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2696 { 2697 if (ft_prio->anchor.fg_goto_table) { 2698 mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table); 2699 ft_prio->anchor.fg_goto_table = NULL; 2700 } 2701 } 2702 2703 static int 2704 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2705 { 2706 struct mlx5_flow_act flow_act = {}; 2707 struct mlx5_flow_handle *handle; 2708 2709 if (ft_prio->anchor.rule_drop) 2710 return 0; 2711 2712 flow_act.fg = ft_prio->anchor.fg_drop; 2713 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2714 2715 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2716 NULL, 0); 2717 if (IS_ERR(handle)) 2718 return PTR_ERR(handle); 2719 2720 ft_prio->anchor.rule_drop = handle; 2721 2722 return 0; 2723 } 2724 2725 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio) 2726 { 2727 if (ft_prio->anchor.rule_drop) { 2728 mlx5_del_flow_rules(ft_prio->anchor.rule_drop); 2729 ft_prio->anchor.rule_drop = NULL; 2730 } 2731 } 2732 2733 static int 2734 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2735 { 2736 struct mlx5_flow_destination dest = {}; 2737 struct mlx5_flow_act flow_act = {}; 2738 struct mlx5_flow_handle *handle; 2739 2740 if (ft_prio->anchor.rule_goto_table) 2741 return 0; 2742 2743 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2744 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 2745 flow_act.fg = ft_prio->anchor.fg_goto_table; 2746 2747 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2748 dest.ft = ft_prio->flow_table; 2749 2750 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act, 2751 &dest, 1); 2752 if (IS_ERR(handle)) 2753 return PTR_ERR(handle); 2754 2755 ft_prio->anchor.rule_goto_table = handle; 2756 2757 return 0; 2758 } 2759 2760 static void 2761 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio) 2762 { 2763 if (ft_prio->anchor.rule_goto_table) { 2764 mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table); 2765 ft_prio->anchor.rule_goto_table = NULL; 2766 } 2767 } 2768 2769 static int steering_anchor_create_res(struct mlx5_ib_dev *dev, 2770 struct mlx5_ib_flow_prio *ft_prio, 2771 enum mlx5_flow_namespace_type ns_type) 2772 { 2773 int err; 2774 2775 err = steering_anchor_create_ft(dev, ft_prio, ns_type); 2776 if (err) 2777 return err; 2778 2779 err = steering_anchor_create_fg_drop(ft_prio); 2780 if (err) 2781 goto destroy_ft; 2782 2783 err = steering_anchor_create_fg_goto_table(ft_prio); 2784 if (err) 2785 goto destroy_fg_drop; 2786 2787 err = steering_anchor_create_rule_drop(ft_prio); 2788 if (err) 2789 goto destroy_fg_goto_table; 2790 2791 err = steering_anchor_create_rule_goto_table(ft_prio); 2792 if (err) 2793 goto destroy_rule_drop; 2794 2795 return 0; 2796 2797 destroy_rule_drop: 2798 steering_anchor_destroy_rule_drop(ft_prio); 2799 destroy_fg_goto_table: 2800 steering_anchor_destroy_fg_goto_table(ft_prio); 2801 destroy_fg_drop: 2802 steering_anchor_destroy_fg_drop(ft_prio); 2803 destroy_ft: 2804 steering_anchor_destroy_ft(ft_prio); 2805 2806 return err; 2807 } 2808 2809 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio) 2810 { 2811 steering_anchor_destroy_rule_goto_table(ft_prio); 2812 steering_anchor_destroy_rule_drop(ft_prio); 2813 steering_anchor_destroy_fg_goto_table(ft_prio); 2814 steering_anchor_destroy_fg_drop(ft_prio); 2815 steering_anchor_destroy_ft(ft_prio); 2816 } 2817 2818 static int steering_anchor_cleanup(struct ib_uobject *uobject, 2819 enum rdma_remove_reason why, 2820 struct uverbs_attr_bundle *attrs) 2821 { 2822 struct mlx5_ib_steering_anchor *obj = uobject->object; 2823 2824 if (atomic_read(&obj->usecnt)) 2825 return -EBUSY; 2826 2827 mutex_lock(&obj->dev->flow_db->lock); 2828 if (!--obj->ft_prio->anchor.rule_goto_table_ref) 2829 steering_anchor_destroy_rule_goto_table(obj->ft_prio); 2830 2831 put_flow_table(obj->dev, obj->ft_prio, true); 2832 mutex_unlock(&obj->dev->flow_db->lock); 2833 2834 kfree(obj); 2835 return 0; 2836 } 2837 2838 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio, 2839 int count) 2840 { 2841 while (count--) 2842 mlx5_steering_anchor_destroy_res(&prio[count]); 2843 } 2844 2845 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev) 2846 { 2847 fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT); 2848 fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT); 2849 fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS); 2850 fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS); 2851 fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS); 2852 fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT); 2853 fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT); 2854 } 2855 2856 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2857 struct mlx5_ib_flow_matcher *obj) 2858 { 2859 enum mlx5_ib_uapi_flow_table_type ft_type = 2860 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2861 u32 flags; 2862 int err; 2863 2864 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2865 * users should switch to it. We leave this to not break userspace 2866 */ 2867 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2868 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2869 return -EINVAL; 2870 2871 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2872 err = uverbs_get_const(&ft_type, attrs, 2873 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2874 if (err) 2875 return err; 2876 2877 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2878 if (err) 2879 return err; 2880 2881 return 0; 2882 } 2883 2884 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2885 err = uverbs_get_flags32(&flags, attrs, 2886 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2887 IB_FLOW_ATTR_FLAGS_EGRESS); 2888 if (err) 2889 return err; 2890 2891 if (flags) 2892 return mlx5_ib_ft_type_to_namespace( 2893 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2894 &obj->ns_type); 2895 } 2896 2897 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2898 2899 return 0; 2900 } 2901 2902 static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps) 2903 { 2904 if (is_mdev_switchdev_mode(dev->mdev)) 2905 return UCAP_ENABLED(enabled_caps, 2906 RDMA_UCAP_MLX5_CTRL_OTHER_VHCA); 2907 2908 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL); 2909 } 2910 2911 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2912 struct uverbs_attr_bundle *attrs) 2913 { 2914 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2915 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2916 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2917 struct mlx5_ib_flow_matcher *obj; 2918 int err; 2919 2920 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2921 if (!obj) 2922 return -ENOMEM; 2923 2924 obj->mask_len = uverbs_attr_get_len( 2925 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2926 err = uverbs_copy_from(&obj->matcher_mask, 2927 attrs, 2928 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2929 if (err) 2930 goto end; 2931 2932 obj->flow_type = uverbs_attr_get_enum_id( 2933 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2934 2935 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2936 err = uverbs_copy_from(&obj->priority, 2937 attrs, 2938 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2939 if (err) 2940 goto end; 2941 } 2942 2943 err = uverbs_copy_from(&obj->match_criteria_enable, 2944 attrs, 2945 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2946 if (err) 2947 goto end; 2948 2949 err = mlx5_ib_matcher_ns(attrs, obj); 2950 if (err) 2951 goto end; 2952 2953 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && 2954 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { 2955 err = -EINVAL; 2956 goto end; 2957 } 2958 2959 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) { 2960 err = uverbs_copy_from(&obj->ib_port, attrs, 2961 MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT); 2962 if (err) 2963 goto end; 2964 if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) { 2965 err = -EINVAL; 2966 goto end; 2967 } 2968 if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX && 2969 obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) { 2970 err = -EINVAL; 2971 goto end; 2972 } 2973 if (!verify_context_caps(dev, uobj->context->enabled_caps)) { 2974 err = -EOPNOTSUPP; 2975 goto end; 2976 } 2977 } 2978 2979 uobj->object = obj; 2980 obj->mdev = dev->mdev; 2981 atomic_set(&obj->usecnt, 0); 2982 return 0; 2983 2984 end: 2985 kfree(obj); 2986 return err; 2987 } 2988 2989 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( 2990 struct uverbs_attr_bundle *attrs) 2991 { 2992 struct ib_uobject *uobj = uverbs_attr_get_uobject( 2993 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE); 2994 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2995 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type; 2996 enum mlx5_flow_namespace_type ns_type; 2997 struct mlx5_ib_steering_anchor *obj; 2998 struct mlx5_ib_flow_prio *ft_prio; 2999 u16 priority; 3000 u32 ft_id; 3001 int err; 3002 3003 if (!rdma_dev_has_raw_cap(&dev->ib_dev)) 3004 return -EPERM; 3005 3006 err = uverbs_get_const(&ib_uapi_ft_type, attrs, 3007 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE); 3008 if (err) 3009 return err; 3010 3011 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type); 3012 if (err) 3013 return err; 3014 3015 err = uverbs_copy_from(&priority, attrs, 3016 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY); 3017 if (err) 3018 return err; 3019 3020 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 3021 if (!obj) 3022 return -ENOMEM; 3023 3024 mutex_lock(&dev->flow_db->lock); 3025 3026 ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0); 3027 if (IS_ERR(ft_prio)) { 3028 err = PTR_ERR(ft_prio); 3029 goto free_obj; 3030 } 3031 3032 ft_prio->refcount++; 3033 3034 if (!ft_prio->anchor.rule_goto_table_ref) { 3035 err = steering_anchor_create_res(dev, ft_prio, ns_type); 3036 if (err) 3037 goto put_flow_table; 3038 } 3039 3040 ft_prio->anchor.rule_goto_table_ref++; 3041 3042 ft_id = mlx5_flow_table_id(ft_prio->anchor.ft); 3043 3044 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3045 &ft_id, sizeof(ft_id)); 3046 if (err) 3047 goto destroy_res; 3048 3049 mutex_unlock(&dev->flow_db->lock); 3050 3051 uobj->object = obj; 3052 obj->dev = dev; 3053 obj->ft_prio = ft_prio; 3054 atomic_set(&obj->usecnt, 0); 3055 3056 return 0; 3057 3058 destroy_res: 3059 --ft_prio->anchor.rule_goto_table_ref; 3060 mlx5_steering_anchor_destroy_res(ft_prio); 3061 put_flow_table: 3062 put_flow_table(dev, ft_prio, true); 3063 free_obj: 3064 mutex_unlock(&dev->flow_db->lock); 3065 kfree(obj); 3066 3067 return err; 3068 } 3069 3070 static struct ib_flow_action * 3071 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 3072 enum mlx5_ib_uapi_flow_table_type ft_type, 3073 u8 num_actions, void *in) 3074 { 3075 enum mlx5_flow_namespace_type namespace; 3076 struct mlx5_ib_flow_action *maction; 3077 int ret; 3078 3079 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3080 if (ret) 3081 return ERR_PTR(-EINVAL); 3082 3083 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3084 if (!maction) 3085 return ERR_PTR(-ENOMEM); 3086 3087 maction->flow_action_raw.modify_hdr = 3088 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 3089 3090 if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 3091 ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 3092 kfree(maction); 3093 return ERR_PTR(ret); 3094 } 3095 maction->flow_action_raw.sub_type = 3096 MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 3097 maction->flow_action_raw.dev = dev; 3098 3099 return &maction->ib_action; 3100 } 3101 3102 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 3103 { 3104 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3105 max_modify_header_actions) || 3106 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 3107 max_modify_header_actions) || 3108 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 3109 max_modify_header_actions); 3110 } 3111 3112 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 3113 struct uverbs_attr_bundle *attrs) 3114 { 3115 struct ib_uobject *uobj = uverbs_attr_get_uobject( 3116 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 3117 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3118 enum mlx5_ib_uapi_flow_table_type ft_type; 3119 struct ib_flow_action *action; 3120 int num_actions; 3121 void *in; 3122 int ret; 3123 3124 if (!mlx5_ib_modify_header_supported(mdev)) 3125 return -EOPNOTSUPP; 3126 3127 in = uverbs_attr_get_alloced_ptr(attrs, 3128 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 3129 3130 num_actions = uverbs_attr_ptr_get_array_size( 3131 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3132 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 3133 if (num_actions < 0) 3134 return num_actions; 3135 3136 ret = uverbs_get_const(&ft_type, attrs, 3137 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 3138 if (ret) 3139 return ret; 3140 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 3141 if (IS_ERR(action)) 3142 return PTR_ERR(action); 3143 3144 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 3145 IB_FLOW_ACTION_UNSPECIFIED); 3146 3147 return 0; 3148 } 3149 3150 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 3151 u8 packet_reformat_type, 3152 u8 ft_type) 3153 { 3154 switch (packet_reformat_type) { 3155 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3156 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3157 return MLX5_CAP_FLOWTABLE(ibdev->mdev, 3158 encap_general_header); 3159 break; 3160 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3161 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 3162 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 3163 reformat_l2_to_l3_tunnel); 3164 break; 3165 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3166 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3167 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 3168 reformat_l3_tunnel_to_l2); 3169 break; 3170 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 3171 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 3172 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 3173 break; 3174 default: 3175 break; 3176 } 3177 3178 return false; 3179 } 3180 3181 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 3182 { 3183 switch (dv_prt) { 3184 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 3185 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 3186 break; 3187 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 3188 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 3189 break; 3190 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 3191 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 3192 break; 3193 default: 3194 return -EINVAL; 3195 } 3196 3197 return 0; 3198 } 3199 3200 static int mlx5_ib_flow_action_create_packet_reformat_ctx( 3201 struct mlx5_ib_dev *dev, 3202 struct mlx5_ib_flow_action *maction, 3203 u8 ft_type, u8 dv_prt, 3204 void *in, size_t len) 3205 { 3206 struct mlx5_pkt_reformat_params reformat_params; 3207 enum mlx5_flow_namespace_type namespace; 3208 u8 prm_prt; 3209 int ret; 3210 3211 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 3212 if (ret) 3213 return ret; 3214 3215 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 3216 if (ret) 3217 return ret; 3218 3219 memset(&reformat_params, 0, sizeof(reformat_params)); 3220 reformat_params.type = prm_prt; 3221 reformat_params.size = len; 3222 reformat_params.data = in; 3223 maction->flow_action_raw.pkt_reformat = 3224 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params, 3225 namespace); 3226 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 3227 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 3228 return ret; 3229 } 3230 3231 maction->flow_action_raw.sub_type = 3232 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 3233 maction->flow_action_raw.dev = dev; 3234 3235 return 0; 3236 } 3237 3238 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 3239 struct uverbs_attr_bundle *attrs) 3240 { 3241 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 3242 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 3243 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 3244 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 3245 enum mlx5_ib_uapi_flow_table_type ft_type; 3246 struct mlx5_ib_flow_action *maction; 3247 int ret; 3248 3249 ret = uverbs_get_const(&ft_type, attrs, 3250 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 3251 if (ret) 3252 return ret; 3253 3254 ret = uverbs_get_const(&dv_prt, attrs, 3255 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 3256 if (ret) 3257 return ret; 3258 3259 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 3260 return -EOPNOTSUPP; 3261 3262 maction = kzalloc(sizeof(*maction), GFP_KERNEL); 3263 if (!maction) 3264 return -ENOMEM; 3265 3266 if (dv_prt == 3267 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 3268 maction->flow_action_raw.sub_type = 3269 MLX5_IB_FLOW_ACTION_DECAP; 3270 maction->flow_action_raw.dev = mdev; 3271 } else { 3272 void *in; 3273 int len; 3274 3275 in = uverbs_attr_get_alloced_ptr(attrs, 3276 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3277 if (IS_ERR(in)) { 3278 ret = PTR_ERR(in); 3279 goto free_maction; 3280 } 3281 3282 len = uverbs_attr_get_len(attrs, 3283 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 3284 3285 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 3286 maction, ft_type, dv_prt, in, len); 3287 if (ret) 3288 goto free_maction; 3289 } 3290 3291 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 3292 IB_FLOW_ACTION_UNSPECIFIED); 3293 return 0; 3294 3295 free_maction: 3296 kfree(maction); 3297 return ret; 3298 } 3299 3300 DECLARE_UVERBS_NAMED_METHOD( 3301 MLX5_IB_METHOD_CREATE_FLOW, 3302 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3303 UVERBS_OBJECT_FLOW, 3304 UVERBS_ACCESS_NEW, 3305 UA_MANDATORY), 3306 UVERBS_ATTR_PTR_IN( 3307 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 3308 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3309 UA_MANDATORY, 3310 UA_ALLOC_AND_COPY), 3311 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 3312 MLX5_IB_OBJECT_FLOW_MATCHER, 3313 UVERBS_ACCESS_READ, 3314 UA_MANDATORY), 3315 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 3316 UVERBS_OBJECT_QP, 3317 UVERBS_ACCESS_READ), 3318 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 3319 MLX5_IB_OBJECT_DEVX_OBJ, 3320 UVERBS_ACCESS_READ), 3321 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 3322 UVERBS_OBJECT_FLOW_ACTION, 3323 UVERBS_ACCESS_READ, 1, 3324 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 3325 UA_OPTIONAL), 3326 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 3327 UVERBS_ATTR_TYPE(u32), 3328 UA_OPTIONAL), 3329 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 3330 MLX5_IB_OBJECT_DEVX_OBJ, 3331 UVERBS_ACCESS_READ, 1, 1, 3332 UA_OPTIONAL), 3333 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 3334 UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 3335 UA_OPTIONAL, 3336 UA_ALLOC_AND_COPY), 3337 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 3338 enum mlx5_ib_create_flow_flags, 3339 UA_OPTIONAL)); 3340 3341 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3342 MLX5_IB_METHOD_DESTROY_FLOW, 3343 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 3344 UVERBS_OBJECT_FLOW, 3345 UVERBS_ACCESS_DESTROY, 3346 UA_MANDATORY)); 3347 3348 ADD_UVERBS_METHODS(mlx5_ib_fs, 3349 UVERBS_OBJECT_FLOW, 3350 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 3351 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 3352 3353 DECLARE_UVERBS_NAMED_METHOD( 3354 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 3355 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 3356 UVERBS_OBJECT_FLOW_ACTION, 3357 UVERBS_ACCESS_NEW, 3358 UA_MANDATORY), 3359 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 3360 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 3361 set_add_copy_action_in_auto)), 3362 UA_MANDATORY, 3363 UA_ALLOC_AND_COPY), 3364 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 3365 enum mlx5_ib_uapi_flow_table_type, 3366 UA_MANDATORY)); 3367 3368 DECLARE_UVERBS_NAMED_METHOD( 3369 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 3370 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 3371 UVERBS_OBJECT_FLOW_ACTION, 3372 UVERBS_ACCESS_NEW, 3373 UA_MANDATORY), 3374 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 3375 UVERBS_ATTR_MIN_SIZE(1), 3376 UA_ALLOC_AND_COPY, 3377 UA_OPTIONAL), 3378 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 3379 enum mlx5_ib_uapi_flow_action_packet_reformat_type, 3380 UA_MANDATORY), 3381 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 3382 enum mlx5_ib_uapi_flow_table_type, 3383 UA_MANDATORY)); 3384 3385 ADD_UVERBS_METHODS( 3386 mlx5_ib_flow_actions, 3387 UVERBS_OBJECT_FLOW_ACTION, 3388 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 3389 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 3390 3391 DECLARE_UVERBS_NAMED_METHOD( 3392 MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 3393 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 3394 MLX5_IB_OBJECT_FLOW_MATCHER, 3395 UVERBS_ACCESS_NEW, 3396 UA_MANDATORY), 3397 UVERBS_ATTR_PTR_IN( 3398 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 3399 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 3400 UA_MANDATORY), 3401 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 3402 mlx5_ib_flow_type, 3403 UA_MANDATORY), 3404 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 3405 UVERBS_ATTR_TYPE(u8), 3406 UA_MANDATORY), 3407 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 3408 enum ib_flow_flags, 3409 UA_OPTIONAL), 3410 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 3411 enum mlx5_ib_uapi_flow_table_type, 3412 UA_OPTIONAL), 3413 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT, 3414 UVERBS_ATTR_TYPE(u32), 3415 UA_OPTIONAL)); 3416 3417 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3418 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 3419 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 3420 MLX5_IB_OBJECT_FLOW_MATCHER, 3421 UVERBS_ACCESS_DESTROY, 3422 UA_MANDATORY)); 3423 3424 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 3425 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 3426 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 3427 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 3428 3429 DECLARE_UVERBS_NAMED_METHOD( 3430 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE, 3431 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE, 3432 MLX5_IB_OBJECT_STEERING_ANCHOR, 3433 UVERBS_ACCESS_NEW, 3434 UA_MANDATORY), 3435 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE, 3436 enum mlx5_ib_uapi_flow_table_type, 3437 UA_MANDATORY), 3438 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY, 3439 UVERBS_ATTR_TYPE(u16), 3440 UA_MANDATORY), 3441 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID, 3442 UVERBS_ATTR_TYPE(u32), 3443 UA_MANDATORY)); 3444 3445 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 3446 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY, 3447 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE, 3448 MLX5_IB_OBJECT_STEERING_ANCHOR, 3449 UVERBS_ACCESS_DESTROY, 3450 UA_MANDATORY)); 3451 3452 DECLARE_UVERBS_NAMED_OBJECT( 3453 MLX5_IB_OBJECT_STEERING_ANCHOR, 3454 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup), 3455 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE), 3456 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY)); 3457 3458 const struct uapi_definition mlx5_ib_flow_defs[] = { 3459 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3460 MLX5_IB_OBJECT_FLOW_MATCHER), 3461 UAPI_DEF_CHAIN_OBJ_TREE( 3462 UVERBS_OBJECT_FLOW, 3463 &mlx5_ib_fs), 3464 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 3465 &mlx5_ib_flow_actions), 3466 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 3467 MLX5_IB_OBJECT_STEERING_ANCHOR, 3468 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)), 3469 {}, 3470 }; 3471 3472 static const struct ib_device_ops flow_ops = { 3473 .create_flow = mlx5_ib_create_flow, 3474 .destroy_flow = mlx5_ib_destroy_flow, 3475 .destroy_flow_action = mlx5_ib_destroy_flow_action, 3476 }; 3477 3478 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 3479 { 3480 int i, j; 3481 3482 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 3483 3484 if (!dev->flow_db) 3485 return -ENOMEM; 3486 3487 for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) { 3488 dev->flow_db->rdma_transport_rx[i] = 3489 kcalloc(dev->num_ports, 3490 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL); 3491 if (!dev->flow_db->rdma_transport_rx[i]) 3492 goto free_rdma_transport_rx; 3493 } 3494 3495 for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) { 3496 dev->flow_db->rdma_transport_tx[j] = 3497 kcalloc(dev->num_ports, 3498 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL); 3499 if (!dev->flow_db->rdma_transport_tx[j]) 3500 goto free_rdma_transport_tx; 3501 } 3502 3503 mutex_init(&dev->flow_db->lock); 3504 3505 ib_set_device_ops(&dev->ib_dev, &flow_ops); 3506 return 0; 3507 3508 free_rdma_transport_tx: 3509 while (j--) 3510 kfree(dev->flow_db->rdma_transport_tx[j]); 3511 free_rdma_transport_rx: 3512 while (i--) 3513 kfree(dev->flow_db->rdma_transport_rx[i]); 3514 kfree(dev->flow_db); 3515 return -ENOMEM; 3516 } 3517