1 /*- 2 * Copyright (c) 2023 NVIDIA corporation & affiliates. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 27 #include "opt_ipsec.h" 28 29 #include <sys/types.h> 30 #include <netinet/in.h> 31 #include <sys/socket.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <net/pfkeyv2.h> 35 #include <netipsec/key_var.h> 36 #include <netipsec/keydb.h> 37 #include <netipsec/ipsec.h> 38 #include <netipsec/xform.h> 39 #include <netipsec/ipsec_offload.h> 40 #include <dev/mlx5/fs.h> 41 #include <dev/mlx5/mlx5_en/en.h> 42 #include <dev/mlx5/qp.h> 43 #include <dev/mlx5/mlx5_accel/ipsec.h> 44 #include <dev/mlx5/mlx5_core/fs_core.h> 45 #include <dev/mlx5/mlx5_core/fs_chains.h> 46 47 /* 48 * TX tables are organized differently for Ethernet and for RoCE: 49 * 50 * +=========+ 51 * Ethernet Tx | SA KSPI | match 52 * --------------------->|Flowtable|----->+ + 53 * | |\ | / \ 54 * +=========+ | | / \ +=========+ +=========+ 55 * miss | | / \ | Status | | | 56 * DROP<--------+ |---->|Encrypt|------>|Flowtable|---->| TX NS | 57 * | \ / | | | | 58 * | \ / +=========+ +=========+ 59 * +=========+ +=========+ | \ / | 60 * RoCE | Policy | match|SA ReqId |match| + | 61 * Tx |Flowtable|----->|Flowtable|---->+ | 62 * ---->|IP header| |ReqId+IP | | 63 * | | | header |--------------------------------+ 64 * +=========+ +=========+ miss | 65 * | | 66 * | miss | 67 * +------------------------------------------------------- 68 * 69 * +=========+ 70 * | RDMA | 71 * |Flowtable| 72 * | | 73 * Rx Tables and rules: +=========+ 74 * + / 75 * +=========+ +=========+ / \ +=========+ +=========+ /match 76 * | Policy | | SA | / \ | Status | | RoCE |/ 77 * ---->|Flowtable| match|Flowtable| match / \ |Flowtable|----->|Flowtable| 78 * |IP header|----->|IP header|----->|Decrypt|----->| | | Roce V2 | 79 * | | |+ESP+SPI | \ / | | | UDP port|\ 80 * +=========+ +=========+ \ / +=========+ +=========+ \miss 81 * | | \ / \ 82 * | | + +=========+ 83 * | miss | miss | Ethernet| 84 * +--------------->---------------------------------------------------->| RX NS | 85 * | | 86 * +=========+ 87 * 88 */ 89 90 #define NUM_IPSEC_FTE BIT(15) 91 #define IPSEC_TUNNEL_DEFAULT_TTL 0x40 92 93 struct mlx5e_ipsec_fc { 94 struct mlx5_fc *cnt; 95 struct mlx5_fc *drop; 96 }; 97 98 struct mlx5e_ipsec_ft { 99 struct mutex mutex; /* Protect changes to this struct */ 100 struct mlx5_flow_table *pol; 101 struct mlx5_flow_table *sa_kspi; 102 struct mlx5_flow_table *sa; 103 struct mlx5_flow_table *status; 104 u32 refcnt; 105 }; 106 107 struct mlx5e_ipsec_tx_roce { 108 struct mlx5_flow_group *g; 109 struct mlx5_flow_table *ft; 110 struct mlx5_flow_handle *rule; 111 struct mlx5_flow_namespace *ns; 112 }; 113 114 struct mlx5e_ipsec_miss { 115 struct mlx5_flow_group *group; 116 struct mlx5_flow_handle *rule; 117 }; 118 119 struct mlx5e_ipsec_tx { 120 struct mlx5e_ipsec_ft ft; 121 struct mlx5e_ipsec_miss pol; 122 struct mlx5e_ipsec_miss kspi_miss; 123 struct mlx5e_ipsec_rule status; 124 struct mlx5e_ipsec_rule kspi_bypass_rule; /*rule for IPSEC bypass*/ 125 struct mlx5_flow_namespace *ns; 126 struct mlx5e_ipsec_fc *fc; 127 struct mlx5_fs_chains *chains; 128 struct mlx5e_ipsec_tx_roce roce; 129 }; 130 131 struct mlx5e_ipsec_rx_roce { 132 struct mlx5_flow_group *g; 133 struct mlx5_flow_table *ft; 134 struct mlx5_flow_handle *rule; 135 struct mlx5e_ipsec_miss roce_miss; 136 137 struct mlx5_flow_table *ft_rdma; 138 struct mlx5_flow_namespace *ns_rdma; 139 }; 140 141 struct mlx5e_ipsec_rx { 142 struct mlx5e_ipsec_ft ft; 143 struct mlx5e_ipsec_miss pol; 144 struct mlx5e_ipsec_miss sa; 145 struct mlx5e_ipsec_rule status; 146 struct mlx5_flow_namespace *ns; 147 struct mlx5e_ipsec_fc *fc; 148 struct mlx5_fs_chains *chains; 149 struct mlx5e_ipsec_rx_roce roce; 150 }; 151 152 static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec, 153 u16 kspi); 154 static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec); 155 156 static void setup_fte_no_frags(struct mlx5_flow_spec *spec) 157 { 158 /* Non fragmented */ 159 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 160 161 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); 162 MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); 163 } 164 165 static void setup_fte_esp(struct mlx5_flow_spec *spec) 166 { 167 /* ESP header */ 168 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 169 170 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 171 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); 172 } 173 174 static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi, bool encap) 175 { 176 /* SPI number */ 177 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 178 179 if (encap) { 180 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.inner_esp_spi); 181 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.inner_esp_spi, spi); 182 } else { 183 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); 184 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi); 185 } 186 } 187 188 static void 189 setup_fte_vid(struct mlx5_flow_spec *spec, u16 vid) 190 { 191 /* virtual lan tag */ 192 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 193 194 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 195 outer_headers.cvlan_tag); 196 MLX5_SET(fte_match_param, spec->match_value, 197 outer_headers.cvlan_tag, 1); 198 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 199 outer_headers.first_vid); 200 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, 201 vid); 202 } 203 204 static void 205 clear_fte_vid(struct mlx5_flow_spec *spec) 206 { 207 MLX5_SET(fte_match_param, spec->match_criteria, 208 outer_headers.cvlan_tag, 0); 209 MLX5_SET(fte_match_param, spec->match_value, 210 outer_headers.cvlan_tag, 0); 211 MLX5_SET(fte_match_param, spec->match_criteria, 212 outer_headers.first_vid, 0); 213 MLX5_SET(fte_match_param, spec->match_value, 214 outer_headers.first_vid, 0); 215 } 216 217 static void 218 setup_fte_no_vid(struct mlx5_flow_spec *spec) 219 { 220 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 221 outer_headers.cvlan_tag); 222 MLX5_SET(fte_match_param, spec->match_value, 223 outer_headers.cvlan_tag, 0); 224 } 225 226 static struct mlx5_fs_chains * 227 ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft, 228 enum mlx5_flow_namespace_type ns, int base_prio, 229 int base_level, struct mlx5_flow_table **root_ft) 230 { 231 struct mlx5_chains_attr attr = {}; 232 struct mlx5_fs_chains *chains; 233 struct mlx5_flow_table *ft; 234 int err; 235 236 attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED | 237 MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 238 attr.max_grp_num = 2; 239 attr.default_ft = miss_ft; 240 attr.ns = ns; 241 attr.fs_base_prio = base_prio; 242 attr.fs_base_level = base_level; 243 chains = mlx5_chains_create(mdev, &attr); 244 if (IS_ERR(chains)) 245 return chains; 246 247 /* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */ 248 ft = mlx5_chains_get_table(chains, 0, 1, 0); 249 if (IS_ERR(ft)) { 250 err = PTR_ERR(ft); 251 goto err_chains_get; 252 } 253 254 *root_ft = ft; 255 return chains; 256 257 err_chains_get: 258 mlx5_chains_destroy(chains); 259 return ERR_PTR(err); 260 } 261 262 static void ipsec_chains_destroy(struct mlx5_fs_chains *chains) 263 { 264 mlx5_chains_put_table(chains, 0, 1, 0); 265 mlx5_chains_destroy(chains); 266 } 267 268 static struct mlx5_flow_table * 269 ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio) 270 { 271 return mlx5_chains_get_table(chains, 0, prio + 1, 0); 272 } 273 274 static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio) 275 { 276 mlx5_chains_put_table(chains, 0, prio + 1, 0); 277 } 278 279 static struct mlx5_flow_table *ipsec_rx_ft_create(struct mlx5_flow_namespace *ns, 280 int level, int prio, 281 int max_num_groups) 282 { 283 struct mlx5_flow_table_attr ft_attr = {}; 284 285 ft_attr.max_fte = NUM_IPSEC_FTE; 286 ft_attr.level = level; 287 ft_attr.prio = prio; 288 ft_attr.autogroup.max_num_groups = max_num_groups; 289 ft_attr.autogroup.num_reserved_entries = 1; 290 291 return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 292 } 293 294 static int ipsec_miss_create(struct mlx5_core_dev *mdev, 295 struct mlx5_flow_table *ft, 296 struct mlx5e_ipsec_miss *miss, 297 struct mlx5_flow_destination *dest) 298 { 299 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 300 struct mlx5_flow_act flow_act = {}; 301 struct mlx5_flow_spec *spec; 302 u32 *flow_group_in; 303 int err = 0; 304 305 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 306 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 307 if (!flow_group_in || !spec) { 308 err = -ENOMEM; 309 goto out; 310 } 311 312 /* Create miss_group */ 313 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); 314 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); 315 miss->group = mlx5_create_flow_group(ft, flow_group_in); 316 if (IS_ERR(miss->group)) { 317 err = PTR_ERR(miss->group); 318 mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", 319 err); 320 goto out; 321 } 322 323 if (dest) 324 flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST; 325 else 326 flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP; 327 /* Create miss rule */ 328 miss->rule = mlx5_add_flow_rules(ft, NULL, &flow_act, dest, 1); 329 if (IS_ERR(miss->rule)) { 330 mlx5_destroy_flow_group(miss->group); 331 err = PTR_ERR(miss->rule); 332 mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", 333 err); 334 goto out; 335 } 336 out: 337 kvfree(flow_group_in); 338 kvfree(spec); 339 return err; 340 } 341 342 static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, 343 struct mlx5_flow_act *flow_act) 344 { 345 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 346 enum mlx5_flow_namespace_type ns_type; 347 struct mlx5_modify_hdr *modify_hdr; 348 349 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); 350 switch (dir) { 351 case IPSEC_DIR_INBOUND: 352 MLX5_SET(set_action_in, action, field, 353 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 354 ns_type = MLX5_FLOW_NAMESPACE_KERNEL; 355 break; 356 case IPSEC_DIR_OUTBOUND: 357 MLX5_SET(set_action_in, action, field, 358 MLX5_ACTION_IN_FIELD_METADATA_REG_C_0); 359 ns_type = MLX5_FLOW_NAMESPACE_EGRESS; 360 break; 361 default: 362 return -EINVAL; 363 } 364 365 MLX5_SET(set_action_in, action, data, val); 366 MLX5_SET(set_action_in, action, offset, 0); 367 MLX5_SET(set_action_in, action, length, 32); 368 369 modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action); 370 if (IS_ERR(modify_hdr)) { 371 mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", 372 PTR_ERR(modify_hdr)); 373 return PTR_ERR(modify_hdr); 374 } 375 376 flow_act->modify_hdr = modify_hdr; 377 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 378 return 0; 379 } 380 381 static int 382 setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs, 383 struct mlx5_pkt_reformat_params *reformat_params) 384 { 385 struct udphdr *udphdr; 386 size_t bfflen = 16; 387 char *reformatbf; 388 __be32 spi; 389 void *hdr; 390 391 if (attrs->family == AF_INET) { 392 if (attrs->encap) 393 reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4; 394 else 395 reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; 396 } else { 397 if (attrs->encap) 398 reformat_params->type = 399 MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6; 400 else 401 reformat_params->type = 402 MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; 403 } 404 405 if (attrs->encap) 406 bfflen += sizeof(*udphdr); 407 reformatbf = kzalloc(bfflen, GFP_KERNEL); 408 if (!reformatbf) 409 return -ENOMEM; 410 411 hdr = reformatbf; 412 if (attrs->encap) { 413 udphdr = (struct udphdr *)reformatbf; 414 udphdr->uh_sport = attrs->sport; 415 udphdr->uh_dport = attrs->dport; 416 hdr += sizeof(*udphdr); 417 } 418 419 /* convert to network format */ 420 spi = htonl(attrs->spi); 421 memcpy(hdr, &spi, 4); 422 423 reformat_params->param_0 = attrs->authsize; 424 reformat_params->size = bfflen; 425 reformat_params->data = reformatbf; 426 427 return 0; 428 } 429 430 static int setup_pkt_reformat(struct mlx5_core_dev *mdev, 431 struct mlx5_accel_esp_xfrm_attrs *attrs, 432 struct mlx5_flow_act *flow_act) 433 { 434 enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS; 435 struct mlx5_pkt_reformat_params reformat_params = {}; 436 struct mlx5_pkt_reformat *pkt_reformat; 437 int ret; 438 439 if (attrs->dir == IPSEC_DIR_INBOUND) { 440 if (attrs->encap) 441 reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP; 442 else 443 reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; 444 ns_type = MLX5_FLOW_NAMESPACE_KERNEL; 445 goto cmd; 446 } 447 448 ret = setup_pkt_transport_reformat(attrs, &reformat_params); 449 if (ret) 450 return ret; 451 cmd: 452 pkt_reformat = 453 mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); 454 if (reformat_params.data) 455 kfree(reformat_params.data); 456 if (IS_ERR(pkt_reformat)) 457 return PTR_ERR(pkt_reformat); 458 459 flow_act->pkt_reformat = pkt_reformat; 460 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 461 return 0; 462 } 463 464 static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr, 465 __be32 *daddr) 466 { 467 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 468 469 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 470 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 471 472 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 473 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); 474 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 475 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); 476 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 477 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 478 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 479 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 480 } 481 482 static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr, 483 __be32 *daddr) 484 { 485 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 486 487 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 488 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 489 490 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 491 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); 492 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 493 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); 494 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 495 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); 496 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 497 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); 498 } 499 500 static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 501 { 502 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 503 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 504 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 505 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 506 struct mlx5_flow_destination dest[2] = {}; 507 struct mlx5_flow_act flow_act = {}; 508 struct mlx5_flow_handle *rule; 509 struct mlx5_flow_spec *spec; 510 struct mlx5e_ipsec_rx *rx; 511 struct mlx5_fc *counter; 512 int err; 513 514 rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6; 515 516 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 517 if (!spec) 518 return -ENOMEM; 519 520 if (!attrs->drop) { 521 err = setup_modify_header(mdev, sa_entry->kspi | BIT(31), IPSEC_DIR_INBOUND, 522 &flow_act); 523 if (err) 524 goto err_mod_header; 525 } 526 527 err = setup_pkt_reformat(mdev, attrs, &flow_act); 528 if (err) 529 goto err_pkt_reformat; 530 531 counter = mlx5_fc_create(mdev, false); 532 if (IS_ERR(counter)) { 533 err = PTR_ERR(counter); 534 goto err_add_cnt; 535 } 536 537 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 538 flow_act.crypto.op = MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT; 539 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 540 flow_act.flags |= FLOW_ACT_NO_APPEND; 541 542 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | 543 MLX5_FLOW_CONTEXT_ACTION_COUNT; 544 545 if (attrs->drop) 546 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 547 else 548 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 549 550 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 551 dest[0].ft = rx->ft.status; 552 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 553 dest[1].counter_id = mlx5_fc_id(counter); 554 555 if (attrs->family == AF_INET) 556 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 557 else 558 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 559 560 if (!attrs->encap) 561 setup_fte_esp(spec); 562 563 setup_fte_spi(spec, attrs->spi, attrs->encap); 564 setup_fte_no_frags(spec); 565 566 if (sa_entry->vid != VLAN_NONE) 567 setup_fte_vid(spec, sa_entry->vid); 568 else 569 setup_fte_no_vid(spec); 570 571 rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2); 572 if (IS_ERR(rule)) { 573 err = PTR_ERR(rule); 574 mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); 575 goto err_add_flow; 576 } 577 ipsec_rule->rule = rule; 578 579 /* Add another rule for zero vid */ 580 if (sa_entry->vid == VLAN_NONE) { 581 clear_fte_vid(spec); 582 setup_fte_vid(spec, 0); 583 rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2); 584 if (IS_ERR(rule)) { 585 err = PTR_ERR(rule); 586 mlx5_core_err(mdev, 587 "fail to add RX ipsec zero vid rule err=%d\n", 588 err); 589 goto err_add_flow; 590 } 591 ipsec_rule->vid_zero_rule = rule; 592 } 593 594 kvfree(spec); 595 ipsec_rule->fc = counter; 596 ipsec_rule->modify_hdr = flow_act.modify_hdr; 597 ipsec_rule->pkt_reformat = flow_act.pkt_reformat; 598 return 0; 599 600 err_add_flow: 601 mlx5_fc_destroy(mdev, counter); 602 if (ipsec_rule->rule != NULL) 603 mlx5_del_flow_rules(&ipsec_rule->rule); 604 err_add_cnt: 605 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 606 err_pkt_reformat: 607 if (flow_act.modify_hdr != NULL) 608 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 609 err_mod_header: 610 kvfree(spec); 611 612 return err; 613 } 614 615 static struct mlx5_flow_table *ipsec_tx_ft_create(struct mlx5_flow_namespace *ns, 616 int level, int prio, 617 int max_num_groups) 618 { 619 struct mlx5_flow_table_attr ft_attr = {}; 620 621 ft_attr.autogroup.num_reserved_entries = 1; 622 ft_attr.autogroup.max_num_groups = max_num_groups; 623 ft_attr.max_fte = NUM_IPSEC_FTE; 624 ft_attr.level = level; 625 ft_attr.prio = prio; 626 627 return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 628 } 629 630 static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 631 { 632 struct mlx5_flow_destination dest = {}; 633 struct mlx5_flow_act flow_act = {}; 634 struct mlx5_flow_handle *fte; 635 int err; 636 637 /* create fte */ 638 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_COUNT | 639 MLX5_FLOW_CONTEXT_ACTION_ALLOW; 640 641 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 642 dest.counter_id = mlx5_fc_id(tx->fc->cnt); 643 fte = mlx5_add_flow_rules(tx->ft.status, NULL, &flow_act, &dest, 1); 644 if (IS_ERR_OR_NULL(fte)) { 645 err = PTR_ERR(fte); 646 mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err); 647 goto err_rule; 648 } 649 650 tx->status.rule = fte; 651 return 0; 652 653 err_rule: 654 return err; 655 } 656 657 static void tx_destroy_roce(struct mlx5e_ipsec_tx *tx) { 658 if (!tx->roce.ft) 659 return; 660 661 mlx5_del_flow_rules(&tx->roce.rule); 662 mlx5_destroy_flow_group(tx->roce.g); 663 mlx5_destroy_flow_table(tx->roce.ft); 664 tx->roce.ft = NULL; 665 } 666 667 /* IPsec TX flow steering */ 668 static void tx_destroy(struct mlx5e_ipsec_tx *tx) 669 { 670 tx_destroy_roce(tx); 671 if (tx->chains) { 672 ipsec_chains_destroy(tx->chains); 673 } else { 674 mlx5_del_flow_rules(&tx->pol.rule); 675 mlx5_destroy_flow_group(tx->pol.group); 676 mlx5_destroy_flow_table(tx->ft.pol); 677 } 678 mlx5_destroy_flow_table(tx->ft.sa); 679 mlx5_del_flow_rules(&tx->kspi_miss.rule); 680 mlx5_destroy_flow_group(tx->kspi_miss.group); 681 mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule); 682 mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule); 683 mlx5_destroy_flow_table(tx->ft.sa_kspi); 684 mlx5_del_flow_rules(&tx->status.rule); 685 mlx5_destroy_flow_table(tx->ft.status); 686 } 687 688 static int ipsec_tx_roce_rule_setup(struct mlx5_core_dev *mdev, 689 struct mlx5e_ipsec_tx *tx) 690 { 691 struct mlx5_flow_destination dst = {}; 692 struct mlx5_flow_act flow_act = {}; 693 struct mlx5_flow_handle *rule; 694 int err = 0; 695 696 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 697 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 698 dst.ft = tx->ft.pol; 699 rule = mlx5_add_flow_rules(tx->roce.ft, NULL, &flow_act, &dst, 1); 700 if (IS_ERR(rule)) { 701 err = PTR_ERR(rule); 702 mlx5_core_err(mdev, "Fail to add TX roce ipsec rule err=%d\n", 703 err); 704 goto out; 705 } 706 tx->roce.rule = rule; 707 708 out: 709 return err; 710 } 711 712 static int ipsec_tx_create_roce(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 713 { 714 struct mlx5_flow_table_attr ft_attr = {}; 715 struct mlx5_flow_table *ft; 716 struct mlx5_flow_group *g; 717 int ix = 0; 718 int err; 719 u32 *in; 720 721 if (!tx->roce.ns) 722 return -EOPNOTSUPP; 723 724 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 725 if (!in) 726 return -ENOMEM; 727 728 ft_attr.max_fte = 1; 729 ft = mlx5_create_flow_table(tx->roce.ns, &ft_attr); 730 if (IS_ERR(ft)) { 731 err = PTR_ERR(ft); 732 mlx5_core_err(mdev, "Fail to create ipsec tx roce ft err=%d\n", 733 err); 734 goto fail_table; 735 } 736 tx->roce.ft = ft; 737 738 MLX5_SET_CFG(in, start_flow_index, ix); 739 ix += 1; 740 MLX5_SET_CFG(in, end_flow_index, ix - 1); 741 g = mlx5_create_flow_group(ft, in); 742 if (IS_ERR(g)) { 743 err = PTR_ERR(g); 744 mlx5_core_err(mdev, "Fail to create ipsec tx roce group err=%d\n", 745 err); 746 goto fail_group; 747 } 748 tx->roce.g = g; 749 750 err = ipsec_tx_roce_rule_setup(mdev, tx); 751 if (err) { 752 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err); 753 goto fail_rule; 754 } 755 756 kvfree(in); 757 return 0; 758 759 fail_rule: 760 mlx5_destroy_flow_group(tx->roce.g); 761 fail_group: 762 mlx5_destroy_flow_table(tx->roce.ft); 763 tx->roce.ft = NULL; 764 fail_table: 765 kvfree(in); 766 return err; 767 } 768 769 /* 770 * Setting a rule in KSPI table for values that should bypass IPSEC. 771 * 772 * mdev - mlx5 core device 773 * tx - IPSEC TX 774 * return - 0 for success errno for failure 775 */ 776 static int tx_create_kspi_bypass_rules(struct mlx5_core_dev *mdev, 777 struct mlx5e_ipsec_tx *tx) 778 { 779 struct mlx5_flow_destination dest = {}; 780 struct mlx5_flow_act flow_act = {}; 781 struct mlx5_flow_act flow_act_kspi = {}; 782 struct mlx5_flow_handle *rule; 783 struct mlx5_flow_spec *spec; 784 int err; 785 786 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 787 if (!spec) 788 return -ENOMEM; 789 790 dest.ft = tx->ft.status; 791 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 792 flow_act_kspi.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 793 794 setup_fte_reg_a_with_tag(spec, IPSEC_ACCEL_DRV_SPI_BYPASS); 795 rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act_kspi, 796 &dest, 1); 797 if (IS_ERR(rule)) { 798 err = PTR_ERR(rule); 799 mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n", 800 err); 801 goto err_add_kspi_rule; 802 } 803 tx->kspi_bypass_rule.kspi_rule = rule; 804 805 /* set the rule for packets withoiut ipsec tag. */ 806 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 807 memset(spec, 0, sizeof(*spec)); 808 setup_fte_reg_a_no_tag(spec); 809 rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act, &dest, 1); 810 if (IS_ERR(rule)) { 811 err = PTR_ERR(rule); 812 mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n", err); 813 goto err_add_rule; 814 } 815 tx->kspi_bypass_rule.rule = rule; 816 817 kvfree(spec); 818 return 0; 819 err_add_rule: 820 mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule); 821 err_add_kspi_rule: 822 kvfree(spec); 823 return err; 824 } 825 826 827 static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 828 { 829 struct mlx5_flow_destination dest = {}; 830 struct mlx5_flow_table *ft; 831 int err; 832 833 /* 834 * Tx flow is different for ethernet traffic then for RoCE packets 835 * For Ethernet packets we start in SA KSPI table that matches KSPI of SA rule 836 * to the KSPI in the packet metadata 837 * For RoCE traffic we start in Policy table, then move to SA table 838 * which matches either reqid of the SA rule to reqid reported by policy table 839 * or ip header fields of SA to the packet IP header fields. 840 * Tables are ordered by their level so we set kspi 841 * with level 0 to have it first one for ethernet traffic. 842 * For RoCE the RoCE TX table direct the packets to policy table explicitly 843 */ 844 ft = ipsec_tx_ft_create(tx->ns, 0, 0, 4); 845 if (IS_ERR(ft)) 846 return PTR_ERR(ft); 847 tx->ft.sa_kspi = ft; 848 849 ft = ipsec_tx_ft_create(tx->ns, 2, 0, 4); 850 if (IS_ERR(ft)) { 851 err = PTR_ERR(ft); 852 goto err_reqid_ft; 853 } 854 tx->ft.sa = ft; 855 856 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 857 tx->chains = ipsec_chains_create( 858 mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 1, 859 &tx->ft.pol); 860 if (IS_ERR(tx->chains)) { 861 err = PTR_ERR(tx->chains); 862 goto err_pol_ft; 863 } 864 } else { 865 ft = ipsec_tx_ft_create(tx->ns, 1, 0, 2); 866 if (IS_ERR(ft)) { 867 err = PTR_ERR(ft); 868 goto err_pol_ft; 869 } 870 tx->ft.pol = ft; 871 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 872 dest.ft = tx->ft.sa; 873 err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); 874 if (err) 875 goto err_pol_miss; 876 } 877 878 ft = ipsec_tx_ft_create(tx->ns, 2, 0, 1); 879 if (IS_ERR(ft)) { 880 err = PTR_ERR(ft); 881 goto err_status_ft; 882 } 883 tx->ft.status = ft; 884 885 /* set miss rule for kspi table with drop action*/ 886 err = ipsec_miss_create(mdev, tx->ft.sa_kspi, &tx->kspi_miss, NULL); 887 if (err) 888 goto err_kspi_miss; 889 890 err = tx_create_kspi_bypass_rules(mdev, tx); 891 if (err) 892 goto err_kspi_rule; 893 894 err = ipsec_counter_rule_tx(mdev, tx); 895 if (err) 896 goto err_status_rule; 897 898 err = ipsec_tx_create_roce(mdev, tx); 899 if (err) 900 goto err_counter_rule; 901 902 return 0; 903 904 err_counter_rule: 905 mlx5_del_flow_rules(&tx->status.rule); 906 err_status_rule: 907 mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule); 908 mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule); 909 err_kspi_rule: 910 mlx5_destroy_flow_table(tx->ft.status); 911 err_status_ft: 912 if (tx->chains) { 913 ipsec_chains_destroy(tx->chains); 914 } else { 915 mlx5_del_flow_rules(&tx->pol.rule); 916 mlx5_destroy_flow_group(tx->pol.group); 917 } 918 err_pol_miss: 919 if (!tx->chains) 920 mlx5_destroy_flow_table(tx->ft.pol); 921 err_pol_ft: 922 mlx5_del_flow_rules(&tx->kspi_miss.rule); 923 mlx5_destroy_flow_group(tx->kspi_miss.group); 924 err_kspi_miss: 925 mlx5_destroy_flow_table(tx->ft.sa); 926 err_reqid_ft: 927 mlx5_destroy_flow_table(tx->ft.sa_kspi); 928 return err; 929 } 930 931 static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 932 struct mlx5e_ipsec_tx *tx) 933 { 934 int err; 935 936 if (tx->ft.refcnt) 937 goto skip; 938 939 err = tx_create(mdev, tx); 940 if (err) 941 return err; 942 943 skip: 944 tx->ft.refcnt++; 945 return 0; 946 } 947 948 static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) 949 { 950 if (--tx->ft.refcnt) 951 return; 952 953 tx_destroy(tx); 954 } 955 956 static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, 957 struct mlx5e_ipsec *ipsec) 958 { 959 struct mlx5e_ipsec_tx *tx = ipsec->tx; 960 int err; 961 962 mutex_lock(&tx->ft.mutex); 963 err = tx_get(mdev, ipsec, tx); 964 mutex_unlock(&tx->ft.mutex); 965 if (err) 966 return ERR_PTR(err); 967 968 return tx; 969 } 970 971 static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev, 972 struct mlx5e_ipsec *ipsec, 973 u32 prio) 974 { 975 struct mlx5e_ipsec_tx *tx = ipsec->tx; 976 struct mlx5_flow_table *ft; 977 int err; 978 979 mutex_lock(&tx->ft.mutex); 980 err = tx_get(mdev, ipsec, tx); 981 if (err) 982 goto err_get; 983 984 ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol; 985 if (IS_ERR(ft)) { 986 err = PTR_ERR(ft); 987 goto err_get_ft; 988 } 989 990 mutex_unlock(&tx->ft.mutex); 991 return ft; 992 993 err_get_ft: 994 tx_put(ipsec, tx); 995 err_get: 996 mutex_unlock(&tx->ft.mutex); 997 return ERR_PTR(err); 998 } 999 1000 static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio) 1001 { 1002 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1003 1004 mutex_lock(&tx->ft.mutex); 1005 if (tx->chains) 1006 ipsec_chains_put_table(tx->chains, prio); 1007 1008 tx_put(ipsec, tx); 1009 mutex_unlock(&tx->ft.mutex); 1010 } 1011 1012 static void tx_ft_put(struct mlx5e_ipsec *ipsec) 1013 { 1014 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1015 1016 mutex_lock(&tx->ft.mutex); 1017 tx_put(ipsec, tx); 1018 mutex_unlock(&tx->ft.mutex); 1019 } 1020 1021 static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec, 1022 u16 kspi) 1023 { 1024 /* Add IPsec indicator in metadata_reg_a. */ 1025 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1026 1027 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1028 misc_parameters_2.metadata_reg_a); 1029 MLX5_SET(fte_match_param, spec->match_value, 1030 misc_parameters_2.metadata_reg_a, 1031 MLX5_ETH_WQE_FT_META_IPSEC << 23 | kspi); 1032 } 1033 1034 static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec) 1035 { 1036 /* Add IPsec indicator in metadata_reg_a. */ 1037 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1038 1039 MLX5_SET(fte_match_param, spec->match_criteria, 1040 misc_parameters_2.metadata_reg_a, 1041 MLX5_ETH_WQE_FT_META_IPSEC << 23); 1042 MLX5_SET(fte_match_param, spec->match_value, 1043 misc_parameters_2.metadata_reg_a, 1044 0); 1045 } 1046 1047 static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) 1048 { 1049 /* Pass policy check before choosing this SA */ 1050 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1051 1052 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1053 misc_parameters_2.metadata_reg_c_0); 1054 MLX5_SET(fte_match_param, spec->match_value, 1055 misc_parameters_2.metadata_reg_c_0, reqid); 1056 } 1057 1058 static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec) 1059 { 1060 switch (upspec->proto) { 1061 case IPPROTO_UDP: 1062 if (upspec->dport) { 1063 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1064 spec->match_criteria, udp_dport); 1065 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1066 udp_dport, upspec->dport); 1067 } 1068 1069 if (upspec->sport) { 1070 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1071 spec->match_criteria, udp_sport); 1072 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1073 udp_dport, upspec->sport); 1074 } 1075 break; 1076 case IPPROTO_TCP: 1077 if (upspec->dport) { 1078 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1079 spec->match_criteria, tcp_dport); 1080 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1081 tcp_dport, upspec->dport); 1082 } 1083 1084 if (upspec->sport) { 1085 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1086 spec->match_criteria, tcp_sport); 1087 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1088 tcp_dport, upspec->sport); 1089 } 1090 break; 1091 default: 1092 return; 1093 } 1094 1095 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1096 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol); 1097 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto); 1098 } 1099 1100 static int tx_add_kspi_rule(struct mlx5e_ipsec_sa_entry *sa_entry, 1101 struct mlx5e_ipsec_tx *tx, 1102 struct mlx5_flow_act *flow_act, 1103 struct mlx5_flow_destination *dest, 1104 int num_dest) 1105 { 1106 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1107 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1108 struct mlx5_flow_handle *rule; 1109 struct mlx5_flow_spec *spec; 1110 int err; 1111 1112 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1113 if (!spec) 1114 return -ENOMEM; 1115 1116 setup_fte_no_frags(spec); 1117 setup_fte_reg_a_with_tag(spec, sa_entry->kspi); 1118 1119 rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, flow_act, dest, num_dest); 1120 if (IS_ERR(rule)) { 1121 err = PTR_ERR(rule); 1122 mlx5_core_err(mdev, "fail to add TX ipsec kspi rule err=%d\n", err); 1123 goto err_add_kspi_flow; 1124 } 1125 ipsec_rule->kspi_rule = rule; 1126 kvfree(spec); 1127 return 0; 1128 1129 err_add_kspi_flow: 1130 kvfree(spec); 1131 return err; 1132 } 1133 1134 static int tx_add_reqid_ip_rules(struct mlx5e_ipsec_sa_entry *sa_entry, 1135 struct mlx5e_ipsec_tx *tx, 1136 struct mlx5_flow_act *flow_act, 1137 struct mlx5_flow_destination *dest, 1138 int num_dest) 1139 { 1140 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1141 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1142 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1143 struct mlx5_flow_handle *rule; 1144 struct mlx5_flow_spec *spec; 1145 int err; 1146 1147 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1148 if (!spec) 1149 return -ENOMEM; 1150 1151 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 1152 1153 if(attrs->reqid) { 1154 setup_fte_no_frags(spec); 1155 setup_fte_reg_c0(spec, attrs->reqid); 1156 rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest); 1157 if (IS_ERR(rule)) { 1158 err = PTR_ERR(rule); 1159 mlx5_core_err(mdev, "fail to add TX ipsec reqid rule err=%d\n", err); 1160 goto err_add_reqid_rule; 1161 } 1162 ipsec_rule->reqid_rule = rule; 1163 memset(spec, 0, sizeof(*spec)); 1164 } 1165 1166 if (attrs->family == AF_INET) 1167 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1168 else 1169 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1170 setup_fte_no_frags(spec); 1171 1172 rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest); 1173 if (IS_ERR(rule)) { 1174 err = PTR_ERR(rule); 1175 mlx5_core_err(mdev, "fail to add TX ipsec ip rule err=%d\n", err); 1176 goto err_add_ip_rule; 1177 } 1178 ipsec_rule->rule = rule; 1179 kvfree(spec); 1180 return 0; 1181 1182 err_add_ip_rule: 1183 mlx5_del_flow_rules(&ipsec_rule->reqid_rule); 1184 err_add_reqid_rule: 1185 kvfree(spec); 1186 return err; 1187 } 1188 1189 static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1190 { 1191 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1192 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1193 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1194 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 1195 struct mlx5_flow_destination dest[2] = {}; 1196 struct mlx5_flow_act flow_act = {}; 1197 struct mlx5e_ipsec_tx *tx; 1198 struct mlx5_fc *counter; 1199 int err; 1200 1201 tx = tx_ft_get(mdev, ipsec); 1202 if (IS_ERR(tx)) 1203 return PTR_ERR(tx); 1204 1205 err = setup_pkt_reformat(mdev, attrs, &flow_act); 1206 if (err) 1207 goto err_pkt_reformat; 1208 1209 counter = mlx5_fc_create(mdev, false); 1210 if (IS_ERR(counter)) { 1211 err = PTR_ERR(counter); 1212 goto err_add_cnt; 1213 } 1214 1215 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 1216 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 1217 flow_act.flags |= FLOW_ACT_NO_APPEND; 1218 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | 1219 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1220 1221 if (attrs->drop) 1222 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1223 else 1224 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1225 1226 dest[0].ft = tx->ft.status; 1227 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1228 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1229 dest[1].counter_id = mlx5_fc_id(counter); 1230 1231 err = tx_add_kspi_rule(sa_entry, tx, &flow_act, dest, 2); 1232 if (err) { 1233 goto err_add_kspi_rule; 1234 } 1235 1236 err = tx_add_reqid_ip_rules(sa_entry, tx, &flow_act, dest, 2); 1237 if (err) { 1238 goto err_add_reqid_ip_rule; 1239 } 1240 1241 ipsec_rule->fc = counter; 1242 ipsec_rule->pkt_reformat = flow_act.pkt_reformat; 1243 return 0; 1244 1245 err_add_reqid_ip_rule: 1246 mlx5_del_flow_rules(&ipsec_rule->kspi_rule); 1247 err_add_kspi_rule: 1248 mlx5_fc_destroy(mdev, counter); 1249 err_add_cnt: 1250 if (flow_act.pkt_reformat) 1251 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 1252 err_pkt_reformat: 1253 tx_ft_put(ipsec); 1254 return err; 1255 } 1256 1257 static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1258 { 1259 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1260 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1261 struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx; 1262 struct mlx5_flow_destination dest[2] = {}; 1263 struct mlx5_flow_act flow_act = {}; 1264 struct mlx5_flow_handle *rule; 1265 struct mlx5_flow_spec *spec; 1266 struct mlx5_flow_table *ft; 1267 int err, dstn = 0; 1268 1269 ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio); 1270 if (IS_ERR(ft)) 1271 return PTR_ERR(ft); 1272 1273 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1274 if (!spec) { 1275 err = -ENOMEM; 1276 goto err_alloc; 1277 } 1278 1279 if (attrs->family == AF_INET) 1280 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1281 else 1282 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1283 1284 setup_fte_no_frags(spec); 1285 setup_fte_upper_proto_match(spec, &attrs->upspec); 1286 1287 switch (attrs->action) { 1288 case IPSEC_POLICY_IPSEC: 1289 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1290 err = setup_modify_header(mdev, attrs->reqid, 1291 IPSEC_DIR_OUTBOUND, &flow_act); 1292 if (err) 1293 goto err_mod_header; 1294 break; 1295 case IPSEC_POLICY_DISCARD: 1296 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | 1297 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1298 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1299 dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop); 1300 dstn++; 1301 break; 1302 default: 1303 err = -EINVAL; 1304 goto err_mod_header; 1305 } 1306 1307 flow_act.flags |= FLOW_ACT_NO_APPEND; 1308 dest[dstn].ft = tx->ft.sa; 1309 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1310 dstn++; 1311 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1312 if (IS_ERR(rule)) { 1313 err = PTR_ERR(rule); 1314 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 1315 goto err_action; 1316 } 1317 1318 kvfree(spec); 1319 pol_entry->ipsec_rule.rule = rule; 1320 pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 1321 return 0; 1322 1323 err_action: 1324 if (flow_act.modify_hdr) 1325 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 1326 err_mod_header: 1327 kvfree(spec); 1328 err_alloc: 1329 tx_ft_put_policy(pol_entry->ipsec, attrs->prio); 1330 return err; 1331 } 1332 1333 static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1334 { 1335 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1336 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1337 struct mlx5e_ipsec *ipsec = pol_entry->ipsec; 1338 struct mlx5_flow_destination dest[2]; 1339 struct mlx5_flow_act flow_act = {}; 1340 struct mlx5_flow_handle *rule; 1341 struct mlx5_flow_spec *spec; 1342 struct mlx5_flow_table *ft; 1343 struct mlx5e_ipsec_rx *rx; 1344 int err, dstn = 0; 1345 1346 rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6; 1347 ft = rx->chains ? ipsec_chains_get_table(rx->chains, attrs->prio) : rx->ft.pol; 1348 if (IS_ERR(ft)) 1349 return PTR_ERR(ft); 1350 1351 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1352 if (!spec) { 1353 err = -ENOMEM; 1354 goto err_alloc; 1355 } 1356 1357 switch (attrs->action) { 1358 case IPSEC_POLICY_IPSEC: 1359 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1360 break; 1361 case IPSEC_POLICY_DISCARD: 1362 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 1363 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1364 dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop); 1365 dstn++; 1366 break; 1367 default: 1368 err = -EINVAL; 1369 goto err_action; 1370 } 1371 1372 flow_act.flags |= FLOW_ACT_NO_APPEND; 1373 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1374 dest[dstn].ft = rx->ft.sa; 1375 dstn++; 1376 1377 if (attrs->family == AF_INET) 1378 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1379 else 1380 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1381 1382 setup_fte_no_frags(spec); 1383 setup_fte_upper_proto_match(spec, &attrs->upspec); 1384 if (attrs->vid != VLAN_NONE) 1385 setup_fte_vid(spec, attrs->vid); 1386 else 1387 setup_fte_no_vid(spec); 1388 1389 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1390 if (IS_ERR(rule)) { 1391 err = PTR_ERR(rule); 1392 mlx5_core_err(mdev, 1393 "Failed to add RX IPsec policy rule err=%d\n", err); 1394 goto err_action; 1395 } 1396 pol_entry->ipsec_rule.rule = rule; 1397 1398 /* Add also rule for zero vid */ 1399 if (attrs->vid == VLAN_NONE) { 1400 clear_fte_vid(spec); 1401 setup_fte_vid(spec, 0); 1402 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1403 if (IS_ERR(rule)) { 1404 err = PTR_ERR(rule); 1405 mlx5_core_err(mdev, 1406 "Failed to add RX IPsec policy rule err=%d\n", 1407 err); 1408 goto err_action; 1409 } 1410 pol_entry->ipsec_rule.vid_zero_rule = rule; 1411 } 1412 1413 kvfree(spec); 1414 return 0; 1415 1416 err_action: 1417 if (pol_entry->ipsec_rule.rule != NULL) 1418 mlx5_del_flow_rules(&pol_entry->ipsec_rule.rule); 1419 kvfree(spec); 1420 err_alloc: 1421 if (rx->chains != NULL) 1422 ipsec_chains_put_table(rx->chains, attrs->prio); 1423 return err; 1424 } 1425 1426 static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) 1427 { 1428 struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; 1429 struct mlx5_core_dev *mdev = ipsec->mdev; 1430 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1431 1432 mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); 1433 mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); 1434 kfree(rx_ipv4->fc); 1435 mlx5_fc_destroy(mdev, tx->fc->drop); 1436 mlx5_fc_destroy(mdev, tx->fc->cnt); 1437 kfree(tx->fc); 1438 } 1439 1440 static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) 1441 { 1442 struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; 1443 struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6; 1444 struct mlx5_core_dev *mdev = ipsec->mdev; 1445 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1446 struct mlx5e_ipsec_fc *fc; 1447 struct mlx5_fc *counter; 1448 int err; 1449 1450 fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); 1451 if (!fc) 1452 return -ENOMEM; 1453 1454 tx->fc = fc; 1455 counter = mlx5_fc_create(mdev, false); 1456 if (IS_ERR(counter)) { 1457 err = PTR_ERR(counter); 1458 goto err_tx_fc_alloc; 1459 } 1460 1461 fc->cnt = counter; 1462 counter = mlx5_fc_create(mdev, false); 1463 if (IS_ERR(counter)) { 1464 err = PTR_ERR(counter); 1465 goto err_tx_fc_cnt; 1466 } 1467 1468 fc->drop = counter; 1469 1470 fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); 1471 if (!fc) { 1472 err = -ENOMEM; 1473 goto err_tx_fc_drop; 1474 } 1475 1476 /* Both IPv4 and IPv6 point to same flow counters struct. */ 1477 rx_ipv4->fc = fc; 1478 rx_ipv6->fc = fc; 1479 counter = mlx5_fc_create(mdev, false); 1480 if (IS_ERR(counter)) { 1481 err = PTR_ERR(counter); 1482 goto err_rx_fc_alloc; 1483 } 1484 1485 fc->cnt = counter; 1486 counter = mlx5_fc_create(mdev, false); 1487 if (IS_ERR(counter)) { 1488 err = PTR_ERR(counter); 1489 goto err_rx_fc_cnt; 1490 } 1491 1492 fc->drop = counter; 1493 return 0; 1494 1495 err_rx_fc_cnt: 1496 mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); 1497 err_rx_fc_alloc: 1498 kfree(rx_ipv4->fc); 1499 err_tx_fc_drop: 1500 mlx5_fc_destroy(mdev, tx->fc->drop); 1501 err_tx_fc_cnt: 1502 mlx5_fc_destroy(mdev, tx->fc->cnt); 1503 err_tx_fc_alloc: 1504 kfree(tx->fc); 1505 return err; 1506 } 1507 1508 static int ipsec_status_rule(struct mlx5_core_dev *mdev, 1509 struct mlx5e_ipsec_rx *rx, 1510 struct mlx5_flow_destination *dest) 1511 { 1512 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1513 struct mlx5_flow_act flow_act = {}; 1514 struct mlx5_modify_hdr *modify_hdr; 1515 struct mlx5_flow_handle *rule; 1516 struct mlx5_flow_spec *spec; 1517 int err; 1518 1519 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1520 if (!spec) 1521 return -ENOMEM; 1522 1523 /* Action to copy 7 bit ipsec_syndrome to regB[24:30] */ 1524 MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); 1525 MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME); 1526 MLX5_SET(copy_action_in, action, src_offset, 0); 1527 MLX5_SET(copy_action_in, action, length, 7); 1528 MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1529 MLX5_SET(copy_action_in, action, dst_offset, 24); 1530 1531 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, 1532 1, action); 1533 1534 if (IS_ERR(modify_hdr)) { 1535 err = PTR_ERR(modify_hdr); 1536 mlx5_core_err(mdev, 1537 "fail to alloc ipsec copy modify_header_id err=%d\n", err); 1538 goto out_spec; 1539 } 1540 1541 /* create fte */ 1542 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | 1543 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1544 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1545 flow_act.modify_hdr = modify_hdr; 1546 1547 rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); 1548 if (IS_ERR(rule)) { 1549 err = PTR_ERR(rule); 1550 mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); 1551 goto out; 1552 } 1553 1554 kvfree(spec); 1555 rx->status.rule = rule; 1556 rx->status.modify_hdr = modify_hdr; 1557 return 0; 1558 1559 out: 1560 mlx5_modify_header_dealloc(mdev, modify_hdr); 1561 out_spec: 1562 kvfree(spec); 1563 return err; 1564 } 1565 1566 static void ipsec_fs_rx_roce_rules_destroy(struct mlx5e_ipsec_rx_roce *rx_roce) 1567 { 1568 if (!rx_roce->ns_rdma) 1569 return; 1570 1571 mlx5_del_flow_rules(&rx_roce->roce_miss.rule); 1572 mlx5_del_flow_rules(&rx_roce->rule); 1573 mlx5_destroy_flow_group(rx_roce->roce_miss.group); 1574 mlx5_destroy_flow_group(rx_roce->g); 1575 } 1576 1577 static void ipsec_fs_rx_catchall_rules_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) 1578 { 1579 mutex_lock(&rx->ft.mutex); 1580 mlx5_del_flow_rules(&rx->sa.rule); 1581 mlx5_destroy_flow_group(rx->sa.group); 1582 if (rx->chains == NULL) { 1583 mlx5_del_flow_rules(&rx->pol.rule); 1584 mlx5_destroy_flow_group(rx->pol.group); 1585 } 1586 mlx5_del_flow_rules(&rx->status.rule); 1587 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 1588 ipsec_fs_rx_roce_rules_destroy(&rx->roce); 1589 mutex_unlock(&rx->ft.mutex); 1590 } 1591 1592 static void ipsec_fs_rx_roce_table_destroy(struct mlx5e_ipsec_rx_roce *rx_roce) 1593 { 1594 if (!rx_roce->ns_rdma) 1595 return; 1596 1597 mlx5_destroy_flow_table(rx_roce->ft_rdma); 1598 mlx5_destroy_flow_table(rx_roce->ft); 1599 } 1600 1601 static void ipsec_fs_rx_table_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) 1602 { 1603 mutex_lock(&rx->ft.mutex); 1604 if (rx->chains) { 1605 ipsec_chains_destroy(rx->chains); 1606 } else { 1607 mlx5_del_flow_rules(&rx->pol.rule); 1608 mlx5_destroy_flow_table(rx->ft.pol); 1609 } 1610 mlx5_destroy_flow_table(rx->ft.sa); 1611 mlx5_destroy_flow_table(rx->ft.status); 1612 ipsec_fs_rx_roce_table_destroy(&rx->roce); 1613 mutex_unlock(&rx->ft.mutex); 1614 } 1615 1616 static void ipsec_roce_setup_udp_dport(struct mlx5_flow_spec *spec, u16 dport) 1617 { 1618 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1619 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 1620 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP); 1621 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport); 1622 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport); 1623 } 1624 1625 static int ipsec_roce_rx_rule_setup(struct mlx5_flow_destination *default_dst, 1626 struct mlx5e_ipsec_rx_roce *roce, struct mlx5_core_dev *mdev) 1627 { 1628 struct mlx5_flow_destination dst = {}; 1629 struct mlx5_flow_act flow_act = {}; 1630 struct mlx5_flow_handle *rule; 1631 struct mlx5_flow_spec *spec; 1632 int err = 0; 1633 1634 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1635 if (!spec) 1636 return -ENOMEM; 1637 1638 ipsec_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT); 1639 1640 //flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;//not needed it is added in command 1641 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 1642 dst.ft = roce->ft_rdma; 1643 1644 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1645 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1); 1646 if (IS_ERR(rule)) { 1647 err = PTR_ERR(rule); 1648 mlx5_core_err(mdev, "Fail to add RX roce ipsec rule err=%d\n", 1649 err); 1650 goto fail_add_rule; 1651 } 1652 1653 roce->rule = rule; 1654 1655 rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, default_dst, 1); 1656 if (IS_ERR(rule)) { 1657 err = PTR_ERR(rule); 1658 mlx5_core_err(mdev, "Fail to add RX roce ipsec miss rule err=%d\n", 1659 err); 1660 goto fail_add_default_rule; 1661 } 1662 1663 roce->roce_miss.rule = rule; 1664 1665 kvfree(spec); 1666 return 0; 1667 1668 fail_add_default_rule: 1669 mlx5_del_flow_rules(&roce->rule); 1670 fail_add_rule: 1671 kvfree(spec); 1672 return err; 1673 } 1674 1675 static int ipsec_roce_rx_rules(struct mlx5e_ipsec_rx *rx, struct mlx5_flow_destination *defdst, 1676 struct mlx5_core_dev *mdev) 1677 { 1678 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1679 struct mlx5_flow_group *g; 1680 void *outer_headers_c; 1681 u32 *in; 1682 int err = 0; 1683 int ix = 0; 1684 u8 *mc; 1685 1686 if (!rx->roce.ns_rdma) 1687 return 0; 1688 1689 in = kvzalloc(inlen, GFP_KERNEL); 1690 if (!in) 1691 return -ENOMEM; 1692 1693 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1694 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 1695 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 1696 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 1697 1698 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1699 MLX5_SET_CFG(in, start_flow_index, ix); 1700 ix += 1; 1701 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1702 g = mlx5_create_flow_group(rx->roce.ft, in); 1703 if (IS_ERR(g)) { 1704 err = PTR_ERR(g); 1705 mlx5_core_err(mdev, "Fail to create ipsec rx roce group at nic err=%d\n", err); 1706 goto fail_group; 1707 } 1708 rx->roce.g = g; 1709 1710 memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in)); 1711 MLX5_SET_CFG(in, start_flow_index, ix); 1712 ix += 1; 1713 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1714 g = mlx5_create_flow_group(rx->roce.ft, in); 1715 if (IS_ERR(g)) { 1716 err = PTR_ERR(g); 1717 mlx5_core_err(mdev, "Fail to create ipsec rx roce miss group at nic err=%d\n", 1718 err); 1719 goto fail_mgroup; 1720 } 1721 rx->roce.roce_miss.group = g; 1722 1723 err = ipsec_roce_rx_rule_setup(defdst, &rx->roce, mdev); 1724 if (err) 1725 goto fail_setup_rule; 1726 1727 kvfree(in); 1728 return 0; 1729 1730 fail_setup_rule: 1731 mlx5_destroy_flow_group(rx->roce.roce_miss.group); 1732 fail_mgroup: 1733 mlx5_destroy_flow_group(rx->roce.g); 1734 fail_group: 1735 kvfree(in); 1736 return err; 1737 } 1738 1739 static int ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv, 1740 struct mlx5e_ipsec_rx *rx, 1741 struct mlx5_flow_destination *defdst) 1742 { 1743 struct mlx5_core_dev *mdev = priv->mdev; 1744 struct mlx5_flow_destination dest[2] = {}; 1745 int err = 0; 1746 1747 mutex_lock(&rx->ft.mutex); 1748 /* IPsec RoCE RX rules */ 1749 err = ipsec_roce_rx_rules(rx, defdst, mdev); 1750 if (err) 1751 goto out; 1752 1753 /* IPsec Rx IP Status table rule */ 1754 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1755 if (rx->roce.ft) 1756 dest[0].ft = rx->roce.ft; 1757 else 1758 dest[0].ft = priv->fts.vlan.t; 1759 1760 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1761 dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); 1762 err = ipsec_status_rule(mdev, rx, dest); 1763 if (err) 1764 goto err_roce_rules_destroy; 1765 1766 if (!rx->chains) { 1767 /* IPsec Rx IP policy default miss rule */ 1768 err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, defdst); 1769 if (err) 1770 goto err_status_rule_destroy; 1771 } 1772 1773 /* FIXME: This is workaround to current design 1774 * which installs SA on firt packet. So we need to forward this 1775 * packet to the stack. It doesn't work with RoCE and eswitch traffic, 1776 */ 1777 err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, defdst); 1778 if (err) 1779 goto err_status_sa_rule_destroy; 1780 1781 mutex_unlock(&rx->ft.mutex); 1782 return 0; 1783 1784 err_status_sa_rule_destroy: 1785 if (!rx->chains) { 1786 mlx5_del_flow_rules(&rx->pol.rule); 1787 mlx5_destroy_flow_group(rx->pol.group); 1788 } 1789 err_status_rule_destroy: 1790 mlx5_del_flow_rules(&rx->status.rule); 1791 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 1792 err_roce_rules_destroy: 1793 ipsec_fs_rx_roce_rules_destroy(&rx->roce); 1794 out: 1795 mutex_unlock(&rx->ft.mutex); 1796 return err; 1797 } 1798 1799 static int ipsec_fs_rx_roce_tables_create(struct mlx5e_ipsec_rx *rx, 1800 int rx_init_level, int rdma_init_level) 1801 { 1802 struct mlx5_flow_table_attr ft_attr = {}; 1803 struct mlx5_flow_table *ft; 1804 int err = 0; 1805 1806 if (!rx->roce.ns_rdma) 1807 return 0; 1808 1809 ft_attr.max_fte = 2; 1810 ft_attr.level = rx_init_level; 1811 ft = mlx5_create_flow_table(rx->ns, &ft_attr); 1812 if (IS_ERR(ft)) { 1813 err = PTR_ERR(ft); 1814 return err; 1815 } 1816 rx->roce.ft = ft; 1817 1818 ft_attr.max_fte = 0; 1819 ft_attr.level = rdma_init_level; 1820 ft = mlx5_create_flow_table(rx->roce.ns_rdma, &ft_attr); 1821 if (IS_ERR(ft)) { 1822 err = PTR_ERR(ft); 1823 goto out; 1824 } 1825 rx->roce.ft_rdma = ft; 1826 1827 return 0; 1828 out: 1829 mlx5_destroy_flow_table(rx->roce.ft); 1830 rx->roce.ft = NULL; 1831 return err; 1832 } 1833 1834 static int ipsec_fs_rx_table_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx, 1835 int rx_init_level, int rdma_init_level) 1836 { 1837 struct mlx5_flow_namespace *ns = rx->ns; 1838 struct mlx5_flow_table *ft; 1839 int err = 0; 1840 1841 mutex_lock(&rx->ft.mutex); 1842 1843 /* IPsec Rx IP SA table create */ 1844 ft = ipsec_rx_ft_create(ns, rx_init_level + 1, 0, 1); 1845 if (IS_ERR(ft)) { 1846 err = PTR_ERR(ft); 1847 goto out; 1848 } 1849 rx->ft.sa = ft; 1850 1851 /* IPsec Rx IP Status table create */ 1852 ft = ipsec_rx_ft_create(ns, rx_init_level + 2, 0, 1); 1853 if (IS_ERR(ft)) { 1854 err = PTR_ERR(ft); 1855 goto err_sa_table_destroy; 1856 } 1857 rx->ft.status = ft; 1858 1859 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 1860 rx->chains = ipsec_chains_create(mdev, rx->ft.sa, 1861 MLX5_FLOW_NAMESPACE_KERNEL, 0, 1862 rx_init_level, &rx->ft.pol); 1863 if (IS_ERR(rx->chains)) { 1864 err = PTR_ERR(rx->chains); 1865 goto err_status_table_destroy; 1866 } 1867 } else { 1868 ft = ipsec_rx_ft_create(ns, rx_init_level, 0, 1); 1869 if (IS_ERR(ft)) { 1870 err = PTR_ERR(ft); 1871 goto err_status_table_destroy; 1872 } 1873 rx->ft.pol = ft; 1874 } 1875 1876 /* IPsec RoCE RX tables create*/ 1877 err = ipsec_fs_rx_roce_tables_create(rx, rx_init_level + 3, 1878 rdma_init_level); 1879 if (err) 1880 goto err_pol_table_destroy; 1881 1882 goto out; 1883 1884 err_pol_table_destroy: 1885 mlx5_destroy_flow_table(rx->ft.pol); 1886 err_status_table_destroy: 1887 mlx5_destroy_flow_table(rx->ft.status); 1888 err_sa_table_destroy: 1889 mlx5_destroy_flow_table(rx->ft.sa); 1890 out: 1891 mutex_unlock(&rx->ft.mutex); 1892 return err; 1893 } 1894 1895 #define NIC_RDMA_BOTH_DIRS_CAPS (MLX5_FT_NIC_RX_2_NIC_RX_RDMA | MLX5_FT_NIC_TX_RDMA_2_NIC_TX) 1896 1897 static void mlx5e_accel_ipsec_fs_init_roce(struct mlx5e_ipsec *ipsec) 1898 { 1899 struct mlx5_core_dev *mdev = ipsec->mdev; 1900 struct mlx5_flow_namespace *ns; 1901 1902 if ((MLX5_CAP_GEN_2(ipsec->mdev, flow_table_type_2_type) & 1903 NIC_RDMA_BOTH_DIRS_CAPS) != NIC_RDMA_BOTH_DIRS_CAPS) { 1904 mlx5_core_dbg(mdev, "Failed to init roce ns, capabilities not supported\n"); 1905 return; 1906 } 1907 1908 ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC); 1909 if (!ns) { 1910 mlx5_core_err(mdev, "Failed to init roce rx ns\n"); 1911 return; 1912 } 1913 1914 ipsec->rx_ipv4->roce.ns_rdma = ns; 1915 ipsec->rx_ipv6->roce.ns_rdma = ns; 1916 1917 ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC); 1918 if (!ns) { 1919 ipsec->rx_ipv4->roce.ns_rdma = NULL; 1920 ipsec->rx_ipv6->roce.ns_rdma = NULL; 1921 mlx5_core_err(mdev, "Failed to init roce tx ns\n"); 1922 return; 1923 } 1924 1925 ipsec->tx->roce.ns = ns; 1926 } 1927 1928 int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1929 { 1930 if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND) 1931 return tx_add_rule(sa_entry); 1932 1933 return rx_add_rule(sa_entry); 1934 } 1935 1936 void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1937 { 1938 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1939 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1940 1941 mlx5_del_flow_rules(&ipsec_rule->rule); 1942 mlx5_del_flow_rules(&ipsec_rule->kspi_rule); 1943 if (ipsec_rule->vid_zero_rule != NULL) 1944 mlx5_del_flow_rules(&ipsec_rule->vid_zero_rule); 1945 if (ipsec_rule->reqid_rule != NULL) 1946 mlx5_del_flow_rules(&ipsec_rule->reqid_rule); 1947 mlx5_fc_destroy(mdev, ipsec_rule->fc); 1948 mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); 1949 if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND) { 1950 tx_ft_put(sa_entry->ipsec); 1951 return; 1952 } 1953 1954 if (ipsec_rule->modify_hdr != NULL) 1955 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1956 } 1957 1958 int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1959 { 1960 if (pol_entry->attrs.dir == IPSEC_DIR_OUTBOUND) 1961 return tx_add_policy(pol_entry); 1962 1963 return rx_add_policy(pol_entry); 1964 } 1965 1966 void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1967 { 1968 struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; 1969 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1970 1971 mlx5_del_flow_rules(&ipsec_rule->rule); 1972 if (ipsec_rule->vid_zero_rule != NULL) 1973 mlx5_del_flow_rules(&ipsec_rule->vid_zero_rule); 1974 1975 if (pol_entry->attrs.dir == IPSEC_DIR_INBOUND) { 1976 struct mlx5e_ipsec_rx *rx; 1977 1978 rx = (pol_entry->attrs.family == AF_INET) 1979 ? pol_entry->ipsec->rx_ipv4 1980 : pol_entry->ipsec->rx_ipv6; 1981 if (rx->chains) 1982 ipsec_chains_put_table(rx->chains, 1983 pol_entry->attrs.prio); 1984 return; 1985 } 1986 1987 if (ipsec_rule->modify_hdr) 1988 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1989 1990 tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio); 1991 } 1992 1993 void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv) 1994 { 1995 /* Check if IPsec supported */ 1996 if (!priv->ipsec) 1997 return; 1998 1999 ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv4); 2000 ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6); 2001 } 2002 2003 int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv) 2004 { 2005 struct mlx5e_ipsec *ipsec = priv->ipsec; 2006 struct mlx5_flow_destination dest = {}; 2007 int err = 0; 2008 2009 /* Check if IPsec supported */ 2010 if (!ipsec) 2011 return 0; 2012 2013 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2014 dest.ft = priv->fts.vlan.t; 2015 err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv6, &dest); 2016 if (err) 2017 goto out; 2018 2019 err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv4, &dest); 2020 if (err) 2021 ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6); 2022 out: 2023 return err; 2024 } 2025 2026 void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv) 2027 { 2028 struct mlx5_core_dev *mdev = priv->mdev; 2029 struct mlx5e_ipsec *ipsec = priv->ipsec; 2030 2031 /* Check if IPsec supported */ 2032 if (!ipsec) 2033 return; 2034 2035 ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv6); 2036 ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv4); 2037 } 2038 2039 int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv) 2040 { 2041 struct mlx5e_ipsec *ipsec = priv->ipsec; 2042 int err = 0; 2043 2044 /* Check if IPsec supported */ 2045 if (!ipsec) 2046 return 0; 2047 2048 err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv4, 0, 0); 2049 if (err) 2050 goto out; 2051 2052 err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv6, 4, 1); 2053 if (err) { 2054 ipsec_fs_rx_table_destroy(priv->mdev, ipsec->rx_ipv4); 2055 goto out; 2056 } 2057 2058 priv->fts.ipsec_ft = priv->ipsec->rx_ipv4->ft.pol; 2059 out: 2060 return err; 2061 } 2062 2063 void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) 2064 { 2065 WARN_ON(ipsec->tx->ft.refcnt); 2066 mutex_destroy(&ipsec->rx_ipv6->ft.mutex); 2067 mutex_destroy(&ipsec->rx_ipv4->ft.mutex); 2068 mutex_destroy(&ipsec->tx->ft.mutex); 2069 ipsec_fs_destroy_counters(ipsec); 2070 kfree(ipsec->rx_ipv6); 2071 kfree(ipsec->rx_ipv4); 2072 kfree(ipsec->tx); 2073 } 2074 2075 int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) 2076 { 2077 struct mlx5_flow_namespace *tns, *rns; 2078 int err = -ENOMEM; 2079 2080 tns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); 2081 if (!tns) 2082 return -EOPNOTSUPP; 2083 2084 rns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_KERNEL); 2085 if (!rns) 2086 return -EOPNOTSUPP; 2087 2088 ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); 2089 if (!ipsec->tx) 2090 return -ENOMEM; 2091 2092 ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); 2093 if (!ipsec->rx_ipv4) 2094 goto err_tx; 2095 2096 ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); 2097 if (!ipsec->rx_ipv6) 2098 goto err_rx_ipv4; 2099 2100 err = ipsec_fs_init_counters(ipsec); 2101 if (err) 2102 goto err_rx_ipv6; 2103 2104 ipsec->tx->ns = tns; 2105 mutex_init(&ipsec->tx->ft.mutex); 2106 ipsec->rx_ipv4->ns = rns; 2107 ipsec->rx_ipv6->ns = rns; 2108 mutex_init(&ipsec->rx_ipv4->ft.mutex); 2109 mutex_init(&ipsec->rx_ipv6->ft.mutex); 2110 2111 mlx5e_accel_ipsec_fs_init_roce(ipsec); 2112 2113 return 0; 2114 2115 err_rx_ipv6: 2116 kfree(ipsec->rx_ipv6); 2117 err_rx_ipv4: 2118 kfree(ipsec->rx_ipv4); 2119 err_tx: 2120 kfree(ipsec->tx); 2121 return err; 2122 } 2123 2124 void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry) 2125 { 2126 struct mlx5e_ipsec_sa_entry sa_entry_shadow = {}; 2127 int err; 2128 2129 memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry)); 2130 memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule)); 2131 2132 err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow); 2133 if (err) 2134 return; 2135 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 2136 memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry)); 2137 } 2138